aboutsummaryrefslogtreecommitdiffstats
path: root/nbs/core.ipynb
diff options
context:
space:
mode:
Diffstat (limited to 'nbs/core.ipynb')
-rw-r--r--nbs/core.ipynb374
1 files changed, 372 insertions, 2 deletions
diff --git a/nbs/core.ipynb b/nbs/core.ipynb
index b3f58f4..6e050ae 100644
--- a/nbs/core.ipynb
+++ b/nbs/core.ipynb
@@ -30,8 +30,11 @@
"outputs": [],
"source": [
"#| export\n",
- "import inspect\n",
- "from typing import Callable"
+ "import enum,inspect,re\n",
+ "from typing import Callable\n",
+ "\n",
+ "import nbformat\n",
+ "from nbformat import NotebookNode"
]
},
{
@@ -41,6 +44,9 @@
"metadata": {},
"outputs": [],
"source": [
+ "import contextlib\n",
+ "from pathlib import Path\n",
+ "from tempfile import TemporaryDirectory\n",
"from nb import test"
]
},
@@ -55,6 +61,370 @@
},
{
"cell_type": "markdown",
+ "id": "bdc4df64",
+ "metadata": {},
+ "source": [
+ "## Notebooks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3f99ca39",
+ "metadata": {},
+ "source": [
+ "### Reading Notebooks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "268db6ac",
+ "metadata": {},
+ "source": [
+ "To read in `ipynb` files, we can use a library provided Jupyter called `nbformat`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d9d0a18c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "type PathOrFileLike = str|Path|IO[str]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ca71fdf5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def read_nb(\n",
+ " fp:PathOrFileLike, # Where to read the notebook from.\n",
+ " v:int=4, # The notebook format major version to read the notebook as.\n",
+ ")->NotebookNode:\n",
+ " \"Read in a notebook as a dictionary-like NotebookNode.\"\n",
+ " return nbformat.read(fp, as_version=v)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b4ca48d4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "nb = nbformat.v4.new_notebook()\n",
+ "with TemporaryDirectory() as tmp, contextlib.chdir(tmp):\n",
+ " with open('nb.ipynb', mode='w+') as fp: nbformat.write(nb, fp)\n",
+ " for fp in ['nb.ipynb', Path('nb.ipynb')]:\n",
+ " nb = read_nb(fp)\n",
+ " test.equal(list(nb.keys()), ['cells', 'metadata', 'nbformat', 'nbformat_minor'])\n",
+ " test.equal(type(nb), NotebookNode)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a9abdd44",
+ "metadata": {},
+ "source": [
+ "We can see that the top-level format of a notebook is:\n",
+ "- `cells`: A list of objects for each cell in the notebook.\n",
+ "- `metadata`: Free-form notebook level metadata used by tools and editors.\n",
+ "- `nbformat`: Notebook format major version\n",
+ "- `nbformat_minor`: Notebook format minor version"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "772d813e",
+ "metadata": {},
+ "source": [
+ "### Filtering Notebooks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8cf77bcc",
+ "metadata": {},
+ "source": [
+ "Within cells, there are generally 3 types of cells:\n",
+ "- `code`\n",
+ "- `markdown`\n",
+ "- `raw`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2ffd7bcf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "class CellType(enum.StrEnum):\n",
+ " \"Possible Jupyter notebook cell types.\"\n",
+ " CODE = 'code'\n",
+ " MD = 'markdown'\n",
+ " RAW = 'raw'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a5b24f3a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def filter_cells(\n",
+ " nb:NotebookNode, # A notebook that has been read in via `read_nb`.\n",
+ " ct:CellType, # The cells to filter down to.\n",
+ ")->list[NotebookNode]:\n",
+ " \"Retrieve specific cell types from a notebook.\"\n",
+ " return list(filter(lambda c: c['cell_type'] == ct, nb['cells']))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "807f0146",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "nb = nbformat.v4.new_notebook(cells=[\n",
+ " nbformat.v4.new_code_cell(),\n",
+ " nbformat.v4.new_markdown_cell(),\n",
+ " nbformat.v4.new_raw_cell(),\n",
+ "])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5b89c5d8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "code_cell = filter_cells(nb, CellType.CODE)[0]\n",
+ "test.equal(list(code_cell.keys()), ['id', 'cell_type', 'metadata', 'execution_count', 'source', 'outputs'])\n",
+ "test.equal(code_cell['cell_type'], CellType.CODE)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "170144a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "md_cell = filter_cells(nb, CellType.MD)[0]\n",
+ "test.equal(list(md_cell.keys()), ['id', 'cell_type', 'source', 'metadata'])\n",
+ "test.equal(md_cell['cell_type'], CellType.MD)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "edbe00dd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "raw_cell = filter_cells(nb, CellType.RAW)[0]\n",
+ "test.equal(list(raw_cell.keys()), ['id', 'cell_type', 'source', 'metadata'])\n",
+ "test.equal(raw_cell['cell_type'], CellType.RAW)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "558e2214",
+ "metadata": {},
+ "source": [
+ "Within each cell object, they will all have:\n",
+ "- `id`: A unique identifier for that cell.\n",
+ "- `cell_type`: One of the 3 cell type names.\n",
+ "- `source`: The content of the cell, whether it is prose or code.\n",
+ "- `metadata`: Free-form metadata for that cell used by various tools/editors.\n",
+ "\n",
+ "Additionally, `code` cell types will have:\n",
+ "- `outputs`: The stdout/stderr of the cell after the code is executed.\n",
+ "- `execution_count`: The number of times the cell was run."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e4d56d07",
+ "metadata": {},
+ "source": [
+ "### Directives"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "97162699",
+ "metadata": {},
+ "source": [
+ "A regular expression ran against each line in a cell should allow us to find each directive we support. Since we'll need to re-create this regression expression for a few directives, we'll create a small helper."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "40433e19",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def mk_directive_re(\n",
+ " name:str, # The name of the directive\n",
+ " arg_re:str|None=None, # A regex pattern to match an argument for the directive\n",
+ ")->re.Pattern:\n",
+ " \"Compile a regex to match a directive, optionally supporting an argument.\"\n",
+ " arg = rf':\\s*{arg_re}' if arg_re else ''\n",
+ " return re.compile(rf'^\\s*#\\|\\s*{name}{arg}\\s*$')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ac907b7c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "_TEST_DIRECTIVE_RE = mk_directive_re('test')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "40f7de66",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_dir_match = [\n",
+ " ' #| test \\t \\n', # Lots of whitespace around each token\n",
+ " '#|test', # No whitespace around each token\n",
+ "]\n",
+ "for line in test_dir_match:\n",
+ " test.is_true(_TEST_DIRECTIVE_RE.match(line))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f115789b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_dir_no_match = [\n",
+ " '| test', # Missing #\n",
+ " '# test', # Missing |\n",
+ " '#| testt', # Misspelled directive\n",
+ " '#| test test', # Extra word after directive\n",
+ "]\n",
+ "for line in test_dir_no_match:\n",
+ " test.is_false(_TEST_DIRECTIVE_RE.match(line))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3f58f798",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "_TEST_ARG_DIRECTIVE_RE = mk_directive_re('test', arg_re=r'\\w')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2bbc9f93",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_dir_arg_match = [\n",
+ " ' #| test: \\t a \\n', # Lots of whitespace around each token\n",
+ " '#|test:b', # No whitespace around each token\n",
+ "]\n",
+ "for line in test_dir_arg_match:\n",
+ " test.is_true(_TEST_ARG_DIRECTIVE_RE.match(line))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c994255e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_arg_dir_no_match = [\n",
+ " '#| test a', # Missing :\n",
+ " '#| test: ', # Missing arg\n",
+ " '#| test: aa', # Invalid arg (should be only 1 letter)\n",
+ "]\n",
+ "for line in test_dir_no_match + test_arg_dir_no_match:\n",
+ " test.is_false(_TEST_ARG_DIRECTIVE_RE.match(line))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "535b3ce2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def has_directive(src:str)->bool:\n",
+ " \"Check whether a line or block of source code has a nb directive.\"\n",
+ " from nb.export import DIRECTIVES as EXPORT_DIRECTIVES\n",
+ " return any(any(r.match(l) for r in EXPORT_DIRECTIVES) for l in src.splitlines())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b42fe082",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test.is_true(has_directive('#| default_exp: bootstrap'))\n",
+ "test.is_true(has_directive('#| export'))\n",
+ "test.is_true(has_directive('max(1,20)\\n#| export'))\n",
+ "test.is_false(has_directive('# comment\\nmax(1, 20)\\n'))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d02bf0bc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def strip_directives(src:str|list[str])->str:\n",
+ " \"Remove all nb directives from the source code.\"\n",
+ " src = src if isinstance(src, list) else src.splitlines()\n",
+ " return \"\\n\".join(filter(lambda l: not has_directive(l), src))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9eca0c79",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test.equal(strip_directives('#| export\\na=5'), 'a=5')\n",
+ "test.equal(strip_directives('#| default_exp: bootstrap\\na=5'), 'a=5')\n",
+ "test.equal(strip_directives(['a=10', 'a+=10', '#|export']), 'a=10\\na+=10')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
"id": "2b6822dd",
"metadata": {},
"source": [