aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEugene Triguba <hello@eugenetriguba.com>2026-05-19 01:48:15 +0000
committerEugene Triguba <hello@eugenetriguba.com>2026-05-19 01:48:15 +0000
commit466b70b4e31c625185c5f70eb7be13c25c7df421 (patch)
tree050d2ac81defb5b0d31a664bea23940a2d751503
downloadnb-466b70b4e31c625185c5f70eb7be13c25c7df421.zip
Initial Commit
-rw-r--r--.gitignore42
-rw-r--r--LICENSE26
-rw-r--r--README.md1
-rw-r--r--nb/__init__.py0
-rw-r--r--nb/core.py36
-rw-r--r--nb/export.py143
-rw-r--r--nb/test.py70
-rw-r--r--nbs/core.ipynb299
-rw-r--r--nbs/export.ipynb950
-rw-r--r--nbs/test.ipynb418
-rw-r--r--pyproject.toml25
11 files changed, 2010 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..739c482
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,42 @@
+# Editors
+tags
+
+# Distribution / packaging
+.Python
+env/
+build/
+conda/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual envs
+.venv
+
+# pyenv
+.python-version
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+
+# macOS
+.DS_Store
+
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..96423e5
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,26 @@
+Copyright 2026 Eugene Triguba
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS
+IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0ae01fb
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# nb
diff --git a/nb/__init__.py b/nb/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/nb/__init__.py
diff --git a/nb/core.py b/nb/core.py
new file mode 100644
index 0000000..0f2769e
--- /dev/null
+++ b/nb/core.py
@@ -0,0 +1,36 @@
+# AUTOGENERATED! DO NOT EDIT!
+
+# cell: 6a2cc7f8-6c65-4a7c-890c-3cf9d1cd314e
+import inspect
+from typing import Callable
+from nb import test
+
+# cell: c4875cc0-2d87-4c87-8789-7dc950be06c6
+def callable_name(c: Callable) -> str:
+ "Get the name of the callable `c`."
+ return getattr(c, '__name__', c.__class__.__name__)
+
+# cell: f2c70dd3
+def pubattr(o):
+ "Retrieve the public attributes of the object `o`."
+ return [a for a in dir(o) if not a.startswith('_')]
+
+# cell: 78d61a07
+def patch_to(o: object) -> Callable:
+ "Decorator that takes in a object and attaches the decorated function onto it."
+ def inner(fn: Callable) -> Callable:
+ nm = callable_name(fn)
+ setattr(o, nm, fn)
+ return fn
+ return inner
+
+# cell: 42580f8b
+@patch_to(_T)
+def say_hello(self: _T, name: str) -> str:
+ return f'Hello, {name}'
+
+# cell: 4eb650ab
+def patch(fn: Callable) -> Callable:
+ "Decorator to patch the object of the type-hinted 'self' argument of `fn` with `fn`."
+ self = inspect.get_annotations(fn)['self']
+ return patch_to(self)(fn)
diff --git a/nb/export.py b/nb/export.py
new file mode 100644
index 0000000..d140293
--- /dev/null
+++ b/nb/export.py
@@ -0,0 +1,143 @@
+# AUTOGENERATED! DO NOT EDIT!
+
+# cell: 96525ac4
+import ast
+import enum
+import pathlib
+import re
+from typing import IO
+
+import nbformat
+
+# cell: 27202360
+def read_nb(
+ fp:str|IO[str], # a path or file-like object to read the notebook from
+ v:int=4, # the notebook format major version to read the notebook as
+)->dict:
+ "Read in a notebook as a dictionary."
+ return nbformat.read(fp, as_version=v)
+
+# cell: 1ee0d459
+class CellType(enum.StrEnum):
+ "Possible Jupyter notebook cell types."
+ CODE = 'code'
+ MD = 'markdown'
+ RAW = 'raw'
+
+# cell: 133ebd88
+def filter_cells(
+ nb:dict, # A notebook that has been read in via `read_nb`
+ ct:CellType, # The cells to filter down to
+)->list[dict]:
+ "Retrieve specific cell types from a notebook."
+ return list(filter(lambda c: c['cell_type'] == ct, nb['cells']))
+
+# cell: 83a898c2
+def _mk_directive_re(
+ name:str, # The name of the directive
+ arg_re:str|None=None, # A regex pattern to match an argument for the directive
+)->re.Pattern:
+ "Compile a regex to match a directive, optionally supporting an argument."
+ arg = rf':\s*{arg_re}' if arg_re else ''
+ return re.compile(rf'^\s*#\|\s*{name}{arg}\s*$')
+
+# cell: ac846aaa
+DIRECTIVE_DEFAULT_EXP_RE = _mk_directive_re('default_exp', arg_re=r'(?P<module>\w+(\.\w+)*)')
+
+# cell: 5b4e8fe6
+DIRECTIVE_EXPORT_RE = _mk_directive_re('export')
+
+# cell: 54459016
+DIRECTIVE_NOEXPORT_RE = _mk_directive_re('noexport')
+
+# cell: 828159b2
+DIRECTIVES = [DIRECTIVE_DEFAULT_EXP_RE, DIRECTIVE_EXPORT_RE, DIRECTIVE_NOEXPORT_RE]
+
+# cell: 10232dd8
+def has_import(src:str)->bool:
+ "Check whether a block of Python source code has any import statements."
+ try: return any(isinstance(node, (ast.Import, ast.ImportFrom)) for node in ast.walk(ast.parse(src)))
+ except SyntaxError: return False
+
+# cell: d327883e
+def has_directive(src:str)->bool:
+ "Check whether a line or block of source code has a nb directive."
+ return any(any(r.match(l) for r in DIRECTIVES) for l in src.splitlines())
+
+# cell: b6f47497
+def strip_directives(src:str|list[str])->str:
+ "Remove all nb directives from the source code."
+ src = src if isinstance(src, list) else src.splitlines()
+ return "\n".join(filter(lambda l: not has_directive(l), src))
+
+# cell: d1f32654
+def is_cell_exported(cell:dict)->bool:
+ "Check whether the code cell should be exported."
+ src = cell['source']
+ lines = src.splitlines()
+ if any(DIRECTIVE_NOEXPORT_RE.match(l) for l in lines): return False
+ if any(DIRECTIVE_EXPORT_RE.match(l) for l in lines): return True
+ return has_import(src)
+
+# cell: ddc5c9e6
+def get_default_export(cells:list[dict])->str:
+ "Retrieve the `default_exp` directive argument from a list of cells."
+ matches = [
+ m.group('module')
+ for c in cells
+ for l in c['source'].splitlines()
+ if c['cell_type'] == CellType.CODE and (m:=DIRECTIVE_DEFAULT_EXP_RE.match(l))
+ ]
+ if len(matches) > 1: raise ValueError('More than one default_exp directive')
+ if len(matches) == 0: raise ValueError('Missing default_exp directive')
+ return matches[0]
+
+# cell: 9ce855f0
+def find_cells_to_export(
+ fp:str|IO[str], # a path or file-like object to read the notebook from
+)->tuple[str,list[dict]]: # returns the module name and list of cells to export
+ "Find the code cells in an ipynb file to export out."
+ code_cells = filter_cells(read_nb(fp), CellType.CODE)
+ module = get_default_export(code_cells)
+ cells = [
+ {**c, 'source': strip_directives(c['source'])}
+ for c in code_cells
+ if is_cell_exported(c)
+ ]
+ return module, cells
+
+# cell: 3070f710
+def fmt_export_cell(cell:dict)->dict:
+ "Format a code cell's source with metadata to prepare for export."
+ cell['source'] = f'# cell: {cell["id"]}\n' + cell['source'] + '\n'
+ return cell
+
+# cell: 633e256b
+def mk_py_mod_scaffold(
+ mod:str, # The Python module name or dotted path e.g. mod.mymod.file
+ src_dir:str|pathlib.Path, # The Python package source code directory scaffold
+):
+ "Create all intermediary __init__.py files and directories, if any, and the final .py file."
+ if not mod: raise ValueError('The module path cannot be empty')
+ src_dir = pathlib.Path(src_dir)
+ mod_file = (src_dir/pathlib.Path(*mod.split('.'))).with_suffix('.py')
+ mod_file.parent.mkdir(parents=True, exist_ok=True)
+ mod_file.touch()
+ pkg_dirs = list(mod_file.parents)
+ for p in pkg_dirs[:pkg_dirs.index(src_dir) + 1]:
+ (p / '__init__.py').touch()
+
+# cell: f3ab7819
+def export_nb(
+ fp:str|IO[str], # The notebook to export
+ src_dir:str|pathlib.Path='.', # The source directory to export the Python module to.
+):
+ "Export the exportable code cells to a Python file."
+ module, cells = find_cells_to_export(fp)
+ mk_py_mod_scaffold(module, src_dir)
+ exp_path = (pathlib.Path(src_dir)/module).with_suffix('.py')
+ src = '\n'.join(
+ ['# AUTOGENERATED! DO NOT EDIT!\n'] +
+ list(map(lambda c: fmt_export_cell(c)['source'], cells))
+ )
+ pathlib.Path(exp_path).write_text(src)
diff --git a/nb/test.py b/nb/test.py
new file mode 100644
index 0000000..8bc6df4
--- /dev/null
+++ b/nb/test.py
@@ -0,0 +1,70 @@
+# AUTOGENERATED! DO NOT EDIT!
+
+# cell: dd03783f-252c-40f5-b92f-0f3f9278ce09
+import ast,copy,contextlib,inspect,textwrap,traceback
+from typing import Any,Generator
+from dataclasses import dataclass
+
+# cell: a6c67919
+def get_call_args(frames_up: int = 1) -> tuple[list[str], dict[str,Any]]:
+ "Get the source code expressions of the caller's arguments."
+ fm_info = inspect.getouterframes(inspect.currentframe())[frames_up+1]
+ src, _ = inspect.getsourcelines(fm_info.frame)
+ srcl = src[fm_info.positions.lineno - 1:fm_info.positions.end_lineno]
+ if len(srcl) == 0:
+ raise RuntimeError(
+ "`get_call_args` is designed to be called from within a function, " +
+ "not at the top-level. If you want to call it at the top level, pass `frames_up=0`."
+ )
+ elif len(srcl) == 1:
+ fn_src = srcl[0][fm_info.positions.col_offset:fm_info.positions.end_col_offset]
+ else:
+ fn_src = "".join(
+ [srcl[0][fm_info.positions.col_offset:]] +
+ srcl[1:-1] +
+ [srcl[-1][:fm_info.positions.end_col_offset]]
+ )
+ call_node = ast.parse(fn_src).body[0].value
+ args_src = [ast.unparse(arg) for arg in call_node.args]
+ kwargs_src = {kw.arg: ast.unparse(kw.value) for kw in call_node.keywords}
+ return args_src, kwargs_src
+
+# cell: 623a480c
+@dataclass
+class ExceptionInfo:
+ type_: type[Exception] | None = None
+ value: Exception | None = None
+ traceback: str | None = None
+
+# cell: 0ad5fcef-f04d-469d-8ad1-94d3a98694e4
+@contextlib.contextmanager
+def raises(
+ exc: type[Exception] = Exception, # The exception type to check is raised.
+ contains: str | None = None, # Optionally, check whether the raised exception contains this message.
+) -> Generator[ExceptionInfo, None, None]:
+ "Asserts that the body of the context manager raises `exc`."
+ raised = True
+ try:
+ info = ExceptionInfo()
+ yield info
+ raised = False
+ except exc as e:
+ info.type_ = type(e)
+ info.value = e
+ info.traceback = traceback.format_tb(e.__traceback__)
+ if contains and contains not in str(e):
+ raise AssertionError(f"Raised {e} but message does not contain {contains}") from e
+ if not raised:
+ raise AssertionError(f"Did not raise {e}")
+
+# cell: 2b503871-3f39-4013-a7ae-f5143af3138e
+def equal(a, b):
+ "Assert that a == b."
+ args_src, _ = get_call_args()
+ if a != b:
+ msg = (
+ f"FAIL: {args_src[0]} != {args_src[1]}\n" +
+ ' ' * (len(AssertionError.__name__)+2) +
+ f" {a!r} != {b!r}"
+ )
+ raise AssertionError(msg)
diff --git a/nbs/core.ipynb b/nbs/core.ipynb
new file mode 100644
index 0000000..24e0330
--- /dev/null
+++ b/nbs/core.ipynb
@@ -0,0 +1,299 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aed9daa9-f05d-4e25-b9a5-a74d0211ef6f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "596de171-5567-43c1-98ce-66eccc28bbe9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| default_exp: core"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6a2cc7f8-6c65-4a7c-890c-3cf9d1cd314e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import inspect\n",
+ "from typing import Callable\n",
+ "from nb import test"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8485c16a",
+ "metadata": {},
+ "source": [
+ "# Core\n",
+ "> Fundamental Utilities"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c4875cc0-2d87-4c87-8789-7dc950be06c6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def callable_name(c: Callable) -> str:\n",
+ " \"Get the name of the callable `c`.\"\n",
+ " return getattr(c, '__name__', c.__class__.__name__)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "406d605d",
+ "metadata": {},
+ "source": [
+ "This works for any function:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0e9c39c5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert callable_name(max) == \"max\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d66b2d42",
+ "metadata": {},
+ "source": [
+ "And for any classes:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "056cb776",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class _A: pass\n",
+ "assert callable_name(_A) == \"_A\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f2c70dd3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def pubattr(o):\n",
+ " \"Retrieve the public attributes of the object `o`.\"\n",
+ " return [a for a in dir(o) if not a.startswith('_')]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "53cb2bd5",
+ "metadata": {},
+ "source": [
+ "`pubattr` will hide any attribute on an object if it starts with an underscore."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ac30af03",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "_A.print = print\n",
+ "_A._private_print = print\n",
+ "_A.__python_internal__ = print\n",
+ "assert pubattr(_A) == [\"print\"]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f892f4f1",
+ "metadata": {},
+ "source": [
+ "## Class Definitions"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4008335c",
+ "metadata": {},
+ "source": [
+ "Iterative notebook-driven development makes class definitions tricky. Typically, you'd write a clas definition in one monolithic block, which leaves no room for prose or incremental execution. Inspired by the wonderful [fastcore](https://github.com/AnswerDotAI/fastcore) library, we'll add `patch` and `patch_to` function to enable piecemeal class definition via monkeypatching."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "78d61a07",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def patch_to(o: object) -> Callable:\n",
+ " \"Decorator that takes in a object and attaches the decorated function onto it.\"\n",
+ " def inner(fn: Callable) -> Callable:\n",
+ " nm = callable_name(fn)\n",
+ " setattr(o, nm, fn)\n",
+ " return fn\n",
+ " return inner"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0d4bb2e6",
+ "metadata": {},
+ "source": [
+ "Let's try it out. We'll create a dummy class named `_T`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c67a3c29",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class _T: pass"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b7d0a482",
+ "metadata": {},
+ "source": [
+ "Now, we'll attach on a function to `_T` in a separate code cell."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "42580f8b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "@patch_to(_T)\n",
+ "def say_hello(self: _T, name: str) -> str:\n",
+ " return f'Hello, {name}'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0edc152b",
+ "metadata": {},
+ "source": [
+ "`_T` should have a `say_hello` function on it. Let's confirm."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "404bd72f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "_t = _T()\n",
+ "test.equal(_t.say_hello('eugene'), 'Hello, eugene')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4acd774c",
+ "metadata": {},
+ "source": [
+ "That's neat and all but notice that `self: _T` and `@patch_to(_T)` are both two different ways to provide the same sort of information (attach this method onto `_T`).\n",
+ "\n",
+ "What if we could look at the type hint to determine which object to patch?\n",
+ "\n",
+ "We already have `patch_to`, so let's create a `patch` decorator. All we'll need to do inside `patch` is find the object to patch and call `patch_to`. If we assume the object we're patching will always be called `self` i.e. patching a class, then this should be easy-peasy."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4eb650ab",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def patch(fn: Callable) -> Callable:\n",
+ " \"Decorator to patch the object of the type-hinted 'self' argument of `fn` with `fn`.\"\n",
+ " self = inspect.get_annotations(fn)['self']\n",
+ " return patch_to(self)(fn)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "97378af3",
+ "metadata": {},
+ "source": [
+ "Let's see if we can try `patch` out now with our dummy `_T` class."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "601e27f2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "@patch\n",
+ "def say_goodbye(self: _T, name: str) -> str:\n",
+ " return f'Goodbye, {name}'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d001522d",
+ "metadata": {},
+ "source": [
+ "Can our `_t` object now say goodbye?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6aa1ffbc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test.equal(_t.say_goodbye('eugene'), 'Goodbye, eugene')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "12b62cf8-2193-4f03-b80a-46304d73896e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| noexport\n",
+ "import nb.export; nb.export.export_nb('core.ipynb', src_dir='../nb/')"
+ ]
+ }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/nbs/export.ipynb b/nbs/export.ipynb
new file mode 100644
index 0000000..d662f7d
--- /dev/null
+++ b/nbs/export.ipynb
@@ -0,0 +1,950 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b92ad576",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| default_exp: export"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "96525ac4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ast\n",
+ "import enum\n",
+ "import pathlib\n",
+ "import re\n",
+ "from typing import IO\n",
+ "\n",
+ "import nbformat"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a682b39f",
+ "metadata": {},
+ "source": [
+ "# Exporting Notebooks to Python Files"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a21a7e2c",
+ "metadata": {},
+ "source": [
+ "To get our notebook-driven source code bootstrapped, we'll first need to be able to convert `ipynb` files to `py` files. Without that, we'd be unable to later import code we want to use in one notebook from another notebook because it's not in a plain importable Python file yet.\n",
+ "\n",
+ "To accomplish that, for any given code well, we'll need to know:\n",
+ "- Are we exporting it to a Python file?\n",
+ "- What Python file are we exporting it to?\n",
+ "\n",
+ "The approach tools like [nbdev](https://nbdev.fast.ai/) or [nbexport](https://github.com/danhorner/nbexport) take is to have certain directives that determine the code export behavior:\n",
+ "- `#|default_exp: mymodule`: Set the output module for this notebook\n",
+ "- `#|export`: Export this cell to the corresponding Python file\n",
+ "- `#|noexport`: Prevent export of this cell. Useful to prevent export of something that would be automatically exported.\n",
+ "\n",
+ "Furthermore, beyond the explicit directives, nbdev and nbexport both use automatic exports of cells with imports in them for less annotation overhead since nearly every import in a notebook is needed by the exported code. This is the primary rationale for a `#|noexport` since sometimes a import is purely for exploratory programming, and you won't use it in the final artifact."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3b670537",
+ "metadata": {},
+ "source": [
+ "To export a notebook, we'll need to:\n",
+ "1. Read in a ipynb notebook\n",
+ "2. Retrieve all code cells from the notebook\n",
+ "3. Find all cells that should be exported:\n",
+ " - Each notebook must have 1 (and only 1) `#| default_exp`. When a `#| default_exp` is found, it must be a single word e.g. `mymodule` or a dotted path e.g. `mymodules.other_module`. If it is a dotted path, the submodules created should have `__init__.py` files also created alongside them.\n",
+ " - Any cells with an `#| export` explicit directive or cells that have an import and *don't* have an explicit `#| noexport` directive should be marked for export.\n",
+ "4. Write out each cell sequentially into the Python file, stripping out any directives. When each cell is written out, it should have a comment above it describing the cell ID it came from and the path to the notebook it came from."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f1c95690",
+ "metadata": {},
+ "source": [
+ "## Reading Notebooks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "037fb28c",
+ "metadata": {},
+ "source": [
+ "To read in `ipynb` files, we can use a library provided Jupyter called `nbformat`:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "39f9ece6",
+ "metadata": {},
+ "source": [
+ "And simply because we'll be reading in this notebook as our working copy while developing this export behavior, let's add a raw cell just so we have an example of all 3 cell types:"
+ ]
+ },
+ {
+ "cell_type": "raw",
+ "id": "d9f3cb10",
+ "metadata": {},
+ "source": [
+ "This is a raw cell just to have an example."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4c0671ed",
+ "metadata": {},
+ "source": [
+ "Now, we can read in our current notebook:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "27202360",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def read_nb(\n",
+ " fp:str|IO[str], # a path or file-like object to read the notebook from\n",
+ " v:int=4, # the notebook format major version to read the notebook as\n",
+ ")->dict:\n",
+ " \"Read in a notebook as a dictionary.\"\n",
+ " return nbformat.read(fp, as_version=v)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a689a07b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "nb = read_nb('export.ipynb')\n",
+ "assert list(nb.keys()) == ['cells', 'metadata', 'nbformat', 'nbformat_minor']"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7d133fff",
+ "metadata": {},
+ "source": [
+ "We can see that the top-level format of a notebook is:\n",
+ "- `cells`: A list of objects for each cell in the notebook.\n",
+ "- `metadata`: Free-form notebook level metadata used by tools and editors.\n",
+ "- `nbformat`: Notebook format major version\n",
+ "- `nbformat_minor`: Notebook format minor version"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "10a6afcb",
+ "metadata": {},
+ "source": [
+ "Within cells, there are generally 3 types of cells:\n",
+ "- `code`\n",
+ "- `markdown`\n",
+ "- `raw`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1ee0d459",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "class CellType(enum.StrEnum):\n",
+ " \"Possible Jupyter notebook cell types.\"\n",
+ " CODE = 'code'\n",
+ " MD = 'markdown'\n",
+ " RAW = 'raw'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "46ed0445",
+ "metadata": {},
+ "source": [
+ "## Retrieving All Code Cells"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "133ebd88",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def filter_cells(\n",
+ " nb:dict, # A notebook that has been read in via `read_nb`\n",
+ " ct:CellType, # The cells to filter down to\n",
+ ")->list[dict]:\n",
+ " \"Retrieve specific cell types from a notebook.\"\n",
+ " return list(filter(lambda c: c['cell_type'] == ct, nb['cells']))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "76917bc5",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'id': 'b92ad576',\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '#| default_exp: export'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "code_cell = filter_cells(nb, CellType.CODE)[0]\n",
+ "assert list(code_cell.keys()) == ['cell_type', 'execution_count', 'id', 'metadata', 'outputs', 'source']\n",
+ "assert code_cell['cell_type'] == CellType.CODE\n",
+ "code_cell"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4186fffe",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'cell_type': 'markdown',\n",
+ " 'id': 'a682b39f',\n",
+ " 'metadata': {},\n",
+ " 'source': '# Exporting Notebooks to Python Files'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "md_cell = filter_cells(nb, CellType.MD)[0]\n",
+ "assert list(md_cell.keys()) == ['cell_type', 'id', 'metadata', 'source']\n",
+ "assert md_cell['cell_type'] == CellType.MD\n",
+ "md_cell"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "46e56abf",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'cell_type': 'raw',\n",
+ " 'id': 'd9f3cb10',\n",
+ " 'metadata': {},\n",
+ " 'source': 'This is a raw cell just to have an example.'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "raw_cell = filter_cells(nb, CellType.RAW)[0]\n",
+ "assert list(raw_cell.keys()) == ['cell_type', 'id', 'metadata', 'source']\n",
+ "assert raw_cell['cell_type'] == CellType.RAW\n",
+ "raw_cell"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4852b261",
+ "metadata": {},
+ "source": [
+ "Within each cell object, they will all have:\n",
+ "- `id`: A unique identifier for that cell.\n",
+ "- `cell_type`: One of the 3 cell type names.\n",
+ "- `metadata`: Free-form metadata for that cell used by various tools/editors.\n",
+ "- `source`: The content of the cell, whether it is prose or code.\n",
+ "\n",
+ "Additionally, `code` cell types will have:\n",
+ "- `outputs`: The stdout/stderr of the cell after the code is executed.\n",
+ "- `execution_count`: The number of times the cell was run."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6b7469e3",
+ "metadata": {},
+ "source": [
+ "## Find Code Cells to Export"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "798249d5",
+ "metadata": {},
+ "source": [
+ "### Matching Directives"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f671cafa",
+ "metadata": {},
+ "source": [
+ "A regular expression ran against each line in a cell should allow us to find each directive we support. Since we'll need to re-create this regression expression for a few directives, we'll create a small helper."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "83a898c2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def _mk_directive_re(\n",
+ " name:str, # The name of the directive\n",
+ " arg_re:str|None=None, # A regex pattern to match an argument for the directive\n",
+ ")->re.Pattern:\n",
+ " \"Compile a regex to match a directive, optionally supporting an argument.\"\n",
+ " arg = rf':\\s*{arg_re}' if arg_re else ''\n",
+ " return re.compile(rf'^\\s*#\\|\\s*{name}{arg}\\s*$')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "698d769b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "_TEST_DIRECTIVE_RE = _mk_directive_re('test')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f6d654fd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_dir_match = [\n",
+ " ' #| test \\t \\n', # Lots of whitespace around each token\n",
+ " '#|test', # No whitespace around each token\n",
+ "]\n",
+ "for line in test_dir_match:\n",
+ " assert _TEST_DIRECTIVE_RE.match(line), f\"No match: {line!r}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "02f06ee6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_dir_no_match = [\n",
+ " '| test', # Missing #\n",
+ " '# test', # Missing |\n",
+ " '#| testt', # Misspelled directive\n",
+ " '#| test test', # Extra word after directive\n",
+ "]\n",
+ "for line in test_dir_no_match:\n",
+ " assert not _TEST_DIRECTIVE_RE.match(line), f\"Unexpected match: {line!r}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "285e2320",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "_TEST_ARG_DIRECTIVE_RE = _mk_directive_re('test', arg_re=r'\\w')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "83fd068c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_dir_arg_match = [\n",
+ " ' #| test: \\t a \\n', # Lots of whitespace around each token\n",
+ " '#|test:b', # No whitespace around each token\n",
+ "]\n",
+ "for line in test_dir_arg_match:\n",
+ " assert _TEST_ARG_DIRECTIVE_RE.match(line), f\"No match: {line!r}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dd9945ab",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_arg_dir_no_match = [\n",
+ " '#| test a', # Missing :\n",
+ " '#| test: ', # Missing arg\n",
+ " '#| test: aa', # Invalid arg (should be only 1 letter)\n",
+ "]\n",
+ "for line in test_dir_no_match + test_arg_dir_no_match:\n",
+ " assert not _TEST_ARG_DIRECTIVE_RE.match(line), f\"Unexpected match: {line!r}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dfd6ac40",
+ "metadata": {},
+ "source": [
+ "#### Supporting the default_exp directive"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ac846aaa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "DIRECTIVE_DEFAULT_EXP_RE = _mk_directive_re('default_exp', arg_re=r'(?P<module>\\w+(\\.\\w+)*)')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5f890ed5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "matching_cells = [\n",
+ " ('#| default_exp: bootstrap', 'bootstrap'), # Module name\n",
+ " ('#| default_exp: bootstrap.another_module', 'bootstrap.another_module'), # Dotted module path\n",
+ "]\n",
+ "for line, module in matching_cells:\n",
+ " match = DIRECTIVE_DEFAULT_EXP_RE.match(line)\n",
+ " assert match, f\"No match: {line!r}\"\n",
+ " assert match.group('module') == module"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d1f22bb8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "not_matching_cells = [\n",
+ " '#| default_exp: ', # Missing module name\n",
+ " '#| default_exp: !$', # Invalid module name\n",
+ " '#| default_export: bootstrap', # Misspelled directive\n",
+ " '#| default_exp: bootstrap.', # Dot without a following word\n",
+ " '#| default_exp: bootstrap..', # Too many dots without a word in between\n",
+ "]\n",
+ "for line in not_matching_cells:\n",
+ " assert not DIRECTIVE_DEFAULT_EXP_RE.match(line), f\"Unexpected match: {line!r}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "50495b37",
+ "metadata": {},
+ "source": [
+ "#### Supporting the export directive"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5b4e8fe6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "DIRECTIVE_EXPORT_RE = _mk_directive_re('export')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "50c2d8ff",
+ "metadata": {},
+ "source": [
+ "#### Supporting the noexport directive"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "54459016",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "DIRECTIVE_NOEXPORT_RE = _mk_directive_re('noexport')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a7a778d2",
+ "metadata": {},
+ "source": [
+ "#### All Directives"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "828159b2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "DIRECTIVES = [DIRECTIVE_DEFAULT_EXP_RE, DIRECTIVE_EXPORT_RE, DIRECTIVE_NOEXPORT_RE]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "020f82b0",
+ "metadata": {},
+ "source": [
+ "### Matching Python Imports"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7863f491",
+ "metadata": {},
+ "source": [
+ "Since we know we're working with Python code cells, we can use the `ast` module to robustly detect if any `import`s are in the code cell."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10232dd8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def has_import(src:str)->bool:\n",
+ " \"Check whether a block of Python source code has any import statements.\"\n",
+ " try: return any(isinstance(node, (ast.Import, ast.ImportFrom)) for node in ast.walk(ast.parse(src)))\n",
+ " except SyntaxError: return False\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a61b2562",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert has_import('1 + 1\\nimport re\\na = 5\\n')\n",
+ "assert has_import('from re import Pattern\\na = 5\\n')\n",
+ "assert not has_import('# import re\\n')\n",
+ "assert not has_import('a = 5\\n')\n",
+ "assert not has_import('max(1, 10)\\n')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "21e3d799",
+ "metadata": {},
+ "source": [
+ "### Identifying Cells to Export"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "295b4a99",
+ "metadata": {},
+ "source": [
+ "So, we should now have all the pieces to determine find all cells we should be exporting and the module we should be exporting to.\n",
+ "\n",
+ "**Algorithm:**\n",
+ "1. Always check if there is a `default_exp` directive. There must be one (and only one) `default_exp` directive.\n",
+ "2. Check if an import is present. If present, check if a noexport directive is present.\n",
+ "3. If no import is present, check for an explicit export directive."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d327883e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def has_directive(src:str)->bool:\n",
+ " \"Check whether a line or block of source code has a nb directive.\"\n",
+ " return any(any(r.match(l) for r in DIRECTIVES) for l in src.splitlines())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6ea5e776",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert has_directive('#| default_exp: bootstrap')\n",
+ "assert has_directive('#| export')\n",
+ "assert has_directive('#| noexport')\n",
+ "assert has_directive('max(1,20)\\n#| export')\n",
+ "assert not has_directive('# comment\\nmax(1, 20)\\n')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b6f47497",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def strip_directives(src:str|list[str])->str:\n",
+ " \"Remove all nb directives from the source code.\"\n",
+ " src = src if isinstance(src, list) else src.splitlines()\n",
+ " return \"\\n\".join(filter(lambda l: not has_directive(l), src))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7acbb8ff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert strip_directives('#| export\\na=5') == 'a=5'\n",
+ "assert strip_directives('#| noexport\\na=5') == 'a=5'\n",
+ "assert strip_directives('#| default_exp: bootstrap\\na=5') == 'a=5'\n",
+ "assert strip_directives(['a=10', 'a+=10', '#|export']) == 'a=10\\na+=10'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d1f32654",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def is_cell_exported(cell:dict)->bool:\n",
+ " \"Check whether the code cell should be exported.\"\n",
+ " src = cell['source']\n",
+ " lines = src.splitlines()\n",
+ " if any(DIRECTIVE_NOEXPORT_RE.match(l) for l in lines): return False\n",
+ " if any(DIRECTIVE_EXPORT_RE.match(l) for l in lines): return True\n",
+ " return has_import(src)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "64175ad3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mk_cell = lambda s,id=1: dict(id=id,source=s,cell_type=CellType.CODE)\n",
+ "assert is_cell_exported(mk_cell('import re'))\n",
+ "assert is_cell_exported(mk_cell('from re import Pattern'))\n",
+ "assert is_cell_exported(mk_cell('#| export'))\n",
+ "assert not is_cell_exported(mk_cell('#| noexport'))\n",
+ "assert not is_cell_exported(mk_cell('#| noexport\\n#| export'))\n",
+ "assert not is_cell_exported(mk_cell('#| export\\n#| noexport'))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ddc5c9e6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def get_default_export(cells:list[dict])->str:\n",
+ " \"Retrieve the `default_exp` directive argument from a list of cells.\"\n",
+ " matches = [\n",
+ " m.group('module')\n",
+ " for c in cells\n",
+ " for l in c['source'].splitlines()\n",
+ " if c['cell_type'] == CellType.CODE and (m:=DIRECTIVE_DEFAULT_EXP_RE.match(l))\n",
+ " ]\n",
+ " if len(matches) > 1: raise ValueError('More than one default_exp directive')\n",
+ " if len(matches) == 0: raise ValueError('Missing default_exp directive')\n",
+ " return matches[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7310f069",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert get_default_export(read_nb('export.ipynb')['cells']) == 'export'\n",
+ "\n",
+ "try: get_default_export([mk_cell('a=5')]); assert False, \"Expected ValueError\"\n",
+ "except ValueError: pass\n",
+ "\n",
+ "try:\n",
+ " get_default_export([mk_cell('#| default_exp: hello'), mk_cell('#| default_exp: test')])\n",
+ " assert False, \"Expected ValueError\"\n",
+ "except ValueError: pass"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9ce855f0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def find_cells_to_export(\n",
+ " fp:str|IO[str], # a path or file-like object to read the notebook from\n",
+ ")->tuple[str,list[dict]]: # returns the module name and list of cells to export\n",
+ " \"Find the code cells in an ipynb file to export out.\"\n",
+ " code_cells = filter_cells(read_nb(fp), CellType.CODE)\n",
+ " module = get_default_export(code_cells)\n",
+ " cells = [\n",
+ " {**c, 'source': strip_directives(c['source'])}\n",
+ " for c in code_cells\n",
+ " if is_cell_exported(c)\n",
+ " ]\n",
+ " return module, cells"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "66574d29",
+ "metadata": {},
+ "source": [
+ "## Exporting a Notebook"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3070f710",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def fmt_export_cell(cell:dict)->dict:\n",
+ " \"Format a code cell's source with metadata to prepare for export.\"\n",
+ " cell['source'] = f'# cell: {cell[\"id\"]}\\n' + cell['source'] + '\\n'\n",
+ " return cell"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e24f633f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert fmt_export_cell(mk_cell('a=5', id=1))['source'] == '# cell: 1\\na=5\\n'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "633e256b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def mk_py_mod_scaffold(\n",
+ " mod:str, # The Python module name or dotted path e.g. mod.mymod.file\n",
+ " src_dir:str|pathlib.Path, # The Python package source code directory scaffold\n",
+ "):\n",
+ " \"Create all intermediary __init__.py files and directories, if any, and the final .py file.\"\n",
+ " if not mod: raise ValueError('The module path cannot be empty')\n",
+ " src_dir = pathlib.Path(src_dir)\n",
+ " mod_file = (src_dir/pathlib.Path(*mod.split('.'))).with_suffix('.py')\n",
+ " mod_file.parent.mkdir(parents=True, exist_ok=True)\n",
+ " mod_file.touch()\n",
+ " pkg_dirs = list(mod_file.parents)\n",
+ " for p in pkg_dirs[:pkg_dirs.index(src_dir) + 1]:\n",
+ " (p / '__init__.py').touch()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c044d6d4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| noexport\n",
+ "import os\n",
+ "import tempfile"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e19b3289",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with tempfile.TemporaryDirectory() as tmp:\n",
+ " try: mk_py_mod_scaffold('', src_dir=tmp); assert False, \"Should raise ValueError\"\n",
+ " except ValueError: pass\n",
+ " assert os.listdir(tmp) == []"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c3a33968",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with tempfile.TemporaryDirectory() as tmp:\n",
+ " mk_py_mod_scaffold('file', src_dir=tmp)\n",
+ " assert os.listdir(tmp) == ['__init__.py', 'file.py']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f2e3fcb4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with tempfile.TemporaryDirectory() as tmp:\n",
+ " mk_py_mod_scaffold('mod.file', src_dir=tmp)\n",
+ " assert os.listdir(tmp) == ['__init__.py', 'mod']\n",
+ " assert os.listdir(pathlib.Path(tmp)/'mod') == ['__init__.py', 'file.py']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f3ab7819",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def export_nb(\n",
+ " fp:str|IO[str], # The notebook to export\n",
+ " src_dir:str|pathlib.Path='.', # The source directory to export the Python module to.\n",
+ "):\n",
+ " \"Export the exportable code cells to a Python file.\"\n",
+ " module, cells = find_cells_to_export(fp)\n",
+ " mk_py_mod_scaffold(module, src_dir)\n",
+ " exp_path = (pathlib.Path(src_dir)/module).with_suffix('.py')\n",
+ " src = '\\n'.join(\n",
+ " ['# AUTOGENERATED! DO NOT EDIT!\\n'] +\n",
+ " list(map(lambda c: fmt_export_cell(c)['source'], cells))\n",
+ " )\n",
+ " pathlib.Path(exp_path).write_text(src)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c85bc348",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| noexport\n",
+ "from io import StringIO"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "55f57bee",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def mk_nb(cells)->StringIO:\n",
+ " nb = nbformat.v4.new_notebook(cells=cells)\n",
+ " buf = StringIO()\n",
+ " nbformat.write(nb, buf)\n",
+ " buf.seek(0)\n",
+ " return buf"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31bb0f0b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cells = [\n",
+ " nbformat.v4.new_code_cell('#| default_exp: testmod'),\n",
+ " nbformat.v4.new_code_cell('import os'),\n",
+ " nbformat.v4.new_code_cell('#| export\\ndef foo(): pass'),\n",
+ " nbformat.v4.new_code_cell('#| noexport\\nimport pathlib'),\n",
+ "]\n",
+ "nb = mk_nb(cells)\n",
+ "src = f\"\"\"# AUTOGENERATED! DO NOT EDIT!\n",
+ "\n",
+ "# cell: {cells[1]['id']}\n",
+ "import os\n",
+ "\n",
+ "# cell: {cells[2]['id']}\n",
+ "def foo(): pass\n",
+ "\"\"\"\n",
+ "with tempfile.TemporaryDirectory() as tmp:\n",
+ " export_nb(nb, src_dir=tmp)\n",
+ " result = pathlib.Path(tmp, 'testmod.py').read_text()\n",
+ " assert result == src, result"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e7c49602",
+ "metadata": {},
+ "source": [
+ "And...we've successfully bootstrapped our notebook! Let's export it out."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8e1d4096",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "export_nb('export.ipynb', src_dir='../nb/')"
+ ]
+ }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/nbs/test.ipynb b/nbs/test.ipynb
new file mode 100644
index 0000000..d446272
--- /dev/null
+++ b/nbs/test.ipynb
@@ -0,0 +1,418 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d42a9120-8d70-4b10-8d6c-b59692fd03ad",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "11cb390f-b0f4-438d-b524-db7242d22ee9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| default_exp: test"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dd03783f-252c-40f5-b92f-0f3f9278ce09",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ast,copy,contextlib,inspect,textwrap,traceback\n",
+ "from typing import Any,Generator\n",
+ "from dataclasses import dataclass"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2cf5d527",
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "NameError",
+ "evalue": "name '_T' is not defined",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+ "\u001b[31mNameError\u001b[39m Traceback (most recent call last)",
+ "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 2\u001b[39m",
+ "\u001b[32m 1\u001b[39m \u001b[38;5;66;03m#| noexport\u001b[39;00m",
+ "\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m nb.core \u001b[38;5;28;01mimport\u001b[39;00m pubattr",
+ "",
+ "\u001b[36mFile \u001b[39m\u001b[32m~/nb/nb/core.py:28\u001b[39m",
+ "\u001b[32m 25\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m inner",
+ "\u001b[32m 27\u001b[39m \u001b[38;5;66;03m# cell: 42580f8b\u001b[39;00m",
+ "\u001b[32m---> \u001b[39m\u001b[32m28\u001b[39m \u001b[38;5;129m@patch_to\u001b[39m(\u001b[30;43m_T\u001b[39;49m)",
+ "\u001b[32m 29\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msay_hello\u001b[39m(\u001b[38;5;28mself\u001b[39m: _T, name: \u001b[38;5;28mstr\u001b[39m) -> \u001b[38;5;28mstr\u001b[39m:",
+ "\u001b[32m 30\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mHello, \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m",
+ "\u001b[32m 32\u001b[39m \u001b[38;5;66;03m# cell: 4eb650ab\u001b[39;00m",
+ "",
+ "\u001b[31mNameError\u001b[39m: name '_T' is not defined"
+ ]
+ }
+ ],
+ "source": [
+ "#| noexport\n",
+ "from nb.core import pubattr"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5b8abf6a-c94a-43e5-a1d1-81b1a1d64656",
+ "metadata": {},
+ "source": [
+ "# Testing\n",
+ "> Utilities for testing within notebooks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "12e34b1f",
+ "metadata": {},
+ "source": [
+ "## Re-constructing source code and program state during runtime"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "93a3d5fe",
+ "metadata": {},
+ "source": [
+ "When writing test libraries, it can be incredibly useful to understand what the call site source code looks like.\n",
+ "\n",
+ "For instance, let's say you call `equal(x in [1, 2, 3], should_exist)`. When we're in `equal`, the arguments have already been evaluated and we simply have two booleans as arguments, each `True` or `False`. If this test fails, it likely isn't *that* helpful to see `False != True`. \n",
+ "\n",
+ "Instead, it is likely much *more* helpful to see something like `x in [1, 2, 3] != should_exist` where `x = 4` and `should_exist=True`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a6c67919",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def get_call_args(frames_up: int = 1) -> tuple[list[str], dict[str,Any]]:\n",
+ " \"Get the source code expressions of the caller's arguments.\"\n",
+ " fm_info = inspect.getouterframes(inspect.currentframe())[frames_up+1]\n",
+ " src, _ = inspect.getsourcelines(fm_info.frame)\n",
+ " srcl = src[fm_info.positions.lineno - 1:fm_info.positions.end_lineno]\n",
+ " if len(srcl) == 0:\n",
+ " raise RuntimeError(\n",
+ " \"`get_call_args` is designed to be called from within a function, \" +\n",
+ " \"not at the top-level. If you want to call it at the top level, pass `frames_up=0`.\"\n",
+ " )\n",
+ " elif len(srcl) == 1:\n",
+ " fn_src = srcl[0][fm_info.positions.col_offset:fm_info.positions.end_col_offset]\n",
+ " else:\n",
+ " fn_src = \"\".join(\n",
+ " [srcl[0][fm_info.positions.col_offset:]] +\n",
+ " srcl[1:-1] +\n",
+ " [srcl[-1][:fm_info.positions.end_col_offset]]\n",
+ " )\n",
+ " call_node = ast.parse(fn_src).body[0].value\n",
+ " args_src = [ast.unparse(arg) for arg in call_node.args]\n",
+ " kwargs_src = {kw.arg: ast.unparse(kw.value) for kw in call_node.keywords}\n",
+ " return args_src, kwargs_src"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "487c29d9",
+ "metadata": {},
+ "source": [
+ "Because `get_call_args()` is expected to be used from within a function, it generally will not work properly by default if you call it from the top-level."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bbe2859c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "`get_call_args` is designed to be called from within a function, not at the top-level. If you want to call it at the top level, pass `frames_up=0`.\n"
+ ]
+ }
+ ],
+ "source": [
+ "try: get_call_args(); assert False, \"should have raised\"\n",
+ "except RuntimeError as e: print(e)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ce788841",
+ "metadata": {},
+ "source": [
+ "However, you _can_ have it work if you pass `frames_up=0` (i.e. the current frame we're calling `get_call_args()` from rather than the caller frame)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1f3b1abb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "args, kwargs = get_call_args(frames_up=0)\n",
+ "assert args == [], f\"Got {args}\"\n",
+ "assert kwargs == {\"frames_up\": \"0\"}, f\"Got {kwargs}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5a48fa09",
+ "metadata": {},
+ "source": [
+ "When you're in a function, it works as expected."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b7730389",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def f(a, b=1):\n",
+ " return get_call_args()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19890aa0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "args, kwargs = f(1+1,b=max(10, 20))\n",
+ "assert args == [\"1 + 1\"], f\"Got {args}\"\n",
+ "assert kwargs == {\"b\": \"max(10, 20)\"}, f\"Got {kwargs}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b9aba848",
+ "metadata": {},
+ "source": [
+ "It also works with multi-line function calls.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b8b7195c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "args, kwargs = f(\n",
+ " 1,\n",
+ " b=10,\n",
+ " \n",
+ " \n",
+ ")\n",
+ "assert args == [\"1\"], f\"Got {args}\"\n",
+ "assert kwargs == {\"b\": \"10\"}, f\"Got {kwargs}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f93a8882",
+ "metadata": {},
+ "source": [
+ "If you use it with a function within a function, you'll get the arguments for the inner function.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ce0666f5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def nested_f(a, b=1):\n",
+ " def inner_f(c, d, e=1):\n",
+ " return get_call_args()\n",
+ " return inner_f(5 * 5, 10 / 10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3555f6cf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "args, kwargs = nested_f(1)\n",
+ "assert args == [\"5 * 5\", \"10 / 10\"], f\"Got {args}\"\n",
+ "assert kwargs == {}, f\"Got {kwargs}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "97d47759",
+ "metadata": {},
+ "source": [
+ "However, you can still get the outer function arguments with the `frames_up` keyword argument."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2a42fc2b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def nested_f(a, b=1):\n",
+ " def inner_f(c, d, e=1):\n",
+ " return get_call_args(frames_up=2)\n",
+ " return inner_f(5 * 5, 10 / 10)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b3687819",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "args, kwargs = nested_f(1)\n",
+ "assert args == [\"1\"], f\"Got {args}\"\n",
+ "assert kwargs == {}, f\"Got {kwargs}\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5d991fd4",
+ "metadata": {},
+ "source": [
+ "## Assertions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "623a480c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "@dataclass\n",
+ "class ExceptionInfo:\n",
+ " type_: type[Exception] | None = None\n",
+ " value: Exception | None = None\n",
+ " traceback: str | None = None"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0ad5fcef-f04d-469d-8ad1-94d3a98694e4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "@contextlib.contextmanager\n",
+ "def raises(\n",
+ " exc: type[Exception] = Exception, # The exception type to check is raised.\n",
+ " contains: str | None = None, # Optionally, check whether the raised exception contains this message.\n",
+ ") -> Generator[ExceptionInfo, None, None]:\n",
+ " \"Asserts that the body of the context manager raises `exc`.\"\n",
+ " raised = True\n",
+ " try:\n",
+ " info = ExceptionInfo()\n",
+ " yield info\n",
+ " raised = False\n",
+ " except exc as e:\n",
+ " info.type_ = type(e)\n",
+ " info.value = e\n",
+ " info.traceback = traceback.format_tb(e.__traceback__)\n",
+ " if contains and contains not in str(e):\n",
+ " raise AssertionError(f\"Raised {e} but message does not contain {contains}\") from e\n",
+ " if not raised:\n",
+ " raise AssertionError(f\"Did not raise {e}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7c478098-7a09-403c-a62b-3c34cf0f7bab",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ExceptionInfo(type_=<class 'Exception'>, value=Exception('abc'), traceback=[' File \"/tmp/ipykernel_1287/933907959.py\", line 11, in raises\\n yield info\\n', ' File \"/tmp/ipykernel_1287/3922684010.py\", line 2, in <module>\\n raise Exception(\"abc\")\\n'])\n"
+ ]
+ }
+ ],
+ "source": [
+ "with raises() as e:\n",
+ " raise Exception(\"abc\")\n",
+ "print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2b503871-3f39-4013-a7ae-f5143af3138e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def equal(a, b):\n",
+ " \"Assert that a == b.\"\n",
+ " args_src, _ = get_call_args()\n",
+ " if a != b:\n",
+ " msg = (\n",
+ " f\"FAIL: {args_src[0]} != {args_src[1]}\\n\" +\n",
+ " ' ' * (len(AssertionError.__name__)+2) +\n",
+ " f\" {a!r} != {b!r}\"\n",
+ " )\n",
+ " raise AssertionError(msg)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cc7872bb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with raises(exc=AssertionError) as e:\n",
+ " equal(1, 2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7a1393a7-6a89-4c5d-a8a7-51e0fd46d190",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| noexport\n",
+ "import nb.export; nb.export.export_nb('test.ipynb', src_dir='../nb/')"
+ ]
+ }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..0e87d5f
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,25 @@
+[project]
+name = "nb"
+version = "0.1.0"
+description = "nb implements a notebook-driven development workflow"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = ["nbconvert"]
+authors = [{ name = "Eugene Triguba", email = "hello@eugenetriguba.com" }]
+classifiers = [
+ "Development Status :: 3 - Alpha",
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Build Tools",
+ "License :: OSI Approved :: MIT License",
+ "Programming Language :: Python :: 3",
+]
+
+[dependency-groups]
+dev = ["jupyter"]
+
+[tool.setuptools]
+packages = ["nb"]
+
+[build-system]
+requires = ["setuptools>=64"]
+build-backend = "setuptools.build_meta"