diff options
| author | Eugene Triguba <hello@eugenetriguba.com> | 2026-05-19 01:48:15 +0000 |
|---|---|---|
| committer | Eugene Triguba <hello@eugenetriguba.com> | 2026-05-19 01:48:15 +0000 |
| commit | 466b70b4e31c625185c5f70eb7be13c25c7df421 (patch) | |
| tree | 050d2ac81defb5b0d31a664bea23940a2d751503 | |
| download | nb-466b70b4e31c625185c5f70eb7be13c25c7df421.zip | |
Initial Commit
| -rw-r--r-- | .gitignore | 42 | ||||
| -rw-r--r-- | LICENSE | 26 | ||||
| -rw-r--r-- | README.md | 1 | ||||
| -rw-r--r-- | nb/__init__.py | 0 | ||||
| -rw-r--r-- | nb/core.py | 36 | ||||
| -rw-r--r-- | nb/export.py | 143 | ||||
| -rw-r--r-- | nb/test.py | 70 | ||||
| -rw-r--r-- | nbs/core.ipynb | 299 | ||||
| -rw-r--r-- | nbs/export.ipynb | 950 | ||||
| -rw-r--r-- | nbs/test.ipynb | 418 | ||||
| -rw-r--r-- | pyproject.toml | 25 |
11 files changed, 2010 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..739c482 --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +# Editors +tags + +# Distribution / packaging +.Python +env/ +build/ +conda/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual envs +.venv + +# pyenv +.python-version + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + + +# Jupyter Notebook +.ipynb_checkpoints + + +# macOS +.DS_Store + @@ -0,0 +1,26 @@ +Copyright 2026 Eugene Triguba + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS +IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0ae01fb --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# nb diff --git a/nb/__init__.py b/nb/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nb/__init__.py diff --git a/nb/core.py b/nb/core.py new file mode 100644 index 0000000..0f2769e --- /dev/null +++ b/nb/core.py @@ -0,0 +1,36 @@ +# AUTOGENERATED! DO NOT EDIT! + +# cell: 6a2cc7f8-6c65-4a7c-890c-3cf9d1cd314e +import inspect +from typing import Callable +from nb import test + +# cell: c4875cc0-2d87-4c87-8789-7dc950be06c6 +def callable_name(c: Callable) -> str: + "Get the name of the callable `c`." + return getattr(c, '__name__', c.__class__.__name__) + +# cell: f2c70dd3 +def pubattr(o): + "Retrieve the public attributes of the object `o`." + return [a for a in dir(o) if not a.startswith('_')] + +# cell: 78d61a07 +def patch_to(o: object) -> Callable: + "Decorator that takes in a object and attaches the decorated function onto it." + def inner(fn: Callable) -> Callable: + nm = callable_name(fn) + setattr(o, nm, fn) + return fn + return inner + +# cell: 42580f8b +@patch_to(_T) +def say_hello(self: _T, name: str) -> str: + return f'Hello, {name}' + +# cell: 4eb650ab +def patch(fn: Callable) -> Callable: + "Decorator to patch the object of the type-hinted 'self' argument of `fn` with `fn`." + self = inspect.get_annotations(fn)['self'] + return patch_to(self)(fn) diff --git a/nb/export.py b/nb/export.py new file mode 100644 index 0000000..d140293 --- /dev/null +++ b/nb/export.py @@ -0,0 +1,143 @@ +# AUTOGENERATED! DO NOT EDIT! + +# cell: 96525ac4 +import ast +import enum +import pathlib +import re +from typing import IO + +import nbformat + +# cell: 27202360 +def read_nb( + fp:str|IO[str], # a path or file-like object to read the notebook from + v:int=4, # the notebook format major version to read the notebook as +)->dict: + "Read in a notebook as a dictionary." + return nbformat.read(fp, as_version=v) + +# cell: 1ee0d459 +class CellType(enum.StrEnum): + "Possible Jupyter notebook cell types." + CODE = 'code' + MD = 'markdown' + RAW = 'raw' + +# cell: 133ebd88 +def filter_cells( + nb:dict, # A notebook that has been read in via `read_nb` + ct:CellType, # The cells to filter down to +)->list[dict]: + "Retrieve specific cell types from a notebook." + return list(filter(lambda c: c['cell_type'] == ct, nb['cells'])) + +# cell: 83a898c2 +def _mk_directive_re( + name:str, # The name of the directive + arg_re:str|None=None, # A regex pattern to match an argument for the directive +)->re.Pattern: + "Compile a regex to match a directive, optionally supporting an argument." + arg = rf':\s*{arg_re}' if arg_re else '' + return re.compile(rf'^\s*#\|\s*{name}{arg}\s*$') + +# cell: ac846aaa +DIRECTIVE_DEFAULT_EXP_RE = _mk_directive_re('default_exp', arg_re=r'(?P<module>\w+(\.\w+)*)') + +# cell: 5b4e8fe6 +DIRECTIVE_EXPORT_RE = _mk_directive_re('export') + +# cell: 54459016 +DIRECTIVE_NOEXPORT_RE = _mk_directive_re('noexport') + +# cell: 828159b2 +DIRECTIVES = [DIRECTIVE_DEFAULT_EXP_RE, DIRECTIVE_EXPORT_RE, DIRECTIVE_NOEXPORT_RE] + +# cell: 10232dd8 +def has_import(src:str)->bool: + "Check whether a block of Python source code has any import statements." + try: return any(isinstance(node, (ast.Import, ast.ImportFrom)) for node in ast.walk(ast.parse(src))) + except SyntaxError: return False + +# cell: d327883e +def has_directive(src:str)->bool: + "Check whether a line or block of source code has a nb directive." + return any(any(r.match(l) for r in DIRECTIVES) for l in src.splitlines()) + +# cell: b6f47497 +def strip_directives(src:str|list[str])->str: + "Remove all nb directives from the source code." + src = src if isinstance(src, list) else src.splitlines() + return "\n".join(filter(lambda l: not has_directive(l), src)) + +# cell: d1f32654 +def is_cell_exported(cell:dict)->bool: + "Check whether the code cell should be exported." + src = cell['source'] + lines = src.splitlines() + if any(DIRECTIVE_NOEXPORT_RE.match(l) for l in lines): return False + if any(DIRECTIVE_EXPORT_RE.match(l) for l in lines): return True + return has_import(src) + +# cell: ddc5c9e6 +def get_default_export(cells:list[dict])->str: + "Retrieve the `default_exp` directive argument from a list of cells." + matches = [ + m.group('module') + for c in cells + for l in c['source'].splitlines() + if c['cell_type'] == CellType.CODE and (m:=DIRECTIVE_DEFAULT_EXP_RE.match(l)) + ] + if len(matches) > 1: raise ValueError('More than one default_exp directive') + if len(matches) == 0: raise ValueError('Missing default_exp directive') + return matches[0] + +# cell: 9ce855f0 +def find_cells_to_export( + fp:str|IO[str], # a path or file-like object to read the notebook from +)->tuple[str,list[dict]]: # returns the module name and list of cells to export + "Find the code cells in an ipynb file to export out." + code_cells = filter_cells(read_nb(fp), CellType.CODE) + module = get_default_export(code_cells) + cells = [ + {**c, 'source': strip_directives(c['source'])} + for c in code_cells + if is_cell_exported(c) + ] + return module, cells + +# cell: 3070f710 +def fmt_export_cell(cell:dict)->dict: + "Format a code cell's source with metadata to prepare for export." + cell['source'] = f'# cell: {cell["id"]}\n' + cell['source'] + '\n' + return cell + +# cell: 633e256b +def mk_py_mod_scaffold( + mod:str, # The Python module name or dotted path e.g. mod.mymod.file + src_dir:str|pathlib.Path, # The Python package source code directory scaffold +): + "Create all intermediary __init__.py files and directories, if any, and the final .py file." + if not mod: raise ValueError('The module path cannot be empty') + src_dir = pathlib.Path(src_dir) + mod_file = (src_dir/pathlib.Path(*mod.split('.'))).with_suffix('.py') + mod_file.parent.mkdir(parents=True, exist_ok=True) + mod_file.touch() + pkg_dirs = list(mod_file.parents) + for p in pkg_dirs[:pkg_dirs.index(src_dir) + 1]: + (p / '__init__.py').touch() + +# cell: f3ab7819 +def export_nb( + fp:str|IO[str], # The notebook to export + src_dir:str|pathlib.Path='.', # The source directory to export the Python module to. +): + "Export the exportable code cells to a Python file." + module, cells = find_cells_to_export(fp) + mk_py_mod_scaffold(module, src_dir) + exp_path = (pathlib.Path(src_dir)/module).with_suffix('.py') + src = '\n'.join( + ['# AUTOGENERATED! DO NOT EDIT!\n'] + + list(map(lambda c: fmt_export_cell(c)['source'], cells)) + ) + pathlib.Path(exp_path).write_text(src) diff --git a/nb/test.py b/nb/test.py new file mode 100644 index 0000000..8bc6df4 --- /dev/null +++ b/nb/test.py @@ -0,0 +1,70 @@ +# AUTOGENERATED! DO NOT EDIT! + +# cell: dd03783f-252c-40f5-b92f-0f3f9278ce09 +import ast,copy,contextlib,inspect,textwrap,traceback +from typing import Any,Generator +from dataclasses import dataclass + +# cell: a6c67919 +def get_call_args(frames_up: int = 1) -> tuple[list[str], dict[str,Any]]: + "Get the source code expressions of the caller's arguments." + fm_info = inspect.getouterframes(inspect.currentframe())[frames_up+1] + src, _ = inspect.getsourcelines(fm_info.frame) + srcl = src[fm_info.positions.lineno - 1:fm_info.positions.end_lineno] + if len(srcl) == 0: + raise RuntimeError( + "`get_call_args` is designed to be called from within a function, " + + "not at the top-level. If you want to call it at the top level, pass `frames_up=0`." + ) + elif len(srcl) == 1: + fn_src = srcl[0][fm_info.positions.col_offset:fm_info.positions.end_col_offset] + else: + fn_src = "".join( + [srcl[0][fm_info.positions.col_offset:]] + + srcl[1:-1] + + [srcl[-1][:fm_info.positions.end_col_offset]] + ) + call_node = ast.parse(fn_src).body[0].value + args_src = [ast.unparse(arg) for arg in call_node.args] + kwargs_src = {kw.arg: ast.unparse(kw.value) for kw in call_node.keywords} + return args_src, kwargs_src + +# cell: 623a480c +@dataclass +class ExceptionInfo: + type_: type[Exception] | None = None + value: Exception | None = None + traceback: str | None = None + +# cell: 0ad5fcef-f04d-469d-8ad1-94d3a98694e4 +@contextlib.contextmanager +def raises( + exc: type[Exception] = Exception, # The exception type to check is raised. + contains: str | None = None, # Optionally, check whether the raised exception contains this message. +) -> Generator[ExceptionInfo, None, None]: + "Asserts that the body of the context manager raises `exc`." + raised = True + try: + info = ExceptionInfo() + yield info + raised = False + except exc as e: + info.type_ = type(e) + info.value = e + info.traceback = traceback.format_tb(e.__traceback__) + if contains and contains not in str(e): + raise AssertionError(f"Raised {e} but message does not contain {contains}") from e + if not raised: + raise AssertionError(f"Did not raise {e}") + +# cell: 2b503871-3f39-4013-a7ae-f5143af3138e +def equal(a, b): + "Assert that a == b." + args_src, _ = get_call_args() + if a != b: + msg = ( + f"FAIL: {args_src[0]} != {args_src[1]}\n" + + ' ' * (len(AssertionError.__name__)+2) + + f" {a!r} != {b!r}" + ) + raise AssertionError(msg) diff --git a/nbs/core.ipynb b/nbs/core.ipynb new file mode 100644 index 0000000..24e0330 --- /dev/null +++ b/nbs/core.ipynb @@ -0,0 +1,299 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aed9daa9-f05d-4e25-b9a5-a74d0211ef6f", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "596de171-5567-43c1-98ce-66eccc28bbe9", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp: core" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a2cc7f8-6c65-4a7c-890c-3cf9d1cd314e", + "metadata": {}, + "outputs": [], + "source": [ + "import inspect\n", + "from typing import Callable\n", + "from nb import test" + ] + }, + { + "cell_type": "markdown", + "id": "8485c16a", + "metadata": {}, + "source": [ + "# Core\n", + "> Fundamental Utilities" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4875cc0-2d87-4c87-8789-7dc950be06c6", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def callable_name(c: Callable) -> str:\n", + " \"Get the name of the callable `c`.\"\n", + " return getattr(c, '__name__', c.__class__.__name__)" + ] + }, + { + "cell_type": "markdown", + "id": "406d605d", + "metadata": {}, + "source": [ + "This works for any function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e9c39c5", + "metadata": {}, + "outputs": [], + "source": [ + "assert callable_name(max) == \"max\"" + ] + }, + { + "cell_type": "markdown", + "id": "d66b2d42", + "metadata": {}, + "source": [ + "And for any classes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "056cb776", + "metadata": {}, + "outputs": [], + "source": [ + "class _A: pass\n", + "assert callable_name(_A) == \"_A\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2c70dd3", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def pubattr(o):\n", + " \"Retrieve the public attributes of the object `o`.\"\n", + " return [a for a in dir(o) if not a.startswith('_')]" + ] + }, + { + "cell_type": "markdown", + "id": "53cb2bd5", + "metadata": {}, + "source": [ + "`pubattr` will hide any attribute on an object if it starts with an underscore." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac30af03", + "metadata": {}, + "outputs": [], + "source": [ + "_A.print = print\n", + "_A._private_print = print\n", + "_A.__python_internal__ = print\n", + "assert pubattr(_A) == [\"print\"]" + ] + }, + { + "cell_type": "markdown", + "id": "f892f4f1", + "metadata": {}, + "source": [ + "## Class Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "4008335c", + "metadata": {}, + "source": [ + "Iterative notebook-driven development makes class definitions tricky. Typically, you'd write a clas definition in one monolithic block, which leaves no room for prose or incremental execution. Inspired by the wonderful [fastcore](https://github.com/AnswerDotAI/fastcore) library, we'll add `patch` and `patch_to` function to enable piecemeal class definition via monkeypatching." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78d61a07", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def patch_to(o: object) -> Callable:\n", + " \"Decorator that takes in a object and attaches the decorated function onto it.\"\n", + " def inner(fn: Callable) -> Callable:\n", + " nm = callable_name(fn)\n", + " setattr(o, nm, fn)\n", + " return fn\n", + " return inner" + ] + }, + { + "cell_type": "markdown", + "id": "0d4bb2e6", + "metadata": {}, + "source": [ + "Let's try it out. We'll create a dummy class named `_T`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c67a3c29", + "metadata": {}, + "outputs": [], + "source": [ + "class _T: pass" + ] + }, + { + "cell_type": "markdown", + "id": "b7d0a482", + "metadata": {}, + "source": [ + "Now, we'll attach on a function to `_T` in a separate code cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42580f8b", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "@patch_to(_T)\n", + "def say_hello(self: _T, name: str) -> str:\n", + " return f'Hello, {name}'" + ] + }, + { + "cell_type": "markdown", + "id": "0edc152b", + "metadata": {}, + "source": [ + "`_T` should have a `say_hello` function on it. Let's confirm." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "404bd72f", + "metadata": {}, + "outputs": [], + "source": [ + "_t = _T()\n", + "test.equal(_t.say_hello('eugene'), 'Hello, eugene')" + ] + }, + { + "cell_type": "markdown", + "id": "4acd774c", + "metadata": {}, + "source": [ + "That's neat and all but notice that `self: _T` and `@patch_to(_T)` are both two different ways to provide the same sort of information (attach this method onto `_T`).\n", + "\n", + "What if we could look at the type hint to determine which object to patch?\n", + "\n", + "We already have `patch_to`, so let's create a `patch` decorator. All we'll need to do inside `patch` is find the object to patch and call `patch_to`. If we assume the object we're patching will always be called `self` i.e. patching a class, then this should be easy-peasy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4eb650ab", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def patch(fn: Callable) -> Callable:\n", + " \"Decorator to patch the object of the type-hinted 'self' argument of `fn` with `fn`.\"\n", + " self = inspect.get_annotations(fn)['self']\n", + " return patch_to(self)(fn)" + ] + }, + { + "cell_type": "markdown", + "id": "97378af3", + "metadata": {}, + "source": [ + "Let's see if we can try `patch` out now with our dummy `_T` class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "601e27f2", + "metadata": {}, + "outputs": [], + "source": [ + "@patch\n", + "def say_goodbye(self: _T, name: str) -> str:\n", + " return f'Goodbye, {name}'" + ] + }, + { + "cell_type": "markdown", + "id": "d001522d", + "metadata": {}, + "source": [ + "Can our `_t` object now say goodbye?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6aa1ffbc", + "metadata": {}, + "outputs": [], + "source": [ + "test.equal(_t.say_goodbye('eugene'), 'Goodbye, eugene')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12b62cf8-2193-4f03-b80a-46304d73896e", + "metadata": {}, + "outputs": [], + "source": [ + "#| noexport\n", + "import nb.export; nb.export.export_nb('core.ipynb', src_dir='../nb/')" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/export.ipynb b/nbs/export.ipynb new file mode 100644 index 0000000..d662f7d --- /dev/null +++ b/nbs/export.ipynb @@ -0,0 +1,950 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b92ad576", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp: export" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96525ac4", + "metadata": {}, + "outputs": [], + "source": [ + "import ast\n", + "import enum\n", + "import pathlib\n", + "import re\n", + "from typing import IO\n", + "\n", + "import nbformat" + ] + }, + { + "cell_type": "markdown", + "id": "a682b39f", + "metadata": {}, + "source": [ + "# Exporting Notebooks to Python Files" + ] + }, + { + "cell_type": "markdown", + "id": "a21a7e2c", + "metadata": {}, + "source": [ + "To get our notebook-driven source code bootstrapped, we'll first need to be able to convert `ipynb` files to `py` files. Without that, we'd be unable to later import code we want to use in one notebook from another notebook because it's not in a plain importable Python file yet.\n", + "\n", + "To accomplish that, for any given code well, we'll need to know:\n", + "- Are we exporting it to a Python file?\n", + "- What Python file are we exporting it to?\n", + "\n", + "The approach tools like [nbdev](https://nbdev.fast.ai/) or [nbexport](https://github.com/danhorner/nbexport) take is to have certain directives that determine the code export behavior:\n", + "- `#|default_exp: mymodule`: Set the output module for this notebook\n", + "- `#|export`: Export this cell to the corresponding Python file\n", + "- `#|noexport`: Prevent export of this cell. Useful to prevent export of something that would be automatically exported.\n", + "\n", + "Furthermore, beyond the explicit directives, nbdev and nbexport both use automatic exports of cells with imports in them for less annotation overhead since nearly every import in a notebook is needed by the exported code. This is the primary rationale for a `#|noexport` since sometimes a import is purely for exploratory programming, and you won't use it in the final artifact." + ] + }, + { + "cell_type": "markdown", + "id": "3b670537", + "metadata": {}, + "source": [ + "To export a notebook, we'll need to:\n", + "1. Read in a ipynb notebook\n", + "2. Retrieve all code cells from the notebook\n", + "3. Find all cells that should be exported:\n", + " - Each notebook must have 1 (and only 1) `#| default_exp`. When a `#| default_exp` is found, it must be a single word e.g. `mymodule` or a dotted path e.g. `mymodules.other_module`. If it is a dotted path, the submodules created should have `__init__.py` files also created alongside them.\n", + " - Any cells with an `#| export` explicit directive or cells that have an import and *don't* have an explicit `#| noexport` directive should be marked for export.\n", + "4. Write out each cell sequentially into the Python file, stripping out any directives. When each cell is written out, it should have a comment above it describing the cell ID it came from and the path to the notebook it came from." + ] + }, + { + "cell_type": "markdown", + "id": "f1c95690", + "metadata": {}, + "source": [ + "## Reading Notebooks" + ] + }, + { + "cell_type": "markdown", + "id": "037fb28c", + "metadata": {}, + "source": [ + "To read in `ipynb` files, we can use a library provided Jupyter called `nbformat`:" + ] + }, + { + "cell_type": "markdown", + "id": "39f9ece6", + "metadata": {}, + "source": [ + "And simply because we'll be reading in this notebook as our working copy while developing this export behavior, let's add a raw cell just so we have an example of all 3 cell types:" + ] + }, + { + "cell_type": "raw", + "id": "d9f3cb10", + "metadata": {}, + "source": [ + "This is a raw cell just to have an example." + ] + }, + { + "cell_type": "markdown", + "id": "4c0671ed", + "metadata": {}, + "source": [ + "Now, we can read in our current notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27202360", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def read_nb(\n", + " fp:str|IO[str], # a path or file-like object to read the notebook from\n", + " v:int=4, # the notebook format major version to read the notebook as\n", + ")->dict:\n", + " \"Read in a notebook as a dictionary.\"\n", + " return nbformat.read(fp, as_version=v)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a689a07b", + "metadata": {}, + "outputs": [], + "source": [ + "nb = read_nb('export.ipynb')\n", + "assert list(nb.keys()) == ['cells', 'metadata', 'nbformat', 'nbformat_minor']" + ] + }, + { + "cell_type": "markdown", + "id": "7d133fff", + "metadata": {}, + "source": [ + "We can see that the top-level format of a notebook is:\n", + "- `cells`: A list of objects for each cell in the notebook.\n", + "- `metadata`: Free-form notebook level metadata used by tools and editors.\n", + "- `nbformat`: Notebook format major version\n", + "- `nbformat_minor`: Notebook format minor version" + ] + }, + { + "cell_type": "markdown", + "id": "10a6afcb", + "metadata": {}, + "source": [ + "Within cells, there are generally 3 types of cells:\n", + "- `code`\n", + "- `markdown`\n", + "- `raw`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ee0d459", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "class CellType(enum.StrEnum):\n", + " \"Possible Jupyter notebook cell types.\"\n", + " CODE = 'code'\n", + " MD = 'markdown'\n", + " RAW = 'raw'" + ] + }, + { + "cell_type": "markdown", + "id": "46ed0445", + "metadata": {}, + "source": [ + "## Retrieving All Code Cells" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "133ebd88", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def filter_cells(\n", + " nb:dict, # A notebook that has been read in via `read_nb`\n", + " ct:CellType, # The cells to filter down to\n", + ")->list[dict]:\n", + " \"Retrieve specific cell types from a notebook.\"\n", + " return list(filter(lambda c: c['cell_type'] == ct, nb['cells']))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76917bc5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'cell_type': 'code',\n", + " 'execution_count': None,\n", + " 'id': 'b92ad576',\n", + " 'metadata': {},\n", + " 'outputs': [],\n", + " 'source': '#| default_exp: export'}" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "code_cell = filter_cells(nb, CellType.CODE)[0]\n", + "assert list(code_cell.keys()) == ['cell_type', 'execution_count', 'id', 'metadata', 'outputs', 'source']\n", + "assert code_cell['cell_type'] == CellType.CODE\n", + "code_cell" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4186fffe", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'cell_type': 'markdown',\n", + " 'id': 'a682b39f',\n", + " 'metadata': {},\n", + " 'source': '# Exporting Notebooks to Python Files'}" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "md_cell = filter_cells(nb, CellType.MD)[0]\n", + "assert list(md_cell.keys()) == ['cell_type', 'id', 'metadata', 'source']\n", + "assert md_cell['cell_type'] == CellType.MD\n", + "md_cell" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46e56abf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'cell_type': 'raw',\n", + " 'id': 'd9f3cb10',\n", + " 'metadata': {},\n", + " 'source': 'This is a raw cell just to have an example.'}" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "raw_cell = filter_cells(nb, CellType.RAW)[0]\n", + "assert list(raw_cell.keys()) == ['cell_type', 'id', 'metadata', 'source']\n", + "assert raw_cell['cell_type'] == CellType.RAW\n", + "raw_cell" + ] + }, + { + "cell_type": "markdown", + "id": "4852b261", + "metadata": {}, + "source": [ + "Within each cell object, they will all have:\n", + "- `id`: A unique identifier for that cell.\n", + "- `cell_type`: One of the 3 cell type names.\n", + "- `metadata`: Free-form metadata for that cell used by various tools/editors.\n", + "- `source`: The content of the cell, whether it is prose or code.\n", + "\n", + "Additionally, `code` cell types will have:\n", + "- `outputs`: The stdout/stderr of the cell after the code is executed.\n", + "- `execution_count`: The number of times the cell was run." + ] + }, + { + "cell_type": "markdown", + "id": "6b7469e3", + "metadata": {}, + "source": [ + "## Find Code Cells to Export" + ] + }, + { + "cell_type": "markdown", + "id": "798249d5", + "metadata": {}, + "source": [ + "### Matching Directives" + ] + }, + { + "cell_type": "markdown", + "id": "f671cafa", + "metadata": {}, + "source": [ + "A regular expression ran against each line in a cell should allow us to find each directive we support. Since we'll need to re-create this regression expression for a few directives, we'll create a small helper." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83a898c2", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _mk_directive_re(\n", + " name:str, # The name of the directive\n", + " arg_re:str|None=None, # A regex pattern to match an argument for the directive\n", + ")->re.Pattern:\n", + " \"Compile a regex to match a directive, optionally supporting an argument.\"\n", + " arg = rf':\\s*{arg_re}' if arg_re else ''\n", + " return re.compile(rf'^\\s*#\\|\\s*{name}{arg}\\s*$')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "698d769b", + "metadata": {}, + "outputs": [], + "source": [ + "_TEST_DIRECTIVE_RE = _mk_directive_re('test')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6d654fd", + "metadata": {}, + "outputs": [], + "source": [ + "test_dir_match = [\n", + " ' #| test \\t \\n', # Lots of whitespace around each token\n", + " '#|test', # No whitespace around each token\n", + "]\n", + "for line in test_dir_match:\n", + " assert _TEST_DIRECTIVE_RE.match(line), f\"No match: {line!r}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02f06ee6", + "metadata": {}, + "outputs": [], + "source": [ + "test_dir_no_match = [\n", + " '| test', # Missing #\n", + " '# test', # Missing |\n", + " '#| testt', # Misspelled directive\n", + " '#| test test', # Extra word after directive\n", + "]\n", + "for line in test_dir_no_match:\n", + " assert not _TEST_DIRECTIVE_RE.match(line), f\"Unexpected match: {line!r}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "285e2320", + "metadata": {}, + "outputs": [], + "source": [ + "_TEST_ARG_DIRECTIVE_RE = _mk_directive_re('test', arg_re=r'\\w')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83fd068c", + "metadata": {}, + "outputs": [], + "source": [ + "test_dir_arg_match = [\n", + " ' #| test: \\t a \\n', # Lots of whitespace around each token\n", + " '#|test:b', # No whitespace around each token\n", + "]\n", + "for line in test_dir_arg_match:\n", + " assert _TEST_ARG_DIRECTIVE_RE.match(line), f\"No match: {line!r}\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd9945ab", + "metadata": {}, + "outputs": [], + "source": [ + "test_arg_dir_no_match = [\n", + " '#| test a', # Missing :\n", + " '#| test: ', # Missing arg\n", + " '#| test: aa', # Invalid arg (should be only 1 letter)\n", + "]\n", + "for line in test_dir_no_match + test_arg_dir_no_match:\n", + " assert not _TEST_ARG_DIRECTIVE_RE.match(line), f\"Unexpected match: {line!r}\"" + ] + }, + { + "cell_type": "markdown", + "id": "dfd6ac40", + "metadata": {}, + "source": [ + "#### Supporting the default_exp directive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac846aaa", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "DIRECTIVE_DEFAULT_EXP_RE = _mk_directive_re('default_exp', arg_re=r'(?P<module>\\w+(\\.\\w+)*)')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f890ed5", + "metadata": {}, + "outputs": [], + "source": [ + "matching_cells = [\n", + " ('#| default_exp: bootstrap', 'bootstrap'), # Module name\n", + " ('#| default_exp: bootstrap.another_module', 'bootstrap.another_module'), # Dotted module path\n", + "]\n", + "for line, module in matching_cells:\n", + " match = DIRECTIVE_DEFAULT_EXP_RE.match(line)\n", + " assert match, f\"No match: {line!r}\"\n", + " assert match.group('module') == module" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1f22bb8", + "metadata": {}, + "outputs": [], + "source": [ + "not_matching_cells = [\n", + " '#| default_exp: ', # Missing module name\n", + " '#| default_exp: !$', # Invalid module name\n", + " '#| default_export: bootstrap', # Misspelled directive\n", + " '#| default_exp: bootstrap.', # Dot without a following word\n", + " '#| default_exp: bootstrap..', # Too many dots without a word in between\n", + "]\n", + "for line in not_matching_cells:\n", + " assert not DIRECTIVE_DEFAULT_EXP_RE.match(line), f\"Unexpected match: {line!r}\"" + ] + }, + { + "cell_type": "markdown", + "id": "50495b37", + "metadata": {}, + "source": [ + "#### Supporting the export directive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b4e8fe6", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "DIRECTIVE_EXPORT_RE = _mk_directive_re('export')" + ] + }, + { + "cell_type": "markdown", + "id": "50c2d8ff", + "metadata": {}, + "source": [ + "#### Supporting the noexport directive" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54459016", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "DIRECTIVE_NOEXPORT_RE = _mk_directive_re('noexport')" + ] + }, + { + "cell_type": "markdown", + "id": "a7a778d2", + "metadata": {}, + "source": [ + "#### All Directives" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "828159b2", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "DIRECTIVES = [DIRECTIVE_DEFAULT_EXP_RE, DIRECTIVE_EXPORT_RE, DIRECTIVE_NOEXPORT_RE]" + ] + }, + { + "cell_type": "markdown", + "id": "020f82b0", + "metadata": {}, + "source": [ + "### Matching Python Imports" + ] + }, + { + "cell_type": "markdown", + "id": "7863f491", + "metadata": {}, + "source": [ + "Since we know we're working with Python code cells, we can use the `ast` module to robustly detect if any `import`s are in the code cell." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10232dd8", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def has_import(src:str)->bool:\n", + " \"Check whether a block of Python source code has any import statements.\"\n", + " try: return any(isinstance(node, (ast.Import, ast.ImportFrom)) for node in ast.walk(ast.parse(src)))\n", + " except SyntaxError: return False\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a61b2562", + "metadata": {}, + "outputs": [], + "source": [ + "assert has_import('1 + 1\\nimport re\\na = 5\\n')\n", + "assert has_import('from re import Pattern\\na = 5\\n')\n", + "assert not has_import('# import re\\n')\n", + "assert not has_import('a = 5\\n')\n", + "assert not has_import('max(1, 10)\\n')" + ] + }, + { + "cell_type": "markdown", + "id": "21e3d799", + "metadata": {}, + "source": [ + "### Identifying Cells to Export" + ] + }, + { + "cell_type": "markdown", + "id": "295b4a99", + "metadata": {}, + "source": [ + "So, we should now have all the pieces to determine find all cells we should be exporting and the module we should be exporting to.\n", + "\n", + "**Algorithm:**\n", + "1. Always check if there is a `default_exp` directive. There must be one (and only one) `default_exp` directive.\n", + "2. Check if an import is present. If present, check if a noexport directive is present.\n", + "3. If no import is present, check for an explicit export directive." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d327883e", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def has_directive(src:str)->bool:\n", + " \"Check whether a line or block of source code has a nb directive.\"\n", + " return any(any(r.match(l) for r in DIRECTIVES) for l in src.splitlines())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ea5e776", + "metadata": {}, + "outputs": [], + "source": [ + "assert has_directive('#| default_exp: bootstrap')\n", + "assert has_directive('#| export')\n", + "assert has_directive('#| noexport')\n", + "assert has_directive('max(1,20)\\n#| export')\n", + "assert not has_directive('# comment\\nmax(1, 20)\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6f47497", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def strip_directives(src:str|list[str])->str:\n", + " \"Remove all nb directives from the source code.\"\n", + " src = src if isinstance(src, list) else src.splitlines()\n", + " return \"\\n\".join(filter(lambda l: not has_directive(l), src))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7acbb8ff", + "metadata": {}, + "outputs": [], + "source": [ + "assert strip_directives('#| export\\na=5') == 'a=5'\n", + "assert strip_directives('#| noexport\\na=5') == 'a=5'\n", + "assert strip_directives('#| default_exp: bootstrap\\na=5') == 'a=5'\n", + "assert strip_directives(['a=10', 'a+=10', '#|export']) == 'a=10\\na+=10'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1f32654", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def is_cell_exported(cell:dict)->bool:\n", + " \"Check whether the code cell should be exported.\"\n", + " src = cell['source']\n", + " lines = src.splitlines()\n", + " if any(DIRECTIVE_NOEXPORT_RE.match(l) for l in lines): return False\n", + " if any(DIRECTIVE_EXPORT_RE.match(l) for l in lines): return True\n", + " return has_import(src)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64175ad3", + "metadata": {}, + "outputs": [], + "source": [ + "mk_cell = lambda s,id=1: dict(id=id,source=s,cell_type=CellType.CODE)\n", + "assert is_cell_exported(mk_cell('import re'))\n", + "assert is_cell_exported(mk_cell('from re import Pattern'))\n", + "assert is_cell_exported(mk_cell('#| export'))\n", + "assert not is_cell_exported(mk_cell('#| noexport'))\n", + "assert not is_cell_exported(mk_cell('#| noexport\\n#| export'))\n", + "assert not is_cell_exported(mk_cell('#| export\\n#| noexport'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ddc5c9e6", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def get_default_export(cells:list[dict])->str:\n", + " \"Retrieve the `default_exp` directive argument from a list of cells.\"\n", + " matches = [\n", + " m.group('module')\n", + " for c in cells\n", + " for l in c['source'].splitlines()\n", + " if c['cell_type'] == CellType.CODE and (m:=DIRECTIVE_DEFAULT_EXP_RE.match(l))\n", + " ]\n", + " if len(matches) > 1: raise ValueError('More than one default_exp directive')\n", + " if len(matches) == 0: raise ValueError('Missing default_exp directive')\n", + " return matches[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7310f069", + "metadata": {}, + "outputs": [], + "source": [ + "assert get_default_export(read_nb('export.ipynb')['cells']) == 'export'\n", + "\n", + "try: get_default_export([mk_cell('a=5')]); assert False, \"Expected ValueError\"\n", + "except ValueError: pass\n", + "\n", + "try:\n", + " get_default_export([mk_cell('#| default_exp: hello'), mk_cell('#| default_exp: test')])\n", + " assert False, \"Expected ValueError\"\n", + "except ValueError: pass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ce855f0", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def find_cells_to_export(\n", + " fp:str|IO[str], # a path or file-like object to read the notebook from\n", + ")->tuple[str,list[dict]]: # returns the module name and list of cells to export\n", + " \"Find the code cells in an ipynb file to export out.\"\n", + " code_cells = filter_cells(read_nb(fp), CellType.CODE)\n", + " module = get_default_export(code_cells)\n", + " cells = [\n", + " {**c, 'source': strip_directives(c['source'])}\n", + " for c in code_cells\n", + " if is_cell_exported(c)\n", + " ]\n", + " return module, cells" + ] + }, + { + "cell_type": "markdown", + "id": "66574d29", + "metadata": {}, + "source": [ + "## Exporting a Notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3070f710", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def fmt_export_cell(cell:dict)->dict:\n", + " \"Format a code cell's source with metadata to prepare for export.\"\n", + " cell['source'] = f'# cell: {cell[\"id\"]}\\n' + cell['source'] + '\\n'\n", + " return cell" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e24f633f", + "metadata": {}, + "outputs": [], + "source": [ + "assert fmt_export_cell(mk_cell('a=5', id=1))['source'] == '# cell: 1\\na=5\\n'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "633e256b", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def mk_py_mod_scaffold(\n", + " mod:str, # The Python module name or dotted path e.g. mod.mymod.file\n", + " src_dir:str|pathlib.Path, # The Python package source code directory scaffold\n", + "):\n", + " \"Create all intermediary __init__.py files and directories, if any, and the final .py file.\"\n", + " if not mod: raise ValueError('The module path cannot be empty')\n", + " src_dir = pathlib.Path(src_dir)\n", + " mod_file = (src_dir/pathlib.Path(*mod.split('.'))).with_suffix('.py')\n", + " mod_file.parent.mkdir(parents=True, exist_ok=True)\n", + " mod_file.touch()\n", + " pkg_dirs = list(mod_file.parents)\n", + " for p in pkg_dirs[:pkg_dirs.index(src_dir) + 1]:\n", + " (p / '__init__.py').touch()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c044d6d4", + "metadata": {}, + "outputs": [], + "source": [ + "#| noexport\n", + "import os\n", + "import tempfile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e19b3289", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as tmp:\n", + " try: mk_py_mod_scaffold('', src_dir=tmp); assert False, \"Should raise ValueError\"\n", + " except ValueError: pass\n", + " assert os.listdir(tmp) == []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3a33968", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as tmp:\n", + " mk_py_mod_scaffold('file', src_dir=tmp)\n", + " assert os.listdir(tmp) == ['__init__.py', 'file.py']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2e3fcb4", + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as tmp:\n", + " mk_py_mod_scaffold('mod.file', src_dir=tmp)\n", + " assert os.listdir(tmp) == ['__init__.py', 'mod']\n", + " assert os.listdir(pathlib.Path(tmp)/'mod') == ['__init__.py', 'file.py']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3ab7819", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def export_nb(\n", + " fp:str|IO[str], # The notebook to export\n", + " src_dir:str|pathlib.Path='.', # The source directory to export the Python module to.\n", + "):\n", + " \"Export the exportable code cells to a Python file.\"\n", + " module, cells = find_cells_to_export(fp)\n", + " mk_py_mod_scaffold(module, src_dir)\n", + " exp_path = (pathlib.Path(src_dir)/module).with_suffix('.py')\n", + " src = '\\n'.join(\n", + " ['# AUTOGENERATED! DO NOT EDIT!\\n'] +\n", + " list(map(lambda c: fmt_export_cell(c)['source'], cells))\n", + " )\n", + " pathlib.Path(exp_path).write_text(src)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c85bc348", + "metadata": {}, + "outputs": [], + "source": [ + "#| noexport\n", + "from io import StringIO" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55f57bee", + "metadata": {}, + "outputs": [], + "source": [ + "def mk_nb(cells)->StringIO:\n", + " nb = nbformat.v4.new_notebook(cells=cells)\n", + " buf = StringIO()\n", + " nbformat.write(nb, buf)\n", + " buf.seek(0)\n", + " return buf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31bb0f0b", + "metadata": {}, + "outputs": [], + "source": [ + "cells = [\n", + " nbformat.v4.new_code_cell('#| default_exp: testmod'),\n", + " nbformat.v4.new_code_cell('import os'),\n", + " nbformat.v4.new_code_cell('#| export\\ndef foo(): pass'),\n", + " nbformat.v4.new_code_cell('#| noexport\\nimport pathlib'),\n", + "]\n", + "nb = mk_nb(cells)\n", + "src = f\"\"\"# AUTOGENERATED! DO NOT EDIT!\n", + "\n", + "# cell: {cells[1]['id']}\n", + "import os\n", + "\n", + "# cell: {cells[2]['id']}\n", + "def foo(): pass\n", + "\"\"\"\n", + "with tempfile.TemporaryDirectory() as tmp:\n", + " export_nb(nb, src_dir=tmp)\n", + " result = pathlib.Path(tmp, 'testmod.py').read_text()\n", + " assert result == src, result" + ] + }, + { + "cell_type": "markdown", + "id": "e7c49602", + "metadata": {}, + "source": [ + "And...we've successfully bootstrapped our notebook! Let's export it out." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e1d4096", + "metadata": {}, + "outputs": [], + "source": [ + "export_nb('export.ipynb', src_dir='../nb/')" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/test.ipynb b/nbs/test.ipynb new file mode 100644 index 0000000..d446272 --- /dev/null +++ b/nbs/test.ipynb @@ -0,0 +1,418 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d42a9120-8d70-4b10-8d6c-b59692fd03ad", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11cb390f-b0f4-438d-b524-db7242d22ee9", + "metadata": {}, + "outputs": [], + "source": [ + "#| default_exp: test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd03783f-252c-40f5-b92f-0f3f9278ce09", + "metadata": {}, + "outputs": [], + "source": [ + "import ast,copy,contextlib,inspect,textwrap,traceback\n", + "from typing import Any,Generator\n", + "from dataclasses import dataclass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cf5d527", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name '_T' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mNameError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[9]\u001b[39m\u001b[32m, line 2\u001b[39m", + "\u001b[32m 1\u001b[39m \u001b[38;5;66;03m#| noexport\u001b[39;00m", + "\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m nb.core \u001b[38;5;28;01mimport\u001b[39;00m pubattr", + "", + "\u001b[36mFile \u001b[39m\u001b[32m~/nb/nb/core.py:28\u001b[39m", + "\u001b[32m 25\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m inner", + "\u001b[32m 27\u001b[39m \u001b[38;5;66;03m# cell: 42580f8b\u001b[39;00m", + "\u001b[32m---> \u001b[39m\u001b[32m28\u001b[39m \u001b[38;5;129m@patch_to\u001b[39m(\u001b[30;43m_T\u001b[39;49m)", + "\u001b[32m 29\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msay_hello\u001b[39m(\u001b[38;5;28mself\u001b[39m: _T, name: \u001b[38;5;28mstr\u001b[39m) -> \u001b[38;5;28mstr\u001b[39m:", + "\u001b[32m 30\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33mHello, \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m", + "\u001b[32m 32\u001b[39m \u001b[38;5;66;03m# cell: 4eb650ab\u001b[39;00m", + "", + "\u001b[31mNameError\u001b[39m: name '_T' is not defined" + ] + } + ], + "source": [ + "#| noexport\n", + "from nb.core import pubattr" + ] + }, + { + "cell_type": "markdown", + "id": "5b8abf6a-c94a-43e5-a1d1-81b1a1d64656", + "metadata": {}, + "source": [ + "# Testing\n", + "> Utilities for testing within notebooks" + ] + }, + { + "cell_type": "markdown", + "id": "12e34b1f", + "metadata": {}, + "source": [ + "## Re-constructing source code and program state during runtime" + ] + }, + { + "cell_type": "markdown", + "id": "93a3d5fe", + "metadata": {}, + "source": [ + "When writing test libraries, it can be incredibly useful to understand what the call site source code looks like.\n", + "\n", + "For instance, let's say you call `equal(x in [1, 2, 3], should_exist)`. When we're in `equal`, the arguments have already been evaluated and we simply have two booleans as arguments, each `True` or `False`. If this test fails, it likely isn't *that* helpful to see `False != True`. \n", + "\n", + "Instead, it is likely much *more* helpful to see something like `x in [1, 2, 3] != should_exist` where `x = 4` and `should_exist=True`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6c67919", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def get_call_args(frames_up: int = 1) -> tuple[list[str], dict[str,Any]]:\n", + " \"Get the source code expressions of the caller's arguments.\"\n", + " fm_info = inspect.getouterframes(inspect.currentframe())[frames_up+1]\n", + " src, _ = inspect.getsourcelines(fm_info.frame)\n", + " srcl = src[fm_info.positions.lineno - 1:fm_info.positions.end_lineno]\n", + " if len(srcl) == 0:\n", + " raise RuntimeError(\n", + " \"`get_call_args` is designed to be called from within a function, \" +\n", + " \"not at the top-level. If you want to call it at the top level, pass `frames_up=0`.\"\n", + " )\n", + " elif len(srcl) == 1:\n", + " fn_src = srcl[0][fm_info.positions.col_offset:fm_info.positions.end_col_offset]\n", + " else:\n", + " fn_src = \"\".join(\n", + " [srcl[0][fm_info.positions.col_offset:]] +\n", + " srcl[1:-1] +\n", + " [srcl[-1][:fm_info.positions.end_col_offset]]\n", + " )\n", + " call_node = ast.parse(fn_src).body[0].value\n", + " args_src = [ast.unparse(arg) for arg in call_node.args]\n", + " kwargs_src = {kw.arg: ast.unparse(kw.value) for kw in call_node.keywords}\n", + " return args_src, kwargs_src" + ] + }, + { + "cell_type": "markdown", + "id": "487c29d9", + "metadata": {}, + "source": [ + "Because `get_call_args()` is expected to be used from within a function, it generally will not work properly by default if you call it from the top-level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbe2859c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "`get_call_args` is designed to be called from within a function, not at the top-level. If you want to call it at the top level, pass `frames_up=0`.\n" + ] + } + ], + "source": [ + "try: get_call_args(); assert False, \"should have raised\"\n", + "except RuntimeError as e: print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "ce788841", + "metadata": {}, + "source": [ + "However, you _can_ have it work if you pass `frames_up=0` (i.e. the current frame we're calling `get_call_args()` from rather than the caller frame)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f3b1abb", + "metadata": {}, + "outputs": [], + "source": [ + "args, kwargs = get_call_args(frames_up=0)\n", + "assert args == [], f\"Got {args}\"\n", + "assert kwargs == {\"frames_up\": \"0\"}, f\"Got {kwargs}\"" + ] + }, + { + "cell_type": "markdown", + "id": "5a48fa09", + "metadata": {}, + "source": [ + "When you're in a function, it works as expected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7730389", + "metadata": {}, + "outputs": [], + "source": [ + "def f(a, b=1):\n", + " return get_call_args()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19890aa0", + "metadata": {}, + "outputs": [], + "source": [ + "args, kwargs = f(1+1,b=max(10, 20))\n", + "assert args == [\"1 + 1\"], f\"Got {args}\"\n", + "assert kwargs == {\"b\": \"max(10, 20)\"}, f\"Got {kwargs}\"" + ] + }, + { + "cell_type": "markdown", + "id": "b9aba848", + "metadata": {}, + "source": [ + "It also works with multi-line function calls.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8b7195c", + "metadata": {}, + "outputs": [], + "source": [ + "args, kwargs = f(\n", + " 1,\n", + " b=10,\n", + " \n", + " \n", + ")\n", + "assert args == [\"1\"], f\"Got {args}\"\n", + "assert kwargs == {\"b\": \"10\"}, f\"Got {kwargs}\"" + ] + }, + { + "cell_type": "markdown", + "id": "f93a8882", + "metadata": {}, + "source": [ + "If you use it with a function within a function, you'll get the arguments for the inner function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce0666f5", + "metadata": {}, + "outputs": [], + "source": [ + "def nested_f(a, b=1):\n", + " def inner_f(c, d, e=1):\n", + " return get_call_args()\n", + " return inner_f(5 * 5, 10 / 10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3555f6cf", + "metadata": {}, + "outputs": [], + "source": [ + "args, kwargs = nested_f(1)\n", + "assert args == [\"5 * 5\", \"10 / 10\"], f\"Got {args}\"\n", + "assert kwargs == {}, f\"Got {kwargs}\"" + ] + }, + { + "cell_type": "markdown", + "id": "97d47759", + "metadata": {}, + "source": [ + "However, you can still get the outer function arguments with the `frames_up` keyword argument." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a42fc2b", + "metadata": {}, + "outputs": [], + "source": [ + "def nested_f(a, b=1):\n", + " def inner_f(c, d, e=1):\n", + " return get_call_args(frames_up=2)\n", + " return inner_f(5 * 5, 10 / 10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3687819", + "metadata": {}, + "outputs": [], + "source": [ + "args, kwargs = nested_f(1)\n", + "assert args == [\"1\"], f\"Got {args}\"\n", + "assert kwargs == {}, f\"Got {kwargs}\"" + ] + }, + { + "cell_type": "markdown", + "id": "5d991fd4", + "metadata": {}, + "source": [ + "## Assertions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "623a480c", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "@dataclass\n", + "class ExceptionInfo:\n", + " type_: type[Exception] | None = None\n", + " value: Exception | None = None\n", + " traceback: str | None = None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ad5fcef-f04d-469d-8ad1-94d3a98694e4", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "@contextlib.contextmanager\n", + "def raises(\n", + " exc: type[Exception] = Exception, # The exception type to check is raised.\n", + " contains: str | None = None, # Optionally, check whether the raised exception contains this message.\n", + ") -> Generator[ExceptionInfo, None, None]:\n", + " \"Asserts that the body of the context manager raises `exc`.\"\n", + " raised = True\n", + " try:\n", + " info = ExceptionInfo()\n", + " yield info\n", + " raised = False\n", + " except exc as e:\n", + " info.type_ = type(e)\n", + " info.value = e\n", + " info.traceback = traceback.format_tb(e.__traceback__)\n", + " if contains and contains not in str(e):\n", + " raise AssertionError(f\"Raised {e} but message does not contain {contains}\") from e\n", + " if not raised:\n", + " raise AssertionError(f\"Did not raise {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c478098-7a09-403c-a62b-3c34cf0f7bab", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ExceptionInfo(type_=<class 'Exception'>, value=Exception('abc'), traceback=[' File \"/tmp/ipykernel_1287/933907959.py\", line 11, in raises\\n yield info\\n', ' File \"/tmp/ipykernel_1287/3922684010.py\", line 2, in <module>\\n raise Exception(\"abc\")\\n'])\n" + ] + } + ], + "source": [ + "with raises() as e:\n", + " raise Exception(\"abc\")\n", + "print(e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b503871-3f39-4013-a7ae-f5143af3138e", + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def equal(a, b):\n", + " \"Assert that a == b.\"\n", + " args_src, _ = get_call_args()\n", + " if a != b:\n", + " msg = (\n", + " f\"FAIL: {args_src[0]} != {args_src[1]}\\n\" +\n", + " ' ' * (len(AssertionError.__name__)+2) +\n", + " f\" {a!r} != {b!r}\"\n", + " )\n", + " raise AssertionError(msg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc7872bb", + "metadata": {}, + "outputs": [], + "source": [ + "with raises(exc=AssertionError) as e:\n", + " equal(1, 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1393a7-6a89-4c5d-a8a7-51e0fd46d190", + "metadata": {}, + "outputs": [], + "source": [ + "#| noexport\n", + "import nb.export; nb.export.export_nb('test.ipynb', src_dir='../nb/')" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0e87d5f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "nb" +version = "0.1.0" +description = "nb implements a notebook-driven development workflow" +readme = "README.md" +requires-python = ">=3.12" +dependencies = ["nbconvert"] +authors = [{ name = "Eugene Triguba", email = "hello@eugenetriguba.com" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", +] + +[dependency-groups] +dev = ["jupyter"] + +[tool.setuptools] +packages = ["nb"] + +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" |