diff --git a/CHANGELOG.md b/CHANGELOG.md index e0d0141..e988daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ Every change a user would notice, newest first ([Keep a Changelog](https://keepa follow [Semantic Versioning](https://semver.org/); before 1.0, a minor version may change what a command does. ## [Unreleased] +### Added + +- Imports are followed from the folders a project puts on the import path, as Python does with `sys.path`: with `src` + named in `pyproject.toml` (pyright's `extraPaths`, pytest's `pythonpath`, mypy's `mypy_path`, setuptools' `where` or + `package-dir`, Poetry's `packages ... from`), in `pyrightconfig.json` or in Pylance's `.vscode/settings.json`, + `from common.db import save` finds `src/common/db.py` from any file, however deep. Before, it was taken for an + installed library and left off the page. A `pyproject.toml` in a subfolder counts from its own folder, and + `--path DIR` adds a folder no setting names. + ## [0.8.0] - 2026-10-21 ### Added diff --git a/README.md b/README.md index 108cf78..9fd418d 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ string is flagged. | `-o`, `--out`, `--no-open` | Where to save (default `~/.cache/code-node/`), and whether to open it | | `--platform NAME`, `--lambda` | Run a platform's checks on any entry point | | `--editor vscode\|cursor\|none` | Editor the line links open (default `vscode`, WSL-aware) | +| `--path DIR` | A folder imports start from, as on `PYTHONPATH` (pyright's `extraPaths` and pytest's `pythonpath` are read on their own) | | `--no-git` | Leave out what git says about each function (who changed it last, when, in which commit) | diff --git a/docs/usage.md b/docs/usage.md index 023981f..fbb87c5 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -70,6 +70,24 @@ functions, or a function's code. Tests, virtual envs and build folders are skipped. +??? more "Imports from folders on the path" + A file's imports are found the way Python finds them: from the top of its package, from the file's own folder, and + from the folders the project puts on the import path. With `src` in one of these, `from common.db import save` in + `src/handlers/orders/app.py` finds `src/common/db.py`: + + | File | Setting | + | --- | --- | + | `pyproject.toml` | `[tool.pyright]` `extraPaths`, `[tool.pytest.ini_options]` `pythonpath`, `[tool.mypy]` `mypy_path` | + | `pyproject.toml` | `[tool.setuptools.packages.find]` `where`, `[tool.setuptools.package-dir]` `""`, Poetry's `packages` `from` | + | `pyrightconfig.json` | `extraPaths` | + | `.vscode/settings.json` | `python.analysis.extraPaths` | + + A `pyproject.toml` in a subfolder counts from its own folder. For a folder no setting names, pass it: + + ```bash + code-node src/handlers/orders/app.py --path src + ``` + ## Go live { #serve } ```bash @@ -138,4 +156,5 @@ to see its commit. The gear in the panel's head turns that off. It's read with ` | `--no-open` | Don't open a browser | | `--platform`, `--lambda` | Run a platform's checks | | `--editor` | What line links open: `vscode`, `cursor` or `none` | +| `--path` | A folder imports start from, as on `PYTHONPATH` | | `--no-git` | Leave out who changed each function, and when | diff --git a/src/code_node/analysis/mapping.py b/src/code_node/analysis/mapping.py index ad741ed..3635227 100644 --- a/src/code_node/analysis/mapping.py +++ b/src/code_node/analysis/mapping.py @@ -64,6 +64,7 @@ def map_code( max_depth: int | None = None, platform: str = "", deployment: Deployment | None = None, + paths: Iterable[str] = (), ) -> CodeMap: """ How Python code works, read with Python's ast module and never run: files is {path: source}, entry the function @@ -75,6 +76,8 @@ def map_code( runs (a code_node.platforms name: 'lambda' adds the checks only a Lambda handler needs), '' for plain Python. deployment is what a deploy file says about where it runs (code_node.deploys: a function in template.yaml): its runtime, environment variables and role are checked against the code, and it sets the platform when none is given. + paths are the other folders imports start from, as on sys.path ('src/' for `from common.db import save` to find + src/common/db.py), each as a prefix of the files' paths: '' for the top. map_code({"app.py": open("app.py").read()}, "app.handler") """ @@ -98,7 +101,14 @@ def map_code( libs = set(libraries) | _RUNTIME_MODULES | _LIBRARIES libs.discard(entry.split(".")[0]) # its own top package is never a library reader = _CodeReader( - texts, names, root, cmap, _python_version(python), libs, bool(deployment is not None and deployment.layers) + texts, + names, + root, + cmap, + _python_version(python), + libs, + bool(deployment is not None and deployment.layers), + tuple(paths), ) if max_depth is not None: reader.max_depth = max_depth diff --git a/src/code_node/analysis/reader.py b/src/code_node/analysis/reader.py index e783a89..69e4176 100644 --- a/src/code_node/analysis/reader.py +++ b/src/code_node/analysis/reader.py @@ -183,6 +183,7 @@ def __init__( version: tuple[int, int] | None, libraries: set[str], layers: bool, + paths: tuple[str, ...] = (), ): self.texts, self.root, self.cmap, self.version, self.layers = texts, root, cmap, version, layers self.platform = get_platform(cmap.platform) @@ -205,20 +206,24 @@ def __init__( self.sources[module] = path here = (cmap.handler_file or "").rpartition("/")[0] here = here + "/" if here else "" - if here.startswith(root) and here != root: - # the entry point's own folder is on sys.path too, as a script's is and a Lambda function's code is: - # `from helpers import x` beside it is its own code, though the folder is also a package from the root - beside: set[str] = set() + # the entry point's own folder is on sys.path too, as a script's is and a Lambda function's code is: + # `from helpers import x` beside it is its own code, though the folder is also a package from the root. So + # are the folders the project puts there (paths: pyright's extraPaths, pytest's pythonpath, --path), where + # `from common.db import save` finds src/common/db.py + places = [here] if here.startswith(root) and here != root else [] + places += [place for place in paths if place != root and place not in places] + beside: set[str] = set() + for place in places: for path in texts: - if not path.startswith(here) or not path.endswith(".py") or "site-packages" in path: + if not path.startswith(place) or not path.endswith(".py") or "site-packages" in path: continue - module = path[len(here) : -3].replace("/", ".") + module = path[len(place) : -3].replace("/", ".") module = module[: -len(".__init__")] if module.endswith(".__init__") else module if module and module != "__init__" and "-" not in module and module not in self.sources: self.sources[module] = path beside.add(module.split(".")[0]) - self.tops |= beside - self.libraries = libraries - beside + self.tops |= beside + self.libraries = libraries - beside self.modules: dict[str, _Module] = {} self.defs: dict[str, tuple[str, Any, str | None]] = {} # key -> (module, node, class key) self.classes: dict[str, dict[str, Any]] = {} diff --git a/src/code_node/cli.py b/src/code_node/cli.py index cc7f47a..380c9a4 100644 --- a/src/code_node/cli.py +++ b/src/code_node/cli.py @@ -114,6 +114,15 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: default="vscode", help="the editor the page's line links open (default vscode; none for no links)", ) + parser.add_argument( + "--path", + dest="paths", + action="append", + default=[], + metavar="DIR", + help="a folder imports start from, as on PYTHONPATH (again for more): --path src for `from common.db import x` " + "to find src/common/db.py. pyright's extraPaths and pytest's pythonpath are read on their own", + ) parser.add_argument( "--no-git", action="store_true", help="leave out what git says about each function (who changed it, when)" ) @@ -306,6 +315,11 @@ def main(argv: list[str] | None = None) -> int: if not project.files: _fail(f"No Python files under {project.root}.") return 1 + paths = [Path(path).expanduser().resolve() for path in args.paths] + outside = [path for path in paths if path != project.root and project.root not in path.parents] + if outside or not all(path.is_dir() for path in paths): + _fail(f"--path {(outside or [p for p in paths if not p.is_dir()])[0]} isn't a folder in {project.root}.") + return 2 if args.list: found = entry_points(project) console.print( @@ -333,6 +347,7 @@ def main(argv: list[str] | None = None) -> int: platform=args.platform, editor=args.editor, git=not args.no_git, + paths=tuple(str(path) for path in paths), ) if args.mermaid: print(make_mermaid(project.root, entry, options)) diff --git a/src/code_node/pages.py b/src/code_node/pages.py index 547cdf8..57ba9a8 100644 --- a/src/code_node/pages.py +++ b/src/code_node/pages.py @@ -14,7 +14,7 @@ from .analysis.model import CodeMap from .deploys import find_deployment from .history import read_history -from .project import EntryPoint, Project, read_project +from .project import EntryPoint, Project, folder_prefixes, read_project from .report.build import page_blocks from .report.editor import editor_link from .report.style import render_page @@ -40,6 +40,7 @@ class PageOptions: platform: str = "" # --platform: its checks run whatever the entry point is ('' leaves it to the entry point) editor: str = "vscode" # --editor: the editor its line links open ('vscode', 'cursor'; 'none' for no links) git: bool = True # what git says about each function (--no-git leaves it out) + paths: tuple[str, ...] = () # --path: more folders imports start from, beside those the project's settings name @property def limits(self) -> tuple[int, int]: @@ -56,6 +57,8 @@ def map_entry(root: Path, entry: EntryPoint, options: PageOptions) -> tuple[Code project = read_project(root) file = entry.file or (entry_file(entry.name, project.files) or ("",))[0] deployment = find_deployment(project.deploy_files, project.files, file, entry.name.rsplit(".", 1)[-1]) + given = folder_prefixes(project.root, [Path(path) for path in options.paths]) + project.paths += [prefix for prefix in given if prefix not in project.paths] cmap = map_code( project.files, entry.name, @@ -64,6 +67,7 @@ def map_entry(root: Path, entry: EntryPoint, options: PageOptions) -> tuple[Code max_depth=_ALL_READ_DEPTH if options.draw_all else None, platform=options.platform or entry.platform, deployment=deployment, + paths=project.paths, ) if options.git: read_history(project.root, cmap.functions) diff --git a/src/code_node/project.py b/src/code_node/project.py index df355ea..05da4fa 100644 --- a/src/code_node/project.py +++ b/src/code_node/project.py @@ -5,6 +5,7 @@ from __future__ import annotations import ast +import json import os import re import sys @@ -48,6 +49,17 @@ _IMPORT_RE = re.compile(r"^[ \t]*(?:from[ \t]+([\w.]+)[ \t]+import|import[ \t]+([\w., \t]+))", re.M) +_PATH_SETTINGS = ( # where a pyproject.toml puts folders on the import path: (table, key); "" is the key "" + ("tool.pyright", "extraPaths"), + ("tool.basedpyright", "extraPaths"), + ("tool.pytest.ini_options", "pythonpath"), + ("tool.mypy", "mypy_path"), + ("tool.setuptools.packages.find", "where"), # where = ["src"]: the packages are in src + ("tool.setuptools.package-dir", ""), # "" = "src": the same +) + +_VSCODE_PATHS_RE = re.compile(r'"python\.(?:analysis|autoComplete)\.extraPaths"\s*:\s*\[(.*?)\]', re.S) + _CLI_APPS = ("Typer",) # web apps are a platform's (FastAPI for the web one, Chalice for Lambda's) @@ -78,6 +90,8 @@ class Project: configs: list[str] = field(default_factory=list) # its pyproject.toml files, the root's first deploy_files: dict[str, str] = field(default_factory=dict) # files deploy plugins read (template.yaml, # serverless.yml): {path from the folder: text} + paths: list[str] = field(default_factory=list) # the other folders imports start from, as the project puts + # them on sys.path (pyright's extraPaths, pytest's pythonpath): 'src/', or '' for the folder itself _trees: dict[str, ast.Module] | None = None @property @@ -105,7 +119,12 @@ def libraries(self) -> set[str]: """ Every top-level module the code imports that isn't its own: read as installed libraries, never as missing. """ - own = self.own_tops + own = self.own_tops | { + rel[len(place) :].split("/")[0].split(".")[0] + for place in self.paths + for rel in self.files + if rel.startswith(place) + } found: set[str] = set() for text in self.files.values(): for match in _IMPORT_RE.finditer(text): @@ -149,6 +168,7 @@ def read_project(root: str | os.PathLike[str]) -> Project: rel = path.relative_to(base).as_posix() if len(project.files) >= _MAX_FILES: project.limited = True + project.paths = import_paths(base, project.configs) return project if path.stat().st_size > _MAX_FILE_BYTES: project.skipped.append(rel) @@ -156,9 +176,96 @@ def read_project(root: str | os.PathLike[str]) -> Project: text = path.read_text("utf-8", "replace") project.files[rel] = text project.modules[rel] = _module_name(path, base) + project.paths = import_paths(base, project.configs) return project +def _pyproject_paths(text: str) -> list[str]: + """ + The folders a pyproject.toml puts on the import path, as it writes them: ['src'] for pyright's + extraPaths = ["src"], setuptools' where = ["src"] or Poetry's packages = [{include = "app", from = "src"}]. + """ + try: + import tomllib + except ImportError: # Python 3.10: find the tables and the keys by hand + found: list[str] = [] + for table, key in _PATH_SETTINGS: + body = re.search(rf"^\[{re.escape(table)}\]\s*$(.*?)(?=^\[|\Z)", text, re.M | re.S) + name = re.escape(key) if key else "(?:\"\"|'')" + value = body and re.search(rf"^\s*{name}\s*=\s*(\[.*?\]|\"[^\"]*\"|'[^']*')", body.group(1), re.M | re.S) + if value and value.group(1).startswith("["): + found += re.findall(r"[\"']([^\"']*)[\"']", value.group(1)) + elif value: + found += re.split(r"[\s,:]+", value.group(1)[1:-1]) + for body in re.findall(r"^\[\[?tool\.poetry(?:\.packages)?\]\]?\s*$(.*?)(?=^\[|\Z)", text, re.M | re.S): + found += re.findall(r"\bfrom\s*=\s*[\"']([^\"']+)[\"']", body) + return [item for item in found if item] + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError: + return [] + found = [] + for table, key in _PATH_SETTINGS: + value: object = data + for part in [*table.split("."), key]: + value = value.get(part) if isinstance(value, dict) else None + if isinstance(value, str): # pytest's pythonpath = "src lib", mypy's mypy_path = "src:lib" + found += re.split(r"[\s,:]+", value) + elif isinstance(value, list): + found += [str(item) for item in value] + packages = data.get("tool", {}).get("poetry", {}).get("packages", []) + found += [str(package["from"]) for package in packages if isinstance(package, dict) and "from" in package] + return [item for item in found if item] + + +def import_paths(root: Path, configs: list[str] | None = None) -> list[str]: + """ + The folders under root the project puts on the import path, as prefixes of its files' paths ('src/', or '' for + root itself): what its pyproject.toml files say (configs, from root: pyright's extraPaths, pytest's pythonpath, + mypy's mypy_path, where setuptools or Poetry finds the packages), each from its own folder, and pyright's (and + Pylance's) extraPaths in pyrightconfig.json or .vscode/settings.json. Folders outside root, or that aren't there, + are left out. + """ + found: list[Path] = [] + for config in configs if configs is not None else ["pyproject.toml"]: + pyproject = root / config + if pyproject.is_file(): + found += [pyproject.parent / item for item in _pyproject_paths(pyproject.read_text("utf-8", "replace"))] + pyright = root / "pyrightconfig.json" + if pyright.is_file(): + try: + data = json.loads(pyright.read_text("utf-8", "replace")) + except ValueError: + data = {} + extra = data.get("extraPaths") if isinstance(data, dict) else None + found += [root / str(item) for item in extra] if isinstance(extra, list) else [] + vscode = root / ".vscode" / "settings.json" + if vscode.is_file(): # JSON with comments, so the lists are found as text + for match in _VSCODE_PATHS_RE.finditer(vscode.read_text("utf-8", "replace")): + found += [ + root / item.replace("${workspaceFolder}", ".") for item in re.findall(r'"([^"]*)"', match.group(1)) + ] + return folder_prefixes(root, found) + + +def folder_prefixes(root: Path, folders: list[Path]) -> list[str]: + """ + Folders as prefixes of the paths of root's files: [root/'src'] -> ['src/']; root itself is ''. Folders outside + root, or that aren't there, are left out. + """ + base = root.resolve() + prefixes: list[str] = [] + for folder in folders: + folder = folder.resolve() + if not folder.is_dir() or (folder != base and base not in folder.parents): + continue + rel = folder.relative_to(base).as_posix() + prefix = "" if rel == "." else rel + "/" + if prefix not in prefixes: + prefixes.append(prefix) + return prefixes + + def _console_scripts(path: Path) -> dict[str, str]: """ A pyproject.toml's console scripts: {'code-node': 'code_node.cli:main'}. diff --git a/tests/test_code_node.py b/tests/test_code_node.py index a804928..8e8198a 100644 --- a/tests/test_code_node.py +++ b/tests/test_code_node.py @@ -1267,3 +1267,75 @@ def page_blocks_for(project, entry: str) -> list: cmap = map_code(project.files, entry, libraries=project.libraries(), platform="lambda") return page_blocks(cmap, project="x", max_functions=24, max_depth=5, sources=project.files) + + +IMPORTED_FROM_SRC = { + "src/common/__init__.py": "", + "src/common/db.py": ( + 'import boto3\n\ntable = boto3.resource("dynamodb").Table("orders")\n\n\n' + "def save(item):\n table.put_item(Item=item)\n" + ), + "src/handlers/orders/app.py": "from common.db import save\n\n\ndef handler(event, context):\n save(event)\n", +} + + +@pytest.mark.parametrize( + ("name", "text"), + [ + ("pyproject.toml", '[tool.pyright]\nextraPaths = ["src"]\n'), + ("pyproject.toml", '[tool.pytest.ini_options]\npythonpath = "src"\n'), + ("pyproject.toml", '[tool.mypy]\nmypy_path = "src"\n'), + ("pyproject.toml", '[tool.setuptools.packages.find]\nwhere = ["src"]\n'), + ("pyproject.toml", '[tool.setuptools.package-dir]\n"" = "src"\n'), + ("pyproject.toml", '[tool.poetry]\npackages = [{ include = "common", from = "src" }]\n'), + ("pyproject.toml", '[[tool.poetry.packages]]\ninclude = "common"\nfrom = "src"\n'), + ("pyrightconfig.json", '{"extraPaths": ["./src"]}'), + (".vscode/settings.json", '{\n // Pylance\n "python.analysis.extraPaths": ["${workspaceFolder}/src"],\n}\n'), + ], +) +def test_imports_from_a_folder_the_project_puts_on_the_path(tmp_path: Path, name: str, text: str, capsys) -> None: + root = write(tmp_path / "repo", {**IMPORTED_FROM_SRC, name: text, ".git/HEAD": "ref: main\n"}) + project = read_project(root) + assert project.paths == ["src/"] and "common" not in project.libraries() + out = tmp_path / "page.html" + assert main([str(root / "src/handlers/orders/app.py:handler"), "--no-open", "-o", str(out)]) == 0 + assert "app.handler: 2 functions, 1 AWS calls" in capsys.readouterr().out + + +def test_imports_from_a_folder_given_with_path(tmp_path: Path, capsys) -> None: + root = write(tmp_path / "repo", {**IMPORTED_FROM_SRC, ".git/HEAD": "ref: main\n"}) + out = tmp_path / "page.html" + file = str(root / "src/handlers/orders/app.py:handler") + assert main([file, "--no-open", "-o", str(out)]) == 0 + assert "app.handler: 1 functions, 0 AWS calls" in capsys.readouterr().out # common.db read as a library + assert main([file, "--path", str(root / "src"), "--no-open", "-o", str(out)]) == 0 + assert "app.handler: 2 functions, 1 AWS calls" in capsys.readouterr().out + assert main([file, "--path", str(tmp_path), "--no-open", "-o", str(out)]) == 2 + assert "isn't a folder in" in capsys.readouterr().err + cmap = map_code( + {path: text for path, text in IMPORTED_FROM_SRC.items()}, + "app.handler", + file="src/handlers/orders/app.py", + paths=["src/"], + ) + assert {f.name for f in cmap.functions} >= {"app.handler", "common.db.save"} + + +def test_a_pyproject_in_a_subfolder_names_paths_from_there(tmp_path: Path) -> None: + files = {f"services/orders/{name}": text for name, text in IMPORTED_FROM_SRC.items()} + files["services/orders/pyproject.toml"] = '[tool.pyright]\nextraPaths = ["src", "../../missing"]\n' + assert read_project(write(tmp_path, files)).paths == ["services/orders/src/"] + + +def test_pyproject_paths_without_tomllib(monkeypatch) -> None: + from code_node.project import _pyproject_paths + + text = ( + '[project]\nname = "x"\n[tool.pyright]\nextraPaths = [\n "src",\n "lib",\n]\n' + '[tool.mypy]\nmypy_path = "a:b"\n[tool.setuptools.package-dir]\n"" = "pkgs"\n' + '[[tool.poetry.packages]]\ninclude = "app"\nfrom = "apps"\n' + ) + expected = ["src", "lib", "a", "b", "pkgs", "apps"] + assert _pyproject_paths(text) == expected + monkeypatch.setitem(__import__("sys").modules, "tomllib", None) # as on Python 3.10 + assert _pyproject_paths(text) == expected