diff --git a/README.md b/README.md index 64d7f94..c930128 100644 --- a/README.md +++ b/README.md @@ -614,6 +614,61 @@ def score(...): ... ``` +Declare the environment in a config file instead with `TaskEnv.from_config(path)`. A relative `path` resolves against the **calling pipeline module**, never the working directory, so the same script selects the same config wherever `tangle` runs from: + +```python +EVAL = TaskEnv.from_config("envs.yaml") # relative to THIS file +``` + +```yaml +# envs.yaml +image: python:3.12 +dependencies_from: pyproject.toml # relative to THIS config file +``` + +The file is read with the same loader `--config` uses, so the `_select` directive works here identically — the case is chosen exactly and case-sensitively by an environment variable, and an unset or unmatched variable is a hard error unless an explicit `default` branch is authored (no implicit default, no implicit production): + +```yaml +_select: + env: DEPLOY_ENVIRONMENT + cases: + production: + image: registry.example/scoring:prod + staging: + image: registry.example/scoring:staging + default: + image: python:3.12 +``` + +The document must resolve to exactly one object, and its keys must be named parameters of the generated `__init__` of the class `from_config` was called on — `InitVar` pseudo-fields included, `ClassVar`s and `field(init=False)` excluded — so these files stay environment-only: pipeline concerns such as file paths, versioning, annotations, schedules, or subscriptions have no field to land in and are rejected with the allowed field names. `from_config` is generic — any `TaskEnv` dataclass subclass inherits it unchanged, returns its own type, and is validated against its own fields: + +```python +from tangle_cli.python_pipeline import TaskEnv + +@dataclass(frozen=True) +class GpuEnv(TaskEnv): + accelerator: str = "" + + def __post_init__(self): + super().__post_init__() + if self.accelerator not in ("gpu", "tpu"): + raise ValueError("GpuEnv.accelerator must be one of: gpu, tpu") + +GPU = GpuEnv.from_config("envs.yaml") # accepts image, dependencies_from, accelerator +``` + +Failures raise `CompileError` naming the resolved config path. A config file is untrusted input, so **no diagnostic echoes a config value**. Keys are rendered through a capped, control-character-scrubbing renderer, and a constructor's own validation text is never quoted — it could embed a value directly, nested inside a structure, or transformed (lower cased, sliced, re-encoded). The rejected exception is also kept off `__cause__`/`__context__`, since `traceback.format_exception` would otherwise print it into the same CI log. The message names the class and the fields present, and points at constructing the class directly to see the validation error: + +``` +GpuEnv.from_config: /repo/envs.yaml case is not a valid GpuEnv (fields present: +accelerator, image). Its validation message is withheld because it can contain +config values; construct GpuEnv(...) directly to see it. +``` + +Calling `GpuEnv(...)` directly is unaffected and raises the ordinary `ValueError` with its full message. + +See `examples/python_pipeline/task_env_from_config/` for a runnable example. + Use `@task(image_id="eval-slim")` when source should carry a logical image name instead of a concrete registry reference. Downstream code can register defaults with `register_image_id(...)`, and callers can override at compile time with repeatable `--image ID=REF`: ```bash diff --git a/examples/python_pipeline/task_env_from_config/envs.yaml b/examples/python_pipeline/task_env_from_config/envs.yaml new file mode 100644 index 0000000..e890ef6 --- /dev/null +++ b/examples/python_pipeline/task_env_from_config/envs.yaml @@ -0,0 +1,25 @@ +# Deployment environments for the tasks in ``pipeline.py``. +# +# ENVIRONMENT-ONLY on purpose: the keys here are exactly the ``TaskEnv`` +# constructor fields (``image``, ``dependencies_from``). Pipeline concerns — +# file paths, versioning, annotations, schedules, subscriptions — have no +# field to land in and are rejected at load time. +# +# ``_select`` is the same directive the CLI's ``--config`` understands: the +# case is chosen by an environment variable, exactly and case sensitively. +# The explicit ``default`` branch is what makes local runs work; without one, +# an unset or unmatched ``DEPLOY_ENVIRONMENT`` is a hard error (there is no +# implicit default and no implicit production). +_select: + env: DEPLOY_ENVIRONMENT + cases: + production: + image: registry.example/scoring:prod + # Relative to THIS file, not to the pipeline module and not to the cwd. + dependencies_from: pyproject.toml + staging: + image: registry.example/scoring:staging + dependencies_from: pyproject.toml + default: + image: python:3.12 + dependencies_from: pyproject.toml diff --git a/examples/python_pipeline/task_env_from_config/pipeline.py b/examples/python_pipeline/task_env_from_config/pipeline.py new file mode 100644 index 0000000..26d020f --- /dev/null +++ b/examples/python_pipeline/task_env_from_config/pipeline.py @@ -0,0 +1,45 @@ +"""Runnable example: load a ``TaskEnv`` from a config file. + +``TaskEnv.from_config`` declares the execution environment once, in +``envs.yaml``, instead of repeating an image string across tasks. The path is +relative to THIS file, so the same script picks the same config wherever +``tangle`` is run from. + +Compile from the repository root with:: + + uv run tangle sdk pipelines compile \ + examples/python_pipeline/task_env_from_config/pipeline.py \ + --pipeline scoring_pipeline \ + --output /tmp/tangle-task-env-from-config-demo/pipeline.yaml + +The config uses ``_select`` over ``DEPLOY_ENVIRONMENT``; with the variable +unset the ``default`` case applies, so the command above works as written:: + + DEPLOY_ENVIRONMENT=production uv run tangle sdk pipelines compile ... +""" + +from tangle_cli.python_pipeline import Out, TaskEnv, pipeline, task + +# One declaration, reused by every task below. Nothing about this is specific +# to a kind of environment: any ``TaskEnv`` dataclass subclass inherits +# ``from_config`` and is validated against its own fields. +SCORING = TaskEnv.from_config("envs.yaml") + + +@task(env=SCORING) +def load_queries(count: str = "3") -> str: + """Produce the queries to score.""" + return ",".join(f"query-{index}" for index in range(int(count))) + + +@task(env=SCORING) +def score_queries(queries: str) -> str: + """Score the queries on the same image, without repeating it.""" + return ";".join(f"{query}=1.0" for query in queries.split(",")) + + +@pipeline("TaskEnv from config demo") +def scoring_pipeline() -> Out[str]: + queries = load_queries(count="3") + scored = score_queries(queries=queries.output) + return scored.output diff --git a/examples/python_pipeline/task_env_from_config/pyproject.toml b/examples/python_pipeline/task_env_from_config/pyproject.toml new file mode 100644 index 0000000..d881ea9 --- /dev/null +++ b/examples/python_pipeline/task_env_from_config/pyproject.toml @@ -0,0 +1,5 @@ +# Pip dependencies for the generated components of this example. +[project] +name = "task-env-from-config-example" +version = "0.0.0" +dependencies = [] diff --git a/packages/tangle-cli/src/tangle_cli/__init__.py b/packages/tangle-cli/src/tangle_cli/__init__.py index 9241a25..e24a271 100644 --- a/packages/tangle-cli/src/tangle_cli/__init__.py +++ b/packages/tangle-cli/src/tangle_cli/__init__.py @@ -14,6 +14,6 @@ try: __version__ = metadata_version("tangle-cli") except PackageNotFoundError: - __version__ = "0.1.16" + __version__ = "0.1.17" __all__ = ["TangleDynamicDiscoveryClient", "__version__"] diff --git a/packages/tangle-cli/src/tangle_cli/args_container.py b/packages/tangle-cli/src/tangle_cli/args_container.py index ca9e368..e4a83e0 100644 --- a/packages/tangle-cli/src/tangle_cli/args_container.py +++ b/packages/tangle-cli/src/tangle_cli/args_container.py @@ -41,10 +41,17 @@ class ConfigFileError(Exception): """Raised when there is an error loading or resolving a config file.""" -def _render_case_key(key: str) -> str: - """Render an already-validated configured case key for diagnostics.""" +def _render_config_key(key: Any) -> str: + """Render a config-document key safely for a diagnostic message. - rendered = "".join(char if char.isprintable() else "?" for char in key) + Config keys come from user files, so a diagnostic must never echo them + verbatim: non-printable characters are replaced and the result is length + capped before being quoted. Accepts non-string keys (YAML permits them) so + callers can report a bad key without first proving it is a string. + """ + + text = key if isinstance(key, str) else str(key) + rendered = "".join(char if char.isprintable() else "?" for char in text) if len(rendered) > _MAX_RENDERED_CASE_KEY_LENGTH: rendered = rendered[: _MAX_RENDERED_CASE_KEY_LENGTH - 3] + "..." return repr(rendered) @@ -172,8 +179,14 @@ def _validate_selector_node( if seen is not None and depth <= seen[0]: return seen[2] + # Every key reaching a diagnostic goes through the ONE capped/scrubbed + # renderer: these keys are user input, and an unbounded repr() would + # let a hostile document paste control characters or a wall of text + # into a compile/CI log. siblings = sorted( - repr(key) for key in node if not (isinstance(key, str) and key.startswith("_")) + _render_config_key(key) + for key in node + if not (isinstance(key, str) and key.startswith("_")) ) if siblings: raise ConfigFileError( @@ -189,7 +202,9 @@ def _validate_selector_node( selector_dict = cast(dict[Any, Any], selector) unexpected = sorted( - repr(key) for key in selector_dict if key not in ("env", "cases", "default") + _render_config_key(key) + for key in selector_dict + if key not in ("env", "cases", "default") ) if unexpected: raise ConfigFileError( @@ -231,7 +246,7 @@ def _validate_selector_node( # Every branch document is shape-checked, not just the selected one, # so a malformed selector fails identically in every environment. ArgsContainer._validate_branch_document( - case_value, f"{SELECT_KEY} case {_render_case_key(case_key)}", depth, memo + case_value, f"{SELECT_KEY} case {_render_config_key(case_key)}", depth, memo ) default_branch: Any = None @@ -241,7 +256,7 @@ def _validate_selector_node( default_branch, f"{SELECT_KEY}.default", depth, memo ) - allowed = ", ".join(_render_case_key(key) for key in sorted(cases_dict)) + allowed = ", ".join(_render_config_key(key) for key in sorted(cases_dict)) summary: _SelectorSummary = (env_name, cases_dict, default_branch, allowed) if memo is not None: previous = memo.get(node_id) diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py index ed1325c..5cc890e 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task_env.py @@ -6,11 +6,10 @@ ``@task(env=...)``. It is the Python equivalent of a ``local_from_python`` YAML anchor. -``TaskEnv`` is **authoring-only**. ``@task(env=...)`` expands it at -decoration time into the existing ``CallableRef._task_image`` / -``CallableRef._task_dependencies_from`` metadata. The compiler, hydrator, -and downstream runner never see a ``TaskEnv`` object — no downstream -component learns the word ``env``. +``TaskEnv`` is **authoring-only**. ``@task(env=...)`` expands it at decoration +time into the existing ``CallableRef._task_image`` / +``CallableRef._task_dependencies_from`` metadata, so the compiler, hydrator, +and runner never see a ``TaskEnv`` object. Example:: @@ -24,41 +23,36 @@ @task(env=TRAINING) def train_model(...): - # docstring carries: Metadata / Name: Train Model ... -The component name comes from the function's docstring ``Metadata: Name:`` -block (auto-derived from the function name if absent); the pipeline block name -(task id) comes from the call-site variable name, or ``.named("Block Name")`` -for an explicit label. +An environment can also live in a config file, loaded with +:meth:`TaskEnv.from_config`, which any ``TaskEnv`` dataclass subclass +inherits unchanged:: + + ENV = TaskEnv.from_config("tangle/envs.yaml") # relative to THIS file """ from __future__ import annotations import inspect from dataclasses import dataclass from pathlib import Path +from typing import TypeVar + +_TaskEnvT = TypeVar("_TaskEnvT", bound="TaskEnv") @dataclass(frozen=True) class TaskEnv: """Reusable execution environment for ``@task`` components. - Bundles the container image and optional dependencies file so a - pipeline author can declare them once and reference the env from many - tasks. This is the Python equivalent of a ``local_from_python`` YAML - anchor. - Attributes: - image: Container image for the component. Required — naming the - image once is the main point of ``TaskEnv``. - dependencies_from: Optional path to a ``pyproject.toml`` (or any - file the hydrator understands) declaring pip - dependencies. A relative path is resolved at the ``TaskEnv`` - *definition site* (the module where ``TaskEnv(...)`` is - written), so a shared ``_envs.py`` resolves intuitively. - Authors can pass an absolute ``Path`` to avoid frame-based - ambiguity. When omitted, the existing hydrator/generator - dependency discovery still applies. + image: Container image for the component. Required. + dependencies_from: Optional path to a ``pyproject.toml`` (or any file + the hydrator understands) declaring pip dependencies. A relative + path is resolved at the ``TaskEnv`` *definition site*, so a shared + ``_envs.py`` resolves intuitively; pass an absolute ``Path`` to + avoid that. When omitted, the hydrator's existing dependency + discovery still applies. """ image: str @@ -72,9 +66,7 @@ def __post_init__(self) -> None: p = Path(self.dependencies_from) if not p.is_absolute(): - # Resolve a relative ``dependencies_from`` at the TaskEnv - # DEFINITION SITE. Walk frames: __post_init__ -> generated - # dataclass __init__ -> the caller that wrote ``TaskEnv(...)``. + # Frames: __post_init__ -> generated __init__ -> the definition site. frame = inspect.currentframe() caller = ( frame.f_back.f_back @@ -86,3 +78,126 @@ def __post_init__(self) -> None: p = caller_dir / p # Frozen dataclass: bypass __setattr__ to store the resolved Path. object.__setattr__(self, "dependencies_from", p.resolve()) + + @classmethod + def from_config(cls: type[_TaskEnvT], path: str | Path) -> _TaskEnvT: + """Build the env from a config file, returning an instance of ``cls``. + + Every ``TaskEnv`` dataclass subclass inherits this and is validated + against its own fields; nothing here is specific to one kind of + environment. + + A relative ``path`` resolves against the **calling file's** directory, + never the working directory, so a pipeline script selects the same + config wherever the CLI is run from. Call this directly on the class — + a subclass that wraps or delegates to it is not supported. + + The file is read with the loader ``--config`` uses, so its ``_select`` + directive behaves identically here, and it must resolve to exactly one + object. The object's keys must be named parameters of ``cls``'s + ``__init__``, which keeps an environment config environment-only: + pipeline concerns such as versioning or schedules have no field to land + in. A relative ``dependencies_from`` inside the object is anchored to + the config file's directory rather than to this module. + + Raises: + CompileError: for every failure — a missing file, an unresolvable + relative path, a load or ``_select`` error, a multi-object + document, an unknown or missing field, or a value the subclass + rejects. + + Note: + A config file is untrusted input, so no diagnostic raised here + echoes a config *value* — not in the message, and not through + ``__cause__``/``__context__``, which a rendered traceback would + print into the same CI log. Construct ``cls(...)`` directly to see + a constructor's own validation message. + """ + + # Lazy import: the authoring surface should not pay for the YAML/CLI + # config stack unless a pipeline actually loads an env from a file. + from tangle_cli.args_container import ( + SELECT_KEY, + ArgsContainer, + ConfigFileError, + _render_config_key, + ) + from tangle_cli.python_pipeline.errors import CompileError + + label = f"{cls.__name__}.from_config" + config_path = Path(path) + if not config_path.is_absolute(): + frame = inspect.currentframe() + caller = frame.f_back if frame is not None else None + filename = caller.f_globals.get("__file__") if caller is not None else None + if not isinstance(filename, str) or not filename: + raise CompileError( + f"{label}({str(path)!r}): a relative path resolves against the " + "calling file, which could not be determined here — pass an " + "absolute path." + ) + config_path = (Path(filename).resolve().parent / config_path).resolve() + if not config_path.exists(): + raise CompileError( + f"{label}: {config_path} does not exist. Declare the environment " + f"there (optionally a {SELECT_KEY} over an environment variable, " + f"with a default case for local runs), or construct " + f"{cls.__name__}(...) directly." + ) + + try: + documents = ArgsContainer._load_config_file(config_path) + except ConfigFileError as exc: + raise CompileError(f"{label} could not resolve {config_path}: {exc}") from exc + if len(documents) != 1 or not isinstance(documents[0], dict): + raise CompileError( + f"{label}: {config_path} must resolve to ONE environment object " + f"(one {SELECT_KEY} case), got a multi-config document." + ) + + case = dict(documents[0]) + # The signature, not dataclasses.fields(): it is what cls(**case) + # actually accepts — InitVar pseudo-fields in, ClassVar and + # field(init=False) out. *args/**kwargs are excluded so a typo stays + # fail-closed. + parameters = { + name: parameter + for name, parameter in inspect.signature(cls).parameters.items() + if parameter.kind + in (parameter.POSITIONAL_OR_KEYWORD, parameter.KEYWORD_ONLY) + } + unknown = sorted(_render_config_key(key) for key in set(case) - set(parameters)) + if unknown: + raise CompileError( + f"{label}: {config_path} case has unknown field(s) " + f"{', '.join(unknown)}. Allowed fields: {', '.join(parameters)}." + ) + missing = sorted( + name + for name, parameter in parameters.items() + if parameter.default is parameter.empty and name not in case + ) + if missing: + raise CompileError( + f"{label}: {config_path} case is missing required field(s): " + f"{', '.join(missing)}." + ) + + dependencies = case.get("dependencies_from") + if isinstance(dependencies, (str, Path)) and not Path(dependencies).is_absolute(): + case["dependencies_from"] = (config_path.parent / Path(dependencies)).resolve() + + try: + return cls(**case) + except (TypeError, ValueError): + # Re-raised below, OUTSIDE this handler: the constructor's message + # may quote a config value, and `from exc` would publish it on + # __cause__ while a raise in-handler would publish it on + # __context__ — both get printed by traceback.format_exception. + pass + raise CompileError( + f"{label}: {config_path} case is not a valid {cls.__name__} " + f"(fields present: {', '.join(sorted(case))}). Its validation " + f"message is withheld because it can contain config values; " + f"construct {cls.__name__}(...) directly to see it." + ) diff --git a/pyproject.toml b/pyproject.toml index 8a061ee..2f981e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tangle-cli" -version = "0.1.16" +version = "0.1.17" description = "CLI for Tangle, the open-source ML pipeline orchestration platform" readme = "README.md" authors = [ diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 6993b5a..e023e97 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")] assert not any(name.startswith("tangle_api/") for name in names) assert "tangle_cli/openapi/openapi.json" not in names - assert "Version: 0.1.16" in metadata + assert "Version: 0.1.17" in metadata assert "Requires-Dist: tangle-api==0.1.1" in requires_dist assert not any("extra == 'native'" in line for line in requires_dist) assert "Provides-Extra: native" in metadata diff --git a/tests/test_task_env_from_config.py b/tests/test_task_env_from_config.py new file mode 100644 index 0000000..c43e02e --- /dev/null +++ b/tests/test_task_env_from_config.py @@ -0,0 +1,833 @@ +"""``TaskEnv.from_config`` — generic, config-backed environment loading. + +The point of these tests is that the behaviour is GENERIC: it lives on the +ordinary :class:`TaskEnv` and every dataclass subclass inherits it, validated +against that subclass's own fields. Each test loads a fabricated pipeline +*module from disk* rather than calling ``from_config`` from this test file, so +"relative paths resolve against the calling module, never the working +directory" is actually exercised — the tests run with the working directory +pointed somewhere else entirely. +""" + +from __future__ import annotations + +import importlib.util +import sys +import traceback +import textwrap +import uuid +from pathlib import Path +from types import ModuleType + +import pytest + +from tangle_cli.python_pipeline import TaskEnv +from tangle_cli.python_pipeline.errors import CompileError + +#: A downstream-style subclass with its own field and validation, declared +#: inside the fabricated pipeline module so the module stays self-contained. +SUBCLASS_SOURCE = """ +from dataclasses import InitVar, dataclass, field +from typing import ClassVar + +from tangle_cli.python_pipeline import TaskEnv + + +@dataclass(frozen=True) +class GpuEnv(TaskEnv): + accelerator: str = "" + + def __post_init__(self): + super().__post_init__() + if not isinstance(self.accelerator, str) or not self.accelerator: + raise ValueError("GpuEnv.accelerator must be a non-empty string") + + +@dataclass(frozen=True) +class LeakyEnv(TaskEnv): + # Hostile-by-accident subclass: it interpolates the offending value into + # its own validation message, the way a careless author would. + secret: str = "" + + def __post_init__(self): + super().__post_init__() + if not self.secret.startswith("ok-"): + raise ValueError(f"invalid secret {self.secret}") + + +@dataclass(frozen=True) +class NestedLeakyEnv(TaskEnv): + # Leaks a SCALAR LEAF of a structured value: no scan of the top-level + # stringified dict would have matched it. + settings: dict = None + + def __post_init__(self): + super().__post_init__() + raise ValueError(f"invalid credential {next(iter(self.settings.values()))}") + + +@dataclass(frozen=True) +class TransformedLeakyEnv(TaskEnv): + # Leaks a TRANSFORMED copy of a flat value: content scanning cannot undo + # a .lower(), a slice, or a re-encode. + secret: str = "" + + def __post_init__(self): + super().__post_init__() + raise ValueError( + f"invalid credential {self.secret.lower()} / {self.secret[3:]} / " + f"{self.secret.encode().hex()}" + ) + + +@dataclass(frozen=True) +class ShortLeakyEnv(TaskEnv): + # A 1-2 character value: short enough that any length-thresholded + # redaction scheme would have exempted it. + pin: str = "" + + def __post_init__(self): + super().__post_init__() + raise ValueError(f"invalid pin {self.pin}") + + +@dataclass(frozen=True) +class InitVarEnv(TaskEnv): + # ``region`` is an InitVar: a real generated __init__ parameter that + # ``dataclasses.fields()`` does NOT report. + region: InitVar[str] = "local" + tier: ClassVar[str] = "class-level" + derived: str = field(init=False, default="") + + def __post_init__(self, region): + super().__post_init__() + object.__setattr__(self, "derived", f"{self.image}@{region}") + + +@dataclass(frozen=True) +class InheritedInitVarEnv(InitVarEnv): + # Inherits the InitVar from its parent and adds one of its own. + zone: InitVar[str] = "a" + + def __post_init__(self, region, zone): + super().__post_init__(region) + object.__setattr__(self, "derived", f"{self.derived}/{zone}") + + +""" + + +def _write_pipeline_module(directory: Path, body: str) -> Path: + """Write a throwaway pipeline module that calls ``from_config``.""" + + directory.mkdir(parents=True, exist_ok=True) + module_path = directory / "pipeline_module.py" + module_path.write_text(textwrap.dedent(body), encoding="utf-8") + return module_path + + +def _load_module(module_path: Path) -> ModuleType: + """Import a module BY PATH, the way a pipeline script is loaded.""" + + name = f"_task_env_from_config_{uuid.uuid4().hex}" + spec = importlib.util.spec_from_file_location(name, module_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(name, None) + return module + + +def _run_pipeline_module(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str) -> ModuleType: + """Load a pipeline module from a directory that is NOT the cwd.""" + + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir(exist_ok=True) + monkeypatch.chdir(elsewhere) + return _load_module(_write_pipeline_module(tmp_path / "project", body)) + + +def test_plain_task_env_loads_from_a_config_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text("image: python:3.12\n", encoding="utf-8") + + module = _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + assert type(module.ENV) is TaskEnv + assert module.ENV.image == "python:3.12" + assert module.ENV.dependencies_from is None + + +def test_relative_path_resolves_against_the_calling_module_not_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A same-named config next to the cwd must NOT win.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text("image: from-module\n", encoding="utf-8") + (tmp_path / "elsewhere").mkdir(parents=True, exist_ok=True) + (tmp_path / "elsewhere" / "envs.yaml").write_text("image: from-cwd\n", encoding="utf-8") + + module = _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + assert module.ENV.image == "from-module" + + +def test_absolute_path_is_used_as_written( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config = tmp_path / "somewhere" / "envs.yaml" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text("image: python:3.12\n", encoding="utf-8") + + module = _run_pipeline_module( + tmp_path, + monkeypatch, + f""" + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config({str(config)!r}) + """, + ) + + assert module.ENV.image == "python:3.12" + + +def test_relative_dependencies_from_anchors_to_the_config_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Not to the pipeline module, and not to this framework module.""" + + config_dir = tmp_path / "project" / "tangle" + config_dir.mkdir(parents=True, exist_ok=True) + (config_dir / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + (config_dir / "envs.yaml").write_text( + "image: python:3.12\ndependencies_from: pyproject.toml\n", encoding="utf-8" + ) + + module = _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("tangle/envs.yaml") + """, + ) + + assert module.ENV.dependencies_from == (config_dir / "pyproject.toml").resolve() + + +def test_select_picks_the_case_named_by_the_environment_variable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + textwrap.dedent( + """ + _select: + env: DEPLOY_ENVIRONMENT + cases: + production: + image: registry.example/prod + staging: + image: registry.example/staging + default: + image: registry.example/local + """ + ), + encoding="utf-8", + ) + body = """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """ + + monkeypatch.setenv("DEPLOY_ENVIRONMENT", "production") + assert _run_pipeline_module(tmp_path, monkeypatch, body).ENV.image == "registry.example/prod" + + monkeypatch.setenv("DEPLOY_ENVIRONMENT", "staging") + assert _run_pipeline_module(tmp_path, monkeypatch, body).ENV.image == "registry.example/staging" + + monkeypatch.delenv("DEPLOY_ENVIRONMENT", raising=False) + assert _run_pipeline_module(tmp_path, monkeypatch, body).ENV.image == "registry.example/local" + + +def test_select_without_a_default_fails_closed_when_the_variable_is_unset( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + textwrap.dedent( + """ + _select: + env: DEPLOY_ENVIRONMENT + cases: + production: + image: registry.example/prod + """ + ), + encoding="utf-8", + ) + monkeypatch.delenv("DEPLOY_ENVIRONMENT", raising=False) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + assert "DEPLOY_ENVIRONMENT" in str(excinfo.value) + assert "TaskEnv.from_config" in str(excinfo.value) + + +def test_missing_file_reports_the_resolved_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + message = str(excinfo.value) + assert "does not exist" in message + assert str(tmp_path / "project" / "envs.yaml") in message + + +def test_multi_config_document_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "- image: one\n- image: two\n", encoding="utf-8" + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + assert "ONE environment object" in str(excinfo.value) + + +def test_unknown_field_lists_the_concrete_classes_allowed_fields( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\nschedule: '@daily'\n", encoding="utf-8" + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + message = str(excinfo.value) + assert "'schedule'" in message + assert "Allowed fields: image, dependencies_from" in message + + +def test_config_is_environment_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Pipeline concerns have no field to land in and are rejected.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + for pipeline_only_key in ("file_path", "versioning", "annotations", "subscription"): + (tmp_path / "project" / "envs.yaml").write_text( + f"image: python:3.12\n{pipeline_only_key}: whatever\n", encoding="utf-8" + ) + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + assert f"'{pipeline_only_key}'" in str(excinfo.value) + + +def test_unknown_key_diagnostics_are_sanitized( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A hostile key cannot inject control characters or a wall of text.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.json").write_text( + '{"image": "python:3.12", "a\\u0007b": 1, "' + "z" * 300 + '": 2}', + encoding="utf-8", + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.json") + """, + ) + + message = str(excinfo.value) + assert "\a" not in message + assert "a?b" in message + assert "z" * 300 not in message + assert "..." in message + + +def test_subclass_gets_its_own_fields_and_returns_its_own_type( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\naccelerator: gpu\n", encoding="utf-8" + ) + + module = _run_pipeline_module( + tmp_path, monkeypatch, SUBCLASS_SOURCE + '\nENV = GpuEnv.from_config("envs.yaml")\n' + ) + + assert type(module.ENV) is module.GpuEnv + assert isinstance(module.ENV, TaskEnv) + assert module.ENV.accelerator == "gpu" + assert module.ENV.image == "python:3.12" + + +def test_subclass_field_validation_is_preserved( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The subclass's ``__post_init__`` still REJECTS the case; only its + message is withheld, since from_config cannot know whether it quotes a + config value.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text("image: python:3.12\n", encoding="utf-8") + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, monkeypatch, SUBCLASS_SOURCE + '\nENV = GpuEnv.from_config("envs.yaml")\n' + ) + + message = str(excinfo.value) + assert "not a valid GpuEnv" in message + assert "construct GpuEnv(...) directly" in message + assert "accelerator must be a non-empty string" not in message + + +def test_subclass_rejects_a_field_it_does_not_declare( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\naccelerator: gpu\ntarget: nope\n", encoding="utf-8" + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, monkeypatch, SUBCLASS_SOURCE + '\nENV = GpuEnv.from_config("envs.yaml")\n' + ) + + message = str(excinfo.value) + assert "'target'" in message + assert "Allowed fields: image, dependencies_from, accelerator" in message + + +def test_relative_path_without_a_calling_file_is_a_clear_error() -> None: + """No ``__file__`` (``exec``/REPL) must not silently mean the cwd.""" + + namespace: dict[str, object] = {} + with pytest.raises(CompileError) as excinfo: + exec( # noqa: S102 - deliberately a frame with no __file__ + "from tangle_cli.python_pipeline import TaskEnv\n" + "ENV = TaskEnv.from_config('envs.yaml')\n", + namespace, + ) + + assert "pass an absolute path" in str(excinfo.value) + + +def test_a_config_loaded_env_drives_an_ordinary_generated_task( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Nothing dbt-specific: a plain ``@task`` consumes the loaded env.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "pyproject.toml").write_text("[project]\n", encoding="utf-8") + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\ndependencies_from: pyproject.toml\n", encoding="utf-8" + ) + + module = _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv, task + + ENV = TaskEnv.from_config("envs.yaml") + + @task(env=ENV) + def score(value: str) -> str: + return value + """, + ) + + assert module.score._task_image == "python:3.12" + assert module.score._task_dependencies_from == ( + tmp_path / "project" / "pyproject.toml" + ).resolve() + + +# --------------------------------------------------------------------------- +# Adversarial: a config file is untrusted input. No diagnostic may echo a +# config VALUE, and no hostile key or subclass message may inject control +# characters or a wall of text into a compile/CI log. + + +def _expect_compile_error(tmp_path, monkeypatch, config_text, call, *, name="envs.yaml"): + """Return everything a caller could SEE: the message plus the rendered + traceback chain. ``str(exc)`` alone is not the disclosure surface — a test + that only checks it misses ``__cause__``/``__context__``, which + ``traceback.format_exception`` prints verbatim and which reaches any CI log + that lets the error escape or logs it with ``exc_info=True``. + """ + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / name).write_text(textwrap.dedent(config_text), encoding="utf-8") + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module(tmp_path, monkeypatch, SUBCLASS_SOURCE + f"\nENV = {call}\n") + rendered = "".join(traceback.format_exception(excinfo.value)) + return str(excinfo.value) + "\n" + rendered + + +def test_arbitrary_subclass_exception_text_is_never_quoted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A plain ValueError is reported generically — its text is withheld.""" + + message = _expect_compile_error( + tmp_path, + monkeypatch, + "image: python:3.12\nsecret: TOP-SECRET-VALUE\n", + 'LeakyEnv.from_config("envs.yaml")', + ) + + assert "TOP-SECRET-VALUE" not in message + assert "invalid secret" not in message + # Still actionable without the text: class, field NAMES, and the remedy. + assert "not a valid LeakyEnv" in message + assert "fields present: image, secret" in message + # Actionable without the text: the author is told how to see it themselves. + assert "construct LeakyEnv(...) directly" in message + + +def test_nested_scalar_leaf_cannot_leak( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Reviewer repro A: the leaked value is a leaf of a structured field, so + no scan of the top-level stringified value could ever have matched it.""" + + message = _expect_compile_error( + tmp_path, + monkeypatch, + """ + image: python:3.12 + settings: + credential: TOP-SECRET-VALUE + """, + 'NestedLeakyEnv.from_config("envs.yaml")', + ) + + assert "TOP-SECRET-VALUE" not in message + assert "invalid credential" not in message + assert "fields present: image, settings" in message + + +def test_transformed_value_cannot_leak( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Reviewer repro B: lower cased, sliced, and hex re-encoded copies of the + value. Content scanning cannot undo any of these; withholding can.""" + + message = _expect_compile_error( + tmp_path, + monkeypatch, + "image: python:3.12\nsecret: TOP-SECRET-VALUE\n", + 'TransformedLeakyEnv.from_config("envs.yaml")', + ) + + for disclosure in ( + "TOP-SECRET-VALUE", + "top-secret-value", + "SECRET-VALUE", + "TOP-SECRET-VALUE".encode().hex(), + ): + assert disclosure not in message + + +def test_short_value_cannot_leak( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """No length threshold exists, because nothing is scanned or quoted.""" + + message = _expect_compile_error( + tmp_path, + monkeypatch, + "image: python:3.12\npin: '7'\n", + 'ShortLeakyEnv.from_config("envs.yaml")', + ) + + assert "invalid pin" not in message + assert "fields present: image, pin" in message + + +def test_missing_required_field_is_reported_from_the_signature( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The common structural mistake keeps a precise message, derived from the + signature rather than from withheld TypeError text.""" + + message = _expect_compile_error( + tmp_path, + monkeypatch, + "dependencies_from: pyproject.toml\n", + 'TaskEnv.from_config("envs.yaml")', + ) + + assert "missing required field(s): image" in message + + +def test_hostile_select_sibling_key_is_capped_through_from_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A ``_select`` error forwarded by from_config must be sanitized too.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.json").write_text( + '{"_select": {"env": "E", "cases": {"a": {"image": "i"}}}, ' + '"' + "s" * 300 + '": 1}', + encoding="utf-8", + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.json") + """, + ) + + message = str(excinfo.value) + assert "only regular key" in message + assert "s" * 300 not in message + assert "..." in message + + +def test_hostile_select_unexpected_key_is_scrubbed_through_from_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.json").write_text( + '{"_select": {"env": "E", "cases": {"a": {"image": "i"}}, ' + '"we\\u0007ird": 1, "' + "u" * 300 + '": 2}}', + encoding="utf-8", + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.json") + """, + ) + + message = str(excinfo.value) + assert "supports only 'env', 'cases', and 'default'" in message + assert "\a" not in message + assert "we?ird" in message + assert "u" * 300 not in message + + +def test_non_string_select_key_is_rendered_safely( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """YAML permits non-string keys; the renderer must not choke on them.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + textwrap.dedent( + """ + _select: + env: E + cases: + a: + image: i + 7: sibling + """ + ), + encoding="utf-8", + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + """ + from tangle_cli.python_pipeline import TaskEnv + + ENV = TaskEnv.from_config("envs.yaml") + """, + ) + + assert "only regular key" in str(excinfo.value) + assert "'7'" in str(excinfo.value) + + +def test_init_var_is_an_accepted_constructor_input( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``dataclasses.fields()`` omits InitVars; the signature does not.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\nregion: prod\n", encoding="utf-8" + ) + + module = _run_pipeline_module( + tmp_path, monkeypatch, SUBCLASS_SOURCE + '\nENV = InitVarEnv.from_config("envs.yaml")\n' + ) + + assert module.ENV.derived == "python:3.12@prod" + + +def test_inherited_init_vars_are_accepted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\nregion: prod\nzone: b\n", encoding="utf-8" + ) + + module = _run_pipeline_module( + tmp_path, + monkeypatch, + SUBCLASS_SOURCE + '\nENV = InheritedInitVarEnv.from_config("envs.yaml")\n', + ) + + assert module.ENV.derived == "python:3.12@prod/b" + + +def test_class_var_and_non_init_field_are_not_accepted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Widening to InitVars must not also open ClassVars/``init=False``.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + for rejected_key in ("tier", "derived"): + (tmp_path / "project" / "envs.yaml").write_text( + f"image: python:3.12\n{rejected_key}: nope\n", encoding="utf-8" + ) + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + SUBCLASS_SOURCE + '\nENV = InitVarEnv.from_config("envs.yaml")\n', + ) + message = str(excinfo.value) + assert f"'{rejected_key}'" in message + assert "Allowed fields: image, dependencies_from, region" in message + + +def test_constructor_exception_is_absent_from_the_traceback_chain( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """``raise ... from exc`` would publish the withheld text on __cause__, and + a bare re-raise inside the handler would publish it on __context__; either + way ``traceback.format_exception`` prints it. Neither may be set.""" + + (tmp_path / "project").mkdir(parents=True, exist_ok=True) + (tmp_path / "project" / "envs.yaml").write_text( + "image: python:3.12\nsecret: TOP-SECRET-VALUE\n", encoding="utf-8" + ) + + with pytest.raises(CompileError) as excinfo: + _run_pipeline_module( + tmp_path, + monkeypatch, + SUBCLASS_SOURCE + '\nENV = TransformedLeakyEnv.from_config("envs.yaml")\n', + ) + + error = excinfo.value + assert error.__cause__ is None + assert error.__context__ is None + rendered = "".join(traceback.format_exception(error)) + for disclosure in ( + "TOP-SECRET-VALUE", + "top-secret-value", + "SECRET-VALUE", + "TOP-SECRET-VALUE".encode().hex(), + "invalid credential", + ): + assert disclosure not in rendered + + +def test_direct_construction_still_raises_an_ordinary_value_error() -> None: + """The withholding policy applies to from_config diagnostics only. Calling + a constructor directly is unchanged, which is what the from_config message + points the author at.""" + + with pytest.raises(ValueError, match="TaskEnv.image must be a non-empty string"): + TaskEnv(image="") diff --git a/uv.lock b/uv.lock index 65cc609..0517168 100644 --- a/uv.lock +++ b/uv.lock @@ -2083,7 +2083,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.0" }] [[package]] name = "tangle-cli" -version = "0.1.16" +version = "0.1.17" source = { editable = "." } dependencies = [ { name = "cloud-pipelines" },