diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c1ecb0..530aad6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `tinybird pull --as-code` generates Python SDK source (`datasources.py`, `pipes.py`, `client.py`) from the live workspace via the existing `codegen` module, instead of raw `.datasource`/`.pipe`/`.connection` files. This wires up `codegen`, which was previously unreachable from any CLI command. Plain `tinybird pull` (without `--as-code`) is unaffected and still delegates to the installed Tinybird CLI. + ## [0.4.0] - 2026-06-29 ### Added diff --git a/README.md b/README.md index e42f34b..28a4b3e 100644 --- a/README.md +++ b/README.md @@ -408,6 +408,13 @@ tinybird pull --output-dir ./tinybird-datafiles tinybird pull --force ``` +Pass `--as-code` to generate Python SDK source (`datasources.py`, `pipes.py`, `client.py`) from the live workspace instead of raw `.datasource`/`.pipe`/`.connection` files: + +```bash +tinybird pull --as-code +tinybird pull --as-code --output-dir ./lib +``` + ### `tinybird login` ```bash diff --git a/src/tinybird_sdk/api/resources.py b/src/tinybird_sdk/api/resources.py index 45a929d..15fb8c9 100644 --- a/src/tinybird_sdk/api/resources.py +++ b/src/tinybird_sdk/api/resources.py @@ -64,7 +64,7 @@ class PipeParam: PipeType = Literal["endpoint", "materialized", "copy", "pipe"] -ResourceFileType = Literal["datasource", "pipe", "connection"] +ResourceFileType = Literal["datasource", "pipe", "connection", "code"] @dataclass(frozen=True, slots=True) diff --git a/src/tinybird_sdk/cli/commands/pull.py b/src/tinybird_sdk/cli/commands/pull.py index bbe8658..2a940d4 100644 --- a/src/tinybird_sdk/cli/commands/pull.py +++ b/src/tinybird_sdk/cli/commands/pull.py @@ -6,7 +6,8 @@ import time from typing import Any, Literal -from ...api.resources import ResourceFile, pull_all_resource_files +from ...api.resources import ResourceFile, fetch_all_resources, pull_all_resource_files +from ...codegen.index import generate_all_files from ..config import load_config_async @@ -15,12 +16,13 @@ class PullCommandOptions: cwd: str | None = None output_dir: str = "." overwrite: bool = False + as_code: bool = False @dataclass(frozen=True, slots=True) class PulledFileResult: name: str - type: Literal["datasource", "pipe", "connection"] + type: Literal["datasource", "pipe", "connection", "code"] filename: str path: str relative_path: str @@ -45,6 +47,43 @@ def _flatten_resources(resources: dict[str, list[ResourceFile]]) -> list[Resourc ] +def _generate_code_files( + api_config: dict[str, Any], +) -> tuple[list[ResourceFile], dict[str, int]]: + resources = fetch_all_resources(api_config) + datasources = resources["datasources"] + pipes = resources["pipes"] + generated = generate_all_files(datasources, pipes) + + files = [ + ResourceFile( + name="datasources", + type="code", + filename="datasources.py", + content=generated.datasources_content, + ), + ResourceFile( + name="pipes", + type="code", + filename="pipes.py", + content=generated.pipes_content, + ), + ResourceFile( + name="client", + type="code", + filename="client.py", + content=generated.client_content, + ), + ] + stats = { + "datasources": generated.datasource_count, + "pipes": generated.pipe_count, + "connections": 0, + "total": len(files), + } + return files, stats + + def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> PullCommandResult: start = int(time.time() * 1000) normalized = ( @@ -64,8 +103,20 @@ def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> Pull success=False, error=str(error), duration_ms=int(time.time() * 1000) - start ) + api_config = {"base_url": config["base_url"], "token": config["token"]} + try: - pulled = pull_all_resource_files({"base_url": config["base_url"], "token": config["token"]}) + if normalized.as_code: + all_files, stats = _generate_code_files(api_config) + else: + pulled = pull_all_resource_files(api_config) + all_files = sorted(_flatten_resources(pulled), key=lambda item: item.filename) + stats = { + "datasources": len(pulled.get("datasources") or []), + "pipes": len(pulled.get("pipes") or []), + "connections": len(pulled.get("connections") or []), + "total": len(all_files), + } except Exception as error: return PullCommandResult( success=False, @@ -73,8 +124,6 @@ def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> Pull duration_ms=int(time.time() * 1000) - start, ) - all_files = sorted(_flatten_resources(pulled), key=lambda item: item.filename) - try: output_dir.mkdir(parents=True, exist_ok=True) written: list[PulledFileResult] = [] @@ -102,12 +151,7 @@ def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> Pull success=True, output_dir=str(output_dir), files=written, - stats={ - "datasources": len(pulled.get("datasources") or []), - "pipes": len(pulled.get("pipes") or []), - "connections": len(pulled.get("connections") or []), - "total": len(written), - }, + stats=stats, duration_ms=int(time.time() * 1000) - start, ) except FileExistsError as error: diff --git a/src/tinybird_sdk/cli/index.py b/src/tinybird_sdk/cli/index.py index 1183961..e9295ce 100644 --- a/src/tinybird_sdk/cli/index.py +++ b/src/tinybird_sdk/cli/index.py @@ -8,8 +8,15 @@ from .commands.generate import run_generate from .commands.init import run_init from .commands.migrate import run_migrate +from .commands.pull import run_pull from .output import output +_SDK_OWNED_COMMANDS = {"init", "generate", "migrate"} + + +def _pull_wants_as_code(argv: list[str]) -> bool: + return "--as-code" in argv[1:] + def _print_json(payload: object) -> None: print(json.dumps(payload, indent=2, default=str)) @@ -81,6 +88,24 @@ def create_cli() -> argparse.ArgumentParser: ) migrate_cmd.add_argument("--json", action="store_true", help="Print migration result as JSON") + # Plain `pull` is delegated to the installed Tinybird CLI (see `owns_command` in main()). + # This subparser only ever runs for `pull --as-code`. + pull_cmd = sub.add_parser( + "pull", help="Generate Python SDK source from the live workspace (--as-code only)" + ) + pull_cmd.add_argument( + "-o", "--output-dir", default=".", help="Target folder for generated files" + ) + pull_cmd.add_argument("--force", action="store_true", help="Overwrite existing files") + pull_cmd.add_argument( + "--as-code", + action="store_true", + help=( + "Generate Python SDK source (datasources.py, pipes.py, client.py) from the live " + "workspace instead of raw .datasource/.pipe/.connection files" + ), + ) + return parser @@ -88,7 +113,13 @@ def main(argv: list[str] | None = None) -> int: normalized_argv = list(argv) if argv is not None else list(sys.argv[1:]) # SDK-owned commands stay local; all other commands are delegated to Tinybird CLI. - if not normalized_argv or normalized_argv[0] not in {"init", "generate", "migrate"}: + # `pull` is delegated too, except for `--as-code`, which the installed CLI has no + # equivalent for (it generates Python SDK source, not raw datafiles). + owns_command = bool(normalized_argv) and ( + normalized_argv[0] in _SDK_OWNED_COMMANDS + or (normalized_argv[0] == "pull" and _pull_wants_as_code(normalized_argv)) + ) + if not owns_command: return _run_installed_tinybird_cli(normalized_argv) parser = create_cli() @@ -142,6 +173,22 @@ def main(argv: list[str] | None = None) -> int: print(f"Completed in {output.format_duration(generate_result.duration_ms)}") return 0 + if args.command == "pull": + pull_result = run_pull( + {"output_dir": args.output_dir, "overwrite": args.force, "as_code": args.as_code} + ) + if not pull_result.success: + output.error(pull_result.error or "Pull failed") + return 1 + + file_count = len(pull_result.files or []) + noun = "source files" if args.as_code else "datafiles" + print(f"Pulled {file_count} {noun}") + if pull_result.output_dir: + print(f"Written to: {pull_result.output_dir}") + print(f"Completed in {output.format_duration(pull_result.duration_ms)}") + return 0 + migrate_result = run_migrate( { "cwd": args.cwd, diff --git a/tests/test_cli_entrypoint.py b/tests/test_cli_entrypoint.py index fe2c493..66f4c23 100644 --- a/tests/test_cli_entrypoint.py +++ b/tests/test_cli_entrypoint.py @@ -210,3 +210,95 @@ def test_cli_entrypoint_migrate_failure_returns_error(monkeypatch: pytest.Monkey lambda *_args, **_kwargs: {"success": False, "errors": ["boom"]}, ) assert cli_index.main(["migrate", "legacy.datasource"]) == 1 + + +def _deny_delegation(monkeypatch: pytest.MonkeyPatch, command: str) -> None: + monkeypatch.setattr( + cli_index, + "_run_installed_tinybird_cli", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError(f"should not delegate {command}") + ), + ) + + +def test_cli_entrypoint_delegates_plain_pull(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[list[str]] = [] + monkeypatch.setattr( + cli_index, "_run_installed_tinybird_cli", lambda argv: calls.append(list(argv)) or 0 + ) + monkeypatch.setattr( + cli_index, + "run_pull", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("pull should not run")), + ) + + assert cli_index.main(["pull"]) == 0 + assert cli_index.main(["pull", "--force"]) == 0 + assert calls == [["pull"], ["pull", "--force"]] + + +def test_cli_entrypoint_runs_pull_as_code_locally( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _deny_delegation(monkeypatch, "pull --as-code") + calls: list[dict] = [] + + def fake_run_pull(options: dict) -> SimpleNamespace: + calls.append(options) + return SimpleNamespace( + success=True, + error=None, + duration_ms=7, + output_dir="./lib", + files=[SimpleNamespace(), SimpleNamespace()], + stats=None, + ) + + monkeypatch.setattr(cli_index, "run_pull", fake_run_pull) + + assert cli_index.main(["pull", "--as-code", "--output-dir", "./lib", "--force"]) == 0 + assert calls == [{"output_dir": "./lib", "overwrite": True, "as_code": True}] + out = capsys.readouterr().out + assert "Pulled 2 source files" in out + assert "Written to: ./lib" in out + + +def test_cli_entrypoint_pull_as_code_defaults( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _deny_delegation(monkeypatch, "pull --as-code") + calls: list[dict] = [] + + def fake_run_pull(options: dict) -> SimpleNamespace: + calls.append(options) + return SimpleNamespace( + success=True, + error=None, + duration_ms=3, + output_dir=".", + files=[SimpleNamespace(), SimpleNamespace(), SimpleNamespace()], + stats=None, + ) + + monkeypatch.setattr(cli_index, "run_pull", fake_run_pull) + + assert cli_index.main(["pull", "--as-code"]) == 0 + assert calls == [{"output_dir": ".", "overwrite": False, "as_code": True}] + out = capsys.readouterr().out + assert "Pulled 3 source files" in out + + +def test_cli_entrypoint_pull_as_code_failure_returns_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _mute_output(monkeypatch) + _deny_delegation(monkeypatch, "pull --as-code") + monkeypatch.setattr( + cli_index, + "run_pull", + lambda *_args, **_kwargs: SimpleNamespace( + success=False, error="Codegen failed", duration_ms=1, output_dir=None, files=None + ), + ) + assert cli_index.main(["pull", "--as-code"]) == 1 diff --git a/tests/test_cli_workflows.py b/tests/test_cli_workflows.py index 84217de..44cfcb2 100644 --- a/tests/test_cli_workflows.py +++ b/tests/test_cli_workflows.py @@ -7,6 +7,7 @@ import tinybird_sdk.cli.commands.deploy as deploy_cmd import tinybird_sdk.cli.commands.dev as dev_cmd import tinybird_sdk.cli.commands.pull as pull_cmd +from tinybird_sdk.api.resources import DatasourceColumn, DatasourceEngine, DatasourceInfo from tinybird_sdk.cli.commands.build import run_build from tinybird_sdk.cli.commands.deploy import run_deploy from tinybird_sdk.cli.commands.init import run_init @@ -114,3 +115,48 @@ def test_pull_migrate_and_dev_once_workflow( ) dev_result = dev_cmd.run_dev({"cwd": str(tmp_path), "once": True}) assert dev_result["success"] is True + + +def test_pull_as_code_generates_python_source( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("TINYBIRD_TOKEN", "p.workspace") + monkeypatch.setenv("TINYBIRD_URL", "https://api.tinybird.co") + (tmp_path / "tinybird.config.json").write_text( + '{"include":["lib/*.py"],"token":"${TINYBIRD_TOKEN}","base_url":"${TINYBIRD_URL}"}\n', + encoding="utf-8", + ) + + monkeypatch.setattr( + pull_cmd, + "fetch_all_resources", + lambda *_args, **_kwargs: { + "datasources": [ + DatasourceInfo( + name="events", + columns=[DatasourceColumn(name="id", type="Int32")], + engine=DatasourceEngine(type="MergeTree", sorting_key="id"), + ) + ], + "pipes": [], + }, + ) + + pull_result = run_pull( + {"cwd": str(tmp_path), "output_dir": "out", "overwrite": True, "as_code": True} + ) + assert pull_result.success is True + assert pull_result.stats == { + "datasources": 1, + "pipes": 0, + "connections": 0, + "total": 3, + } + + written_names = sorted(file.filename for file in pull_result.files or []) + assert written_names == ["client.py", "datasources.py", "pipes.py"] + + datasources_content = (tmp_path / "out" / "datasources.py").read_text(encoding="utf-8") + assert "define_datasource('events'" in datasources_content + pipes_content = (tmp_path / "out" / "pipes.py").read_text(encoding="utf-8") + assert "No pipes found in workspace" in pipes_content