Wire reverse codegen into tinybird pull --as-code

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- `tinybird pull --as-code` generates Python SDK source (`datasources.py`, `pipes.py`, `client.py`) from the live workspace via the existing `codegen` module, instead of raw `.datasource`/`.pipe`/`.connection` files. This wires up `codegen`, which was previously unreachable from any CLI command. Plain `tinybird pull` (without `--as-code`) is unaffected and still delegates to the installed Tinybird CLI.
+
 ## [0.4.0] - 2026-06-29
 
 ### Added

README.md

@@ -408,6 +408,13 @@ tinybird pull --output-dir ./tinybird-datafiles
 tinybird pull --force
 ```
 
+Pass `--as-code` to generate Python SDK source (`datasources.py`, `pipes.py`, `client.py`) from the live workspace instead of raw `.datasource`/`.pipe`/`.connection` files:
+
+```bash
+tinybird pull --as-code
+tinybird pull --as-code --output-dir ./lib
+```
+
 ### `tinybird login`
 
 ```bash

src/tinybird_sdk/api/resources.py

@@ -64,7 +64,7 @@ class PipeParam:
 
 
 PipeType = Literal["endpoint", "materialized", "copy", "pipe"]
-ResourceFileType = Literal["datasource", "pipe", "connection"]
+ResourceFileType = Literal["datasource", "pipe", "connection", "code"]
 
 
 @dataclass(frozen=True, slots=True)

src/tinybird_sdk/cli/commands/pull.py

@@ -6,7 +6,8 @@
 import time
 from typing import Any, Literal
 
-from ...api.resources import ResourceFile, pull_all_resource_files
+from ...api.resources import ResourceFile, fetch_all_resources, pull_all_resource_files
+from ...codegen.index import generate_all_files
 from ..config import load_config_async
 
 
@@ -15,12 +16,13 @@ class PullCommandOptions:
     cwd: str | None = None
     output_dir: str = "."
     overwrite: bool = False
+    as_code: bool = False
 
 
 @dataclass(frozen=True, slots=True)
 class PulledFileResult:
     name: str
-    type: Literal["datasource", "pipe", "connection"]
+    type: Literal["datasource", "pipe", "connection", "code"]
     filename: str
     path: str
     relative_path: str
@@ -45,6 +47,43 @@ def _flatten_resources(resources: dict[str, list[ResourceFile]]) -> list[Resourc
     ]
 
 
+def _generate_code_files(
+    api_config: dict[str, Any],
+) -> tuple[list[ResourceFile], dict[str, int]]:
+    resources = fetch_all_resources(api_config)
+    datasources = resources["datasources"]
+    pipes = resources["pipes"]
+    generated = generate_all_files(datasources, pipes)
+
+    files = [
+        ResourceFile(
+            name="datasources",
+            type="code",
+            filename="datasources.py",
+            content=generated.datasources_content,
+        ),
+        ResourceFile(
+            name="pipes",
+            type="code",
+            filename="pipes.py",
+            content=generated.pipes_content,
+        ),
+        ResourceFile(
+            name="client",
+            type="code",
+            filename="client.py",
+            content=generated.client_content,
+        ),
+    ]
+    stats = {
+        "datasources": generated.datasource_count,
+        "pipes": generated.pipe_count,
+        "connections": 0,
+        "total": len(files),
+    }
+    return files, stats
+
+
 def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> PullCommandResult:
     start = int(time.time() * 1000)
     normalized = (
@@ -64,17 +103,27 @@ def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> Pull
             success=False, error=str(error), duration_ms=int(time.time() * 1000) - start
         )
 
+    api_config = {"base_url": config["base_url"], "token": config["token"]}
+
     try:
-        pulled = pull_all_resource_files({"base_url": config["base_url"], "token": config["token"]})
+        if normalized.as_code:
+            all_files, stats = _generate_code_files(api_config)
+        else:
+            pulled = pull_all_resource_files(api_config)
+            all_files = sorted(_flatten_resources(pulled), key=lambda item: item.filename)
+            stats = {
+                "datasources": len(pulled.get("datasources") or []),
+                "pipes": len(pulled.get("pipes") or []),
+                "connections": len(pulled.get("connections") or []),
+                "total": len(all_files),
+            }
     except Exception as error:
         return PullCommandResult(
             success=False,
             error=f"Pull failed: {error}",
             duration_ms=int(time.time() * 1000) - start,
         )
 
-    all_files = sorted(_flatten_resources(pulled), key=lambda item: item.filename)
-
     try:
         output_dir.mkdir(parents=True, exist_ok=True)
         written: list[PulledFileResult] = []
@@ -102,12 +151,7 @@ def run_pull(options: PullCommandOptions | dict[str, Any] | None = None) -> Pull
             success=True,
             output_dir=str(output_dir),
             files=written,
-            stats={
-                "datasources": len(pulled.get("datasources") or []),
-                "pipes": len(pulled.get("pipes") or []),
-                "connections": len(pulled.get("connections") or []),
-                "total": len(written),
-            },
+            stats=stats,
             duration_ms=int(time.time() * 1000) - start,
         )
     except FileExistsError as error:

src/tinybird_sdk/cli/index.py

@@ -8,8 +8,15 @@
 from .commands.generate import run_generate
 from .commands.init import run_init
 from .commands.migrate import run_migrate
+from .commands.pull import run_pull
 from .output import output
 
+_SDK_OWNED_COMMANDS = {"init", "generate", "migrate"}
+
+
+def _pull_wants_as_code(argv: list[str]) -> bool:
+    return "--as-code" in argv[1:]
+
 
 def _print_json(payload: object) -> None:
     print(json.dumps(payload, indent=2, default=str))
@@ -81,14 +88,38 @@ def create_cli() -> argparse.ArgumentParser:
     )
     migrate_cmd.add_argument("--json", action="store_true", help="Print migration result as JSON")
 
+    # Plain `pull` is delegated to the installed Tinybird CLI (see `owns_command` in main()).
+    # This subparser only ever runs for `pull --as-code`.
+    pull_cmd = sub.add_parser(
+        "pull", help="Generate Python SDK source from the live workspace (--as-code only)"
+    )
+    pull_cmd.add_argument(
+        "-o", "--output-dir", default=".", help="Target folder for generated files"
+    )
+    pull_cmd.add_argument("--force", action="store_true", help="Overwrite existing files")
+    pull_cmd.add_argument(
+        "--as-code",
+        action="store_true",
+        help=(
+            "Generate Python SDK source (datasources.py, pipes.py, client.py) from the live "
+            "workspace instead of raw .datasource/.pipe/.connection files"
+        ),
+    )
+
     return parser
 
 
 def main(argv: list[str] | None = None) -> int:
     normalized_argv = list(argv) if argv is not None else list(sys.argv[1:])
 
     # SDK-owned commands stay local; all other commands are delegated to Tinybird CLI.
-    if not normalized_argv or normalized_argv[0] not in {"init", "generate", "migrate"}:
+    # `pull` is delegated too, except for `--as-code`, which the installed CLI has no
+    # equivalent for (it generates Python SDK source, not raw datafiles).
+    owns_command = bool(normalized_argv) and (
+        normalized_argv[0] in _SDK_OWNED_COMMANDS
+        or (normalized_argv[0] == "pull" and _pull_wants_as_code(normalized_argv))
+    )
+    if not owns_command:
         return _run_installed_tinybird_cli(normalized_argv)
 
     parser = create_cli()
@@ -142,6 +173,22 @@ def main(argv: list[str] | None = None) -> int:
         print(f"Completed in {output.format_duration(generate_result.duration_ms)}")
         return 0
 
+    if args.command == "pull":
+        pull_result = run_pull(
+            {"output_dir": args.output_dir, "overwrite": args.force, "as_code": args.as_code}
+        )
+        if not pull_result.success:
+            output.error(pull_result.error or "Pull failed")
+            return 1
+
+        file_count = len(pull_result.files or [])
+        noun = "source files" if args.as_code else "datafiles"
+        print(f"Pulled {file_count} {noun}")
+        if pull_result.output_dir:
+            print(f"Written to: {pull_result.output_dir}")
+        print(f"Completed in {output.format_duration(pull_result.duration_ms)}")
+        return 0
+
     migrate_result = run_migrate(
         {
             "cwd": args.cwd,

tests/test_cli_entrypoint.py

@@ -210,3 +210,95 @@ def test_cli_entrypoint_migrate_failure_returns_error(monkeypatch: pytest.Monkey
         lambda *_args, **_kwargs: {"success": False, "errors": ["boom"]},
     )
     assert cli_index.main(["migrate", "legacy.datasource"]) == 1
+
+
+def _deny_delegation(monkeypatch: pytest.MonkeyPatch, command: str) -> None:
+    monkeypatch.setattr(
+        cli_index,
+        "_run_installed_tinybird_cli",
+        lambda *_args, **_kwargs: (_ for _ in ()).throw(
+            AssertionError(f"should not delegate {command}")
+        ),
+    )
+
+
+def test_cli_entrypoint_delegates_plain_pull(monkeypatch: pytest.MonkeyPatch) -> None:
+    calls: list[list[str]] = []
+    monkeypatch.setattr(
+        cli_index, "_run_installed_tinybird_cli", lambda argv: calls.append(list(argv)) or 0
+    )
+    monkeypatch.setattr(
+        cli_index,
+        "run_pull",
+        lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("pull should not run")),
+    )
+
+    assert cli_index.main(["pull"]) == 0
+    assert cli_index.main(["pull", "--force"]) == 0
+    assert calls == [["pull"], ["pull", "--force"]]
+
+
+def test_cli_entrypoint_runs_pull_as_code_locally(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    _deny_delegation(monkeypatch, "pull --as-code")
+    calls: list[dict] = []
+
+    def fake_run_pull(options: dict) -> SimpleNamespace:
+        calls.append(options)
+        return SimpleNamespace(
+            success=True,
+            error=None,
+            duration_ms=7,
+            output_dir="./lib",
+            files=[SimpleNamespace(), SimpleNamespace()],
+            stats=None,
+        )
+
+    monkeypatch.setattr(cli_index, "run_pull", fake_run_pull)
+
+    assert cli_index.main(["pull", "--as-code", "--output-dir", "./lib", "--force"]) == 0
+    assert calls == [{"output_dir": "./lib", "overwrite": True, "as_code": True}]
+    out = capsys.readouterr().out
+    assert "Pulled 2 source files" in out
+    assert "Written to: ./lib" in out
+
+
+def test_cli_entrypoint_pull_as_code_defaults(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    _deny_delegation(monkeypatch, "pull --as-code")
+    calls: list[dict] = []
+
+    def fake_run_pull(options: dict) -> SimpleNamespace:
+        calls.append(options)
+        return SimpleNamespace(
+            success=True,
+            error=None,
+            duration_ms=3,
+            output_dir=".",
+            files=[SimpleNamespace(), SimpleNamespace(), SimpleNamespace()],
+            stats=None,
+        )
+
+    monkeypatch.setattr(cli_index, "run_pull", fake_run_pull)
+
+    assert cli_index.main(["pull", "--as-code"]) == 0
+    assert calls == [{"output_dir": ".", "overwrite": False, "as_code": True}]
+    out = capsys.readouterr().out
+    assert "Pulled 3 source files" in out
+
+
+def test_cli_entrypoint_pull_as_code_failure_returns_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _mute_output(monkeypatch)
+    _deny_delegation(monkeypatch, "pull --as-code")
+    monkeypatch.setattr(
+        cli_index,
+        "run_pull",
+        lambda *_args, **_kwargs: SimpleNamespace(
+            success=False, error="Codegen failed", duration_ms=1, output_dir=None, files=None
+        ),
+    )
+    assert cli_index.main(["pull", "--as-code"]) == 1

tests/test_cli_workflows.py

@@ -7,6 +7,7 @@
 import tinybird_sdk.cli.commands.deploy as deploy_cmd
 import tinybird_sdk.cli.commands.dev as dev_cmd
 import tinybird_sdk.cli.commands.pull as pull_cmd
+from tinybird_sdk.api.resources import DatasourceColumn, DatasourceEngine, DatasourceInfo
 from tinybird_sdk.cli.commands.build import run_build
 from tinybird_sdk.cli.commands.deploy import run_deploy
 from tinybird_sdk.cli.commands.init import run_init
@@ -114,3 +115,48 @@ def test_pull_migrate_and_dev_once_workflow(
     )
     dev_result = dev_cmd.run_dev({"cwd": str(tmp_path), "once": True})
     assert dev_result["success"] is True
+
+
+def test_pull_as_code_generates_python_source(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv("TINYBIRD_TOKEN", "p.workspace")
+    monkeypatch.setenv("TINYBIRD_URL", "https://api.tinybird.co")
+    (tmp_path / "tinybird.config.json").write_text(
+        '{"include":["lib/*.py"],"token":"${TINYBIRD_TOKEN}","base_url":"${TINYBIRD_URL}"}\n',
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(
+        pull_cmd,
+        "fetch_all_resources",
+        lambda *_args, **_kwargs: {
+            "datasources": [
+                DatasourceInfo(
+                    name="events",
+                    columns=[DatasourceColumn(name="id", type="Int32")],
+                    engine=DatasourceEngine(type="MergeTree", sorting_key="id"),
+                )
+            ],
+            "pipes": [],
+        },
+    )
+
+    pull_result = run_pull(
+        {"cwd": str(tmp_path), "output_dir": "out", "overwrite": True, "as_code": True}
+    )
+    assert pull_result.success is True
+    assert pull_result.stats == {
+        "datasources": 1,
+        "pipes": 0,
+        "connections": 0,
+        "total": 3,
+    }
+
+    written_names = sorted(file.filename for file in pull_result.files or [])
+    assert written_names == ["client.py", "datasources.py", "pipes.py"]
+
+    datasources_content = (tmp_path / "out" / "datasources.py").read_text(encoding="utf-8")
+    assert "define_datasource('events'" in datasources_content
+    pipes_content = (tmp_path / "out" / "pipes.py").read_text(encoding="utf-8")
+    assert "No pipes found in workspace" in pipes_content