From 1bbc1d93da0c9c04a303180654025b38d4d03c1a Mon Sep 17 00:00:00 2001 From: Tommy Healy Date: Thu, 1 Oct 2026 17:44:06 +0200 Subject: [PATCH] Add secrets management API (list/set/remove) to TinybirdClient Wraps /v0/variables so the SDK can manage the values behind the secret() placeholder helper, matching tb secret ls/set/rm. set() creates the secret if it doesn't exist yet, otherwise updates its value. Secret values are never returned by list() or surfaced in error messages. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 6 + README.md | 15 +++ src/tinybird_sdk/api/api.py | 80 +++++++++++++ src/tinybird_sdk/client/base.py | 39 +++++++ tests/test_secrets.py | 194 ++++++++++++++++++++++++++++++++ 5 files changed, 334 insertions(+) create mode 100644 tests/test_secrets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c1ecb0..68cb147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `TinybirdClient.secrets` namespace (`list`/`set`/`remove`), wrapping `/v0/variables` to manage the values behind the `secret()` placeholder helper, matching `tb secret ls/set/rm`. `set` creates the secret if it doesn't exist yet, otherwise updates its value; secret values are never returned by `list` or included in error messages. + ## [0.4.0] - 2026-06-29 ### Added diff --git a/README.md b/README.md index e42f34b..f5d2cc5 100644 --- a/README.md +++ b/README.md @@ -854,6 +854,21 @@ top_events = define_endpoint( ) ``` +### Secrets Management + +The `secret()` helper used above only emits a `{{ tb_secret("NAME") }}` placeholder reference in generated datafiles — it doesn't set the underlying value. Use `client.secrets` to manage the values those placeholders resolve to: + +```python +# Create or update a secret's value (idempotent) +client.secrets.set("KAFKA_KEY", "") + +# List secrets (name/created_at/updated_at only — values are never returned) +client.secrets.list() + +# Remove a secret +client.secrets.remove("KAFKA_KEY") +``` + ## Type Validators Use `t.*` to define column types: diff --git a/src/tinybird_sdk/api/api.py b/src/tinybird_sdk/api/api.py index 8ee18b5..d71da23 100644 --- a/src/tinybird_sdk/api/api.py +++ b/src/tinybird_sdk/api/api.py @@ -363,6 +363,86 @@ def create_token( self._raise_for_error(response.status_code, response.text) return response.json() + def list_secrets(self, options: dict[str, Any] | None = None) -> list[dict[str, Any]]: + """List secrets. Only name/created_at/updated_at are returned; values are never exposed.""" + options = options or {} + response = self.request( + "/v0/variables", + method="GET", + token=options.get("token"), + timeout=options.get("timeout"), + ) + if not response.ok: + self._raise_for_error(response.status_code, response.text) + data = response.json() + return data.get("variables", []) if isinstance(data, dict) else data + + def set_secret( + self, + name: str, + value: str, + options: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Create the secret if it doesn't exist yet, otherwise update its value.""" + options = options or {} + token = options.get("token") + timeout = options.get("timeout") + + existing_response = self.request( + f"/v0/variables/{name}", method="GET", token=token, timeout=timeout + ) + if existing_response.ok: + exists = True + elif existing_response.status_code == 404: + exists = False + else: + self._raise_for_error(existing_response.status_code, existing_response.text) + exists = False # unreachable, _raise_for_error always raises + + if exists: + response = self.request( + f"/v0/variables/{name}", + method="PUT", + token=token, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=urlencode({"value": value}), + timeout=timeout, + ) + else: + response = self.request( + "/v0/variables", + method="POST", + token=token, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=urlencode({"name": name, "value": value}), + timeout=timeout, + ) + if not response.ok: + self._raise_for_error(response.status_code, response.text) + if not response.text.strip(): + return {} + try: + return response.json() + except json.JSONDecodeError: + return {} + + def delete_secret(self, name: str, options: dict[str, Any] | None = None) -> dict[str, Any]: + options = options or {} + response = self.request( + f"/v0/variables/{name}", + method="DELETE", + token=options.get("token"), + timeout=options.get("timeout"), + ) + if not response.ok: + self._raise_for_error(response.status_code, response.text) + if not response.text.strip(): + return {} + try: + return response.json() + except json.JSONDecodeError: + return {} + def _timeout_seconds(self, timeout_ms: int | None) -> float: timeout = timeout_ms if timeout_ms is not None else self._default_timeout return max(timeout / 1000.0, 0.001) diff --git a/src/tinybird_sdk/client/base.py b/src/tinybird_sdk/client/base.py index b48ba09..d73c5c6 100644 --- a/src/tinybird_sdk/client/base.py +++ b/src/tinybird_sdk/client/base.py @@ -35,6 +35,20 @@ def truncate( return self._client._truncate_datasource(datasource_name, options or {}) +class _SecretsNamespace: + def __init__(self, client: "TinybirdClient"): + self._client = client + + def list(self, options: dict[str, Any] | None = None) -> list[dict[str, Any]]: + return self._client._list_secrets(options or {}) + + def set(self, name: str, value: str, options: dict[str, Any] | None = None) -> dict[str, Any]: + return self._client._set_secret(name, value, options or {}) + + def remove(self, name: str, options: dict[str, Any] | None = None) -> dict[str, Any]: + return self._client._delete_secret(name, options or {}) + + class TinybirdClient: def __init__(self, config: dict[str, Any]): if not config.get("base_url"): @@ -47,6 +61,7 @@ def __init__(self, config: dict[str, Any]): self._resolved_context: ClientContext | None = None self.datasources = _DatasourcesNamespace(self) + self.secrets = _SecretsNamespace(self) self.tokens = TokensNamespace( self._get_token, self._config["base_url"], @@ -87,6 +102,30 @@ def _truncate_datasource(self, datasource_name: str, options: dict[str, Any]) -> self._rethrow_api_error(error) raise AssertionError("unreachable") + def _list_secrets(self, options: dict[str, Any]) -> list[dict[str, Any]]: + token = self._get_token() + try: + return self._get_api(token).list_secrets(options) + except Exception as error: + self._rethrow_api_error(error) + raise AssertionError("unreachable") + + def _set_secret(self, name: str, value: str, options: dict[str, Any]) -> dict[str, Any]: + token = self._get_token() + try: + return self._get_api(token).set_secret(name, value, options) + except Exception as error: + self._rethrow_api_error(error) + raise AssertionError("unreachable") + + def _delete_secret(self, name: str, options: dict[str, Any]) -> dict[str, Any]: + token = self._get_token() + try: + return self._get_api(token).delete_secret(name, options) + except Exception as error: + self._rethrow_api_error(error) + raise AssertionError("unreachable") + def _ingest_datasource( self, datasource_name: str, event: dict[str, Any], options: dict[str, Any] ) -> dict[str, Any]: diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 0000000..b538406 --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import json +from typing import Any +from urllib.parse import parse_qs + +import pytest + +import tinybird_sdk.api.api as api_module +from tinybird_sdk.api.api import TinybirdApi, TinybirdApiError +from tinybird_sdk.client.base import TinybirdClient +from tinybird_sdk.client.types import TinybirdError + + +class _FakeResponse: + def __init__( + self, + status_code: int, + payload: dict[str, Any] | None = None, + text: str | None = None, + ): + self.status_code = status_code + self._payload = payload or {} + self._text = ( + text if text is not None else (json.dumps(self._payload) if payload is not None else "") + ) + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 300 + + @property + def text(self) -> str: + return self._text + + def json(self) -> dict[str, Any]: + return self._payload + + +def _make_api() -> TinybirdApi: + return TinybirdApi({"base_url": "https://api.tinybird.co", "token": "p.test"}) + + +def test_list_secrets(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, dict[str, Any]]] = [] + + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + calls.append((url, kwargs)) + return _FakeResponse( + 200, + { + "variables": [ + {"name": "API_KEY", "created_at": "2026-01-01", "updated_at": "2026-01-01"} + ] + }, + ) + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + result = _make_api().list_secrets() + + assert result == [{"name": "API_KEY", "created_at": "2026-01-01", "updated_at": "2026-01-01"}] + assert calls[0][0].endswith("/v0/variables") + assert calls[0][1]["method"] == "GET" + # Secret values are never part of the list response shape. + assert all("value" not in secret for secret in result) + + +def test_set_secret_creates_when_not_found(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, dict[str, Any]]] = [] + + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + calls.append((url, kwargs)) + if kwargs.get("method") == "GET": + return _FakeResponse(404, text="not found") + return _FakeResponse(200, {"name": "API_KEY"}) + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + result = _make_api().set_secret("API_KEY", "super-secret-value") + + assert result == {"name": "API_KEY"} + get_call, create_call = calls + assert get_call[1]["method"] == "GET" + assert create_call[1]["method"] == "POST" + assert create_call[0].endswith("/v0/variables") + body = parse_qs(create_call[1]["body"]) + assert body["name"] == ["API_KEY"] + assert body["value"] == ["super-secret-value"] + + +def test_set_secret_updates_when_found(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, dict[str, Any]]] = [] + + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + calls.append((url, kwargs)) + if kwargs.get("method") == "GET": + return _FakeResponse(200, {"name": "API_KEY"}) + return _FakeResponse(200, {"name": "API_KEY"}) + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + _make_api().set_secret("API_KEY", "rotated-value") + + get_call, update_call = calls + assert update_call[1]["method"] == "PUT" + assert update_call[0].endswith("/v0/variables/API_KEY") + body = parse_qs(update_call[1]["body"]) + assert body["value"] == ["rotated-value"] + assert "name" not in body + + +def test_set_secret_propagates_non_404_lookup_error(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + return _FakeResponse(403, text='{"error": "forbidden"}') + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + with pytest.raises(TinybirdApiError, match="forbidden"): + _make_api().set_secret("API_KEY", "value") + + +def test_set_secret_value_never_leaks_into_error_message(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + if kwargs.get("method") == "GET": + return _FakeResponse(404, text="not found") + return _FakeResponse(400, text='{"error": "invalid secret name"}') + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + with pytest.raises(TinybirdApiError) as exc_info: + _make_api().set_secret("API_KEY", "super-secret-value-should-not-leak") + + assert "super-secret-value-should-not-leak" not in str(exc_info.value) + + +def test_delete_secret(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, dict[str, Any]]] = [] + + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + calls.append((url, kwargs)) + return _FakeResponse(204, text="") + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + result = _make_api().delete_secret("API_KEY") + + assert result == {} + assert calls[0][1]["method"] == "DELETE" + assert calls[0][0].endswith("/v0/variables/API_KEY") + + +def test_delete_secret_raises_on_error(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_fetch(url: str, **kwargs: Any) -> _FakeResponse: + return _FakeResponse(404, text='{"error": "secret not found"}') + + monkeypatch.setattr(api_module, "tinybird_fetch", fake_fetch) + with pytest.raises(TinybirdApiError, match="secret not found"): + _make_api().delete_secret("MISSING") + + +def test_client_secrets_namespace_delegates(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeApi: + def __init__(self, config: dict[str, Any]): + self.config = config + + def list_secrets(self, options: dict[str, Any]) -> list[dict[str, Any]]: + return [{"name": "A"}] + + def set_secret(self, name: str, value: str, options: dict[str, Any]) -> dict[str, Any]: + return {"name": name} + + def delete_secret(self, name: str, options: dict[str, Any]) -> dict[str, Any]: + return {} + + import tinybird_sdk.client.base as client_base + + monkeypatch.setattr(client_base, "TinybirdApi", FakeApi) + + client = TinybirdClient({"base_url": "https://api.tinybird.co", "token": "workspace_token"}) + assert client.secrets.list() == [{"name": "A"}] + assert client.secrets.set("API_KEY", "value") == {"name": "API_KEY"} + assert client.secrets.remove("API_KEY") == {} + + +def test_client_secrets_namespace_wraps_api_errors(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeApi: + def __init__(self, config: dict[str, Any]): + self.config = config + + def list_secrets(self, options: dict[str, Any]) -> list[dict[str, Any]]: + raise TinybirdApiError("boom", 500, '{"error":"boom"}', {"error": "boom"}) + + import tinybird_sdk.client.base as client_base + + monkeypatch.setattr(client_base, "TinybirdApi", FakeApi) + + client = TinybirdClient({"base_url": "https://api.tinybird.co", "token": "workspace_token"}) + with pytest.raises(TinybirdError, match="boom"): + client.secrets.list()