diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 56be6ddb..2f2cc32e 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -54,7 +54,9 @@ endpoint ``OPENAI_BASE_URL`` when it names one (``run_env``). The licences a benchmark's environments declare are never in either: the run reads them from the benchmark's store (``nullius.secrets``) on this machine, or, on a host, from -the one line its launcher handed it (``licences``). +the one line its launcher handed it (``licences``). A data provider's token the +benchmark declares is the exception: a run without ``--from-run`` takes the +provider's variable (``HF_TOKEN``) from the environment first. """ from __future__ import annotations @@ -106,16 +108,18 @@ def run_env(provider: str, env=os.environ) -> dict: return out -def licences(bench: dict, env=os.environ): +def licences(bench: dict, env=os.environ, from_run: bool = False): """(``run.Licences``, the folder to remove after) for a run on a host, whose launcher handed it the benchmark's licences as one line (``secrets.HANDED``, ``execution``): a licence file is written to ``NULLIUS_LICENCE_DIR`` - in memory, ``/dev/shm`` - and nowhere else. - The line leaves this process's environment once read. (None, None) on - this machine: the run reads the benchmark's own store.""" + The line leaves this process's environment once read. On this machine + the run reads the benchmark's own store; a headless run (not + ``from_run``) first takes a data provider's token the benchmark declares + from ``env``, under the provider's variable (``HF_TOKEN``).""" line = env.pop(secrets.HANDED, None) if not line: - return None, None + return (None if from_run else run.Licences(bench, environ=env)), None shm = "/dev/shm" if os.path.isdir("/dev/shm") else tempfile.gettempdir() spill = env.get("NULLIUS_LICENCE_DIR") or os.path.join(shm, f"nullius-{os.getpid()}") return run.Licences.handed(bench, secrets.unpack(line), spill), spill @@ -517,7 +521,7 @@ def cmd_run(a): if why: print("refused: constraints:", why) return 2 - held, spill = licences(b) + held, spill = licences(b, from_run=a.from_run) try: job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit, concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, @@ -596,7 +600,7 @@ def cmd_fidelity(a): def cmd_prove(a): b = _bench(a.benchmark) - held, spill = licences(b) + held, spill = licences(b, from_run=a.from_run) try: got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model, stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch, diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 200ddac0..2feeb402 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -304,18 +304,23 @@ class Licences: On this machine they are read from the benchmark's store (``nullius.secrets``) when asked - when a unit starts - and a file is used where it is kept; a data provider's token (``provider``) from that - provider's store alone. A run on a host is handed them instead + provider's store alone, unless ``environ`` - a headless run's own + environment, ``standard.cli.licences`` - holds that provider's variable + (``HF_TOKEN``), which then wins. A run on a host is handed them instead (``handed``): its launcher's stdin, a file written to a folder in memory (``/dev/shm``) that the run removes when it ends. Only the names the benchmark's environments declare are ever looked up, each as the kind it is declared.""" - def __init__(self, bench: dict, given: Optional[Dict[str, dict]] = None): + def __init__(self, bench: dict, given: Optional[Dict[str, dict]] = None, environ=None): + from nullius.secrets import PROVIDERS self.key = bench["key"] declared = definitions.secrets_of(bench) self.kinds = {s["name"]: s["kind"] for s in declared} self.providers = {s["name"]: s.get("provider") or "" for s in declared} self.given = given + self.environ = {name: v for name, p in self.providers.items() if p in PROVIDERS + for v in [(environ or {}).get(PROVIDERS[p][1], "").strip()] if v} @classmethod def handed(cls, bench: dict, values: Dict[str, bytes], spill: str) -> "Licences": @@ -345,6 +350,8 @@ def get(self, name: str) -> Optional[dict]: return None if self.given is not None: return self.given.get(name) + if name in self.environ: + return {"kind": kind, "value": self.environ[name]} from nullius import secrets store, kept = secrets.store_of(self.key, name, self.providers.get(name, "")) if kind == "file": diff --git a/app/tests/fixtures/standard/envbench/agent/probe.sh b/app/tests/fixtures/standard/envbench/agent/probe.sh index 4e70cfb0..66a0126d 100644 --- a/app/tests/fixtures/standard/envbench/agent/probe.sh +++ b/app/tests/fixtures/standard/envbench/agent/probe.sh @@ -4,6 +4,7 @@ if command -v jq >/dev/null 2>&1; then echo jq=yes; else echo jq=no; fi if [ -n "$FAKE_TOOL_LICENSE" ]; then echo secret=yes; else echo secret=no; fi if [ -n "$FAKE_SCORER_TOKEN" ]; then echo scorer=yes; else echo scorer=no; fi + if [ -n "$HF_TOKEN" ]; then echo hf=yes; else echo hf=no; fi if [ -f /opt/licensed-tool/marker.txt ]; then echo marker=yes; else echo marker=no; fi # the licence file: by the variable that points to it, and at its declared path (in a container) if [ -n "$FAKE_TOOL_LICENSE_PATH" ] && grep -q "^FAKE-LICENCE-FILE" "$FAKE_TOOL_LICENSE_PATH" 2>/dev/null; then echo licfile=yes; else echo licfile=no; fi diff --git a/app/tests/test_secrets.py b/app/tests/test_secrets.py index aba3e76e..fd8a7216 100644 --- a/app/tests/test_secrets.py +++ b/app/tests/test_secrets.py @@ -56,6 +56,34 @@ def test_a_run_reads_a_data_providers_token_from_its_provider_alone(self): secrets.set(secrets.HUGGINGFACE, "HF_TOKEN", "shared-synthetic") self.assertEqual(held.get("HF_TOKEN"), {"kind": "value", "value": "shared-synthetic"}) + def test_a_headless_run_takes_a_data_providers_token_from_its_environment_first(self): + from nullius.standard import cli, run + bench = {"key": "bench-a", "environments": {"e": {"needs_secrets": [ + {"name": "HF_TOKEN", "to": "driver", "provider": "huggingface"}, {"name": "LICENSE_KEY", "to": "driver"}]}}} + env = {"HF_TOKEN": "env-synthetic", "LICENSE_KEY": "env-licence-synthetic"} + held, spill = cli.licences(bench, env=dict(env)) + self.assertIsNone(spill) + self.assertEqual(held.get("HF_TOKEN"), {"kind": "value", "value": "env-synthetic"}) # nothing in the store + self.assertIsNone(held.get("LICENSE_KEY")) # not a provider's: the store alone + secrets.set(secrets.HUGGINGFACE, "HF_TOKEN", "stored-synthetic") + self.assertEqual(held.get("HF_TOKEN"), {"kind": "value", "value": "env-synthetic"}) # the environment wins + for unset in ({}, {"HF_TOKEN": ""}, {"HF_TOKEN": " "}): + with self.subTest(env=unset): + self.assertEqual(cli.licences(bench, env=unset)[0].get("HF_TOKEN"), {"kind": "value", "value": "stored-synthetic"}) + self.assertIsNone(cli.licences(bench, env=dict(env), from_run=True)[0]) # the app's run: as before + handed = {secrets.HANDED: base64.b64encode(json.dumps({"HF_TOKEN": base64.b64encode(b"handed-synthetic").decode()}).encode()).decode(), "HF_TOKEN": "env-synthetic"} + self.assertEqual(cli.licences(bench, env=handed)[0].get("HF_TOKEN"), {"kind": "value", "value": "handed-synthetic"}) + + def test_a_benchmark_that_declares_no_provider_takes_nothing_from_the_environment(self): + from nullius.standard import cli, run + bench = {"key": "bench-a", "environments": {"e": {"needs_secrets": [{"name": "HF_TOKEN", "to": "driver"}]}}} + held = cli.licences(bench, env={"HF_TOKEN": "env-synthetic"})[0] + self.assertEqual(held.environ, {}) + self.assertIsNone(held.get("HF_TOKEN")) + bare = run.Licences({"key": "bench-b"}, environ={"HF_TOKEN": "env-synthetic"}) + self.assertEqual(bare.environ, {}) + self.assertIsNone(bare.get("HF_TOKEN")) + def mode(self, p): return stat.S_IMODE(os.stat(p).st_mode) diff --git a/app/tests/test_standard_environments.py b/app/tests/test_standard_environments.py index fcf2e526..915cc590 100644 --- a/app/tests/test_standard_environments.py +++ b/app/tests/test_standard_environments.py @@ -22,7 +22,7 @@ from nullius import secrets, settings from nullius.runs import classify, parse -from nullius.standard import container, definitions, install, run +from nullius.standard import cli, container, definitions, install, run from tests import docker_tier HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "envbench") @@ -327,10 +327,10 @@ def store(self, server_host="127.0.0.1"): secrets.set("envbench", "FAKE_TOOL_LICENSE_FILE", LIC_FILE, kind="file", filename="tool.lic") secrets.set("envbench", "FAKE_LICENSE_SERVER", f"{server_host}:{self.server.port}", kind="server") - def run_it(self, backend, unsandboxed=True, other=None, b=None): + def run_it(self, backend, unsandboxed=True, other=None, b=None, licences=None): env = {"PATH": os.environ.get("PATH", os.defpath), "MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "sk-test-not-real"} job_dir = run.run(b or bench(), other or probe_agent(), model="m", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), - concurrency=3, env=env, say=lambda s: None, unsandboxed=unsandboxed, backend=backend) + concurrency=3, env=env, say=lambda s: None, unsandboxed=unsandboxed, backend=backend, licences=licences) trials = {} for name in os.listdir(job_dir): p = os.path.join(job_dir, name, "result.json") @@ -341,6 +341,22 @@ def run_it(self, backend, unsandboxed=True, other=None, b=None): trials[r["standard"]["unit"]] = r return job_dir, trials + def from_environment(self, server_host="127.0.0.1"): + """The fixture with its driver's token declared a data provider's, given only as ``HF_TOKEN`` in a + headless run's environment (``standard.cli.licences``), the rest in its store; and the process's own + ``HF_TOKEN`` set too, so no child can inherit it.""" + b = bench() + for s in b["environments"]["licensed"]["needs_secrets"]: + if s["name"] == "FAKE_SCORER_TOKEN": + s["provider"] = "huggingface" + self.store(server_host) + secrets.delete("envbench", "FAKE_SCORER_TOKEN") + shell = mock.patch.dict(os.environ, {"HF_TOKEN": SECRETS["FAKE_SCORER_TOKEN"]}) + shell.start() + self.addCleanup(shell.stop) + held, _ = cli.licences(b, env={"HF_TOKEN": SECRETS["FAKE_SCORER_TOKEN"]}) + return b, held + def assertNoSecretIn(self, job_dir): for root, _, files in os.walk(job_dir): for n in files: @@ -383,6 +399,15 @@ def test_with_them_each_goes_where_it_is_sent_and_nowhere_else(self): self.assertEqual((plain["secret"], plain["licfile"], plain["server"]), ("no", "no", "no")) # another environment's stay there self.assertNoSecretIn(job_dir) + def test_a_data_providers_token_from_the_environment_reaches_its_driver_alone(self): + b, held = self.from_environment() + job_dir, t = self.run_it("local", b=b, licences=held) + lic = t["licensed"] + self.assertEqual(lic["verifier_result"]["rewards"]["reward"], 1.0, lic.get("exception_info") or lic["_score"]) + detail = lic["_score"]["detail"] + self.assertEqual((detail["scorer"], detail["hf"], detail["driver_has_scorer_token"]), ("no", "no", True)) # the driver's alone + self.assertNoSecretIn(job_dir) # redacted from every log + def test_another_benchmarks_licences_are_not_its_own(self): for n, v in SECRETS.items(): secrets.set("otherbench", n, v) @@ -436,6 +461,14 @@ def test_each_unit_runs_in_its_own_environment(self): capture_output=True, text=True).stdout.split() self.assertEqual(left, []) # the forwarder went with its unit + def test_a_data_providers_token_from_the_environment_reaches_its_driver_alone(self): + b, held = self.from_environment(server_host="host.docker.internal") + job_dir, t = self.run_it("docker", unsandboxed=False, b=b, licences=held) + lic = t["licensed"] + self.assertEqual(lic["verifier_result"]["rewards"]["reward"], 1.0, lic.get("exception_info") or lic["_score"]) + self.assertEqual((lic["_score"]["detail"]["scorer"], lic["_score"]["detail"]["hf"]), ("no", "no")) # not in the agent's container + self.assertNoSecretIn(job_dir) + if __name__ == "__main__": unittest.main() diff --git a/docs/cli/run.md b/docs/cli/run.md index 8414a44f..7ab8d2fe 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -45,6 +45,7 @@ nullius standard run locomo model --model gpt-4o-mini | --- | --- | | The pair must fit | Else refused, naming the missing port | | Keys | Environment, then user `settings.json`; the agent never gets one | +| A data provider's token | `HF_TOKEN` from the environment, then Settings → Data providers, for a benchmark that declares Hugging Face; only its driver gets it. Not with `--from-run` | | A control | No `--model`, no `--unsandboxed`; only a judge is called | | Machine programs | An option needing them is refused without them | | Progress | stderr, one line per stage (`[nullius] stage=install ...`, `stage=units n=10`, `stage=done`, `stage=refused`), also `progress.jsonl` | diff --git a/docs/settings/data-providers.md b/docs/settings/data-providers.md index 362d2e1d..63bbb597 100644 --- a/docs/settings/data-providers.md +++ b/docs/settings/data-providers.md @@ -4,5 +4,5 @@ | Provider | | | --- | --- | -| Hugging Face | A read token; accept each gated dataset's access conditions on Hugging Face first. Kept in `secrets/data-provider-huggingface/`; sent only to a Nullius benchmark's driver that declares it, never to an agent. Harbor runs use the `HF_TOKEN` of the environment Nullius started in, or a host's own. **Endpoint URL**, optional: a mirror such as `https://hf-mirror.com` in place of `https://huggingface.co`, saved as `HF_ENDPOINT` in `settings.json`; `HF_ENDPOINT` in the environment wins, also for a headless run. A run's drivers get it from their start, on either backend and on a host, and a dataset file pinned to huggingface.co is fetched from it, checked by the same digest. A gated dataset still needs the token | +| Hugging Face | A read token; accept each gated dataset's access conditions on Hugging Face first. Kept in `secrets/data-provider-huggingface/`; sent only to a Nullius benchmark's driver that declares it, never to an agent. A headless `nullius standard run` or `prove` reads `HF_TOKEN` from the environment first. Harbor runs use the `HF_TOKEN` of the environment Nullius started in, or a host's own. **Endpoint URL**, optional: a mirror such as `https://hf-mirror.com` in place of `https://huggingface.co`, saved as `HF_ENDPOINT` in `settings.json`; `HF_ENDPOINT` in the environment wins, also for a headless run. A run's drivers get it from their start, on either backend and on a host, and a dataset file pinned to huggingface.co is fetched from it, checked by the same digest. A gated dataset still needs the token | | Docker Hub | A username and a read-only access token, given to `docker login --password-stdin` (`podman login ... docker.io`) once. The engine keeps them (Docker’s `~/.docker/config.json` or its credential helper, Podman’s `auth.json`); Nullius reads back the username only. **Remove** signs out. This machine only; a run host keeps its own login. Without a login, anonymous pull limits can stop a benchmark with an image per task part-way |