Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions app/nullius/standard/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@
endpoint ``OPENAI_BASE_URL`` when it names one (``run_env``). The licences a
benchmark's environments declare are never in either: the run reads them from
the benchmark's store (``nullius.secrets``) on this machine, or, on a host, from
the one line its launcher handed it (``licences``).
the one line its launcher handed it (``licences``). A data provider's token the
benchmark declares is the exception: a run without ``--from-run`` takes the
provider's variable (``HF_TOKEN``) from the environment first.
"""
from __future__ import annotations

Expand Down Expand Up @@ -106,16 +108,18 @@ def run_env(provider: str, env=os.environ) -> dict:
return out


def licences(bench: dict, env=os.environ):
def licences(bench: dict, env=os.environ, from_run: bool = False):
"""(``run.Licences``, the folder to remove after) for a run on a host,
whose launcher handed it the benchmark's licences as one line
(``secrets.HANDED``, ``execution``): a licence file is written to
``NULLIUS_LICENCE_DIR`` - in memory, ``/dev/shm`` - and nowhere else.
The line leaves this process's environment once read. (None, None) on
this machine: the run reads the benchmark's own store."""
The line leaves this process's environment once read. On this machine
the run reads the benchmark's own store; a headless run (not
``from_run``) first takes a data provider's token the benchmark declares
from ``env``, under the provider's variable (``HF_TOKEN``)."""
line = env.pop(secrets.HANDED, None)
if not line:
return None, None
return (None if from_run else run.Licences(bench, environ=env)), None
shm = "/dev/shm" if os.path.isdir("/dev/shm") else tempfile.gettempdir()
spill = env.get("NULLIUS_LICENCE_DIR") or os.path.join(shm, f"nullius-{os.getpid()}")
return run.Licences.handed(bench, secrets.unpack(line), spill), spill
Expand Down Expand Up @@ -517,7 +521,7 @@ def cmd_run(a):
if why:
print("refused: constraints:", why)
return 2
held, spill = licences(b)
held, spill = licences(b, from_run=a.from_run)
try:
job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit,
concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend,
Expand Down Expand Up @@ -596,7 +600,7 @@ def cmd_fidelity(a):

def cmd_prove(a):
b = _bench(a.benchmark)
held, spill = licences(b)
held, spill = licences(b, from_run=a.from_run)
try:
got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model,
stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch,
Expand Down
11 changes: 9 additions & 2 deletions app/nullius/standard/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,18 +304,23 @@ class Licences:
On this machine they are read from the benchmark's store
(``nullius.secrets``) when asked - when a unit starts - and a file is
used where it is kept; a data provider's token (``provider``) from that
provider's store alone. A run on a host is handed them instead
provider's store alone, unless ``environ`` - a headless run's own
environment, ``standard.cli.licences`` - holds that provider's variable
(``HF_TOKEN``), which then wins. A run on a host is handed them instead
(``handed``): its launcher's stdin, a file written to a folder in memory
(``/dev/shm``) that the run removes when it ends. Only the names the
benchmark's environments declare are ever looked up, each as the kind
it is declared."""

def __init__(self, bench: dict, given: Optional[Dict[str, dict]] = None):
def __init__(self, bench: dict, given: Optional[Dict[str, dict]] = None, environ=None):
from nullius.secrets import PROVIDERS
self.key = bench["key"]
declared = definitions.secrets_of(bench)
self.kinds = {s["name"]: s["kind"] for s in declared}
self.providers = {s["name"]: s.get("provider") or "" for s in declared}
self.given = given
self.environ = {name: v for name, p in self.providers.items() if p in PROVIDERS
for v in [(environ or {}).get(PROVIDERS[p][1], "").strip()] if v}

@classmethod
def handed(cls, bench: dict, values: Dict[str, bytes], spill: str) -> "Licences":
Expand Down Expand Up @@ -345,6 +350,8 @@ def get(self, name: str) -> Optional[dict]:
return None
if self.given is not None:
return self.given.get(name)
if name in self.environ:
return {"kind": kind, "value": self.environ[name]}
from nullius import secrets
store, kept = secrets.store_of(self.key, name, self.providers.get(name, ""))
if kind == "file":
Expand Down
1 change: 1 addition & 0 deletions app/tests/fixtures/standard/envbench/agent/probe.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
if command -v jq >/dev/null 2>&1; then echo jq=yes; else echo jq=no; fi
if [ -n "$FAKE_TOOL_LICENSE" ]; then echo secret=yes; else echo secret=no; fi
if [ -n "$FAKE_SCORER_TOKEN" ]; then echo scorer=yes; else echo scorer=no; fi
if [ -n "$HF_TOKEN" ]; then echo hf=yes; else echo hf=no; fi
if [ -f /opt/licensed-tool/marker.txt ]; then echo marker=yes; else echo marker=no; fi
# the licence file: by the variable that points to it, and at its declared path (in a container)
if [ -n "$FAKE_TOOL_LICENSE_PATH" ] && grep -q "^FAKE-LICENCE-FILE" "$FAKE_TOOL_LICENSE_PATH" 2>/dev/null; then echo licfile=yes; else echo licfile=no; fi
Expand Down
28 changes: 28 additions & 0 deletions app/tests/test_secrets.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,34 @@ def test_a_run_reads_a_data_providers_token_from_its_provider_alone(self):
secrets.set(secrets.HUGGINGFACE, "HF_TOKEN", "shared-synthetic")
self.assertEqual(held.get("HF_TOKEN"), {"kind": "value", "value": "shared-synthetic"})

def test_a_headless_run_takes_a_data_providers_token_from_its_environment_first(self):
from nullius.standard import cli, run
bench = {"key": "bench-a", "environments": {"e": {"needs_secrets": [
{"name": "HF_TOKEN", "to": "driver", "provider": "huggingface"}, {"name": "LICENSE_KEY", "to": "driver"}]}}}
env = {"HF_TOKEN": "env-synthetic", "LICENSE_KEY": "env-licence-synthetic"}
held, spill = cli.licences(bench, env=dict(env))
self.assertIsNone(spill)
self.assertEqual(held.get("HF_TOKEN"), {"kind": "value", "value": "env-synthetic"}) # nothing in the store
self.assertIsNone(held.get("LICENSE_KEY")) # not a provider's: the store alone
secrets.set(secrets.HUGGINGFACE, "HF_TOKEN", "stored-synthetic")
self.assertEqual(held.get("HF_TOKEN"), {"kind": "value", "value": "env-synthetic"}) # the environment wins
for unset in ({}, {"HF_TOKEN": ""}, {"HF_TOKEN": " "}):
with self.subTest(env=unset):
self.assertEqual(cli.licences(bench, env=unset)[0].get("HF_TOKEN"), {"kind": "value", "value": "stored-synthetic"})
self.assertIsNone(cli.licences(bench, env=dict(env), from_run=True)[0]) # the app's run: as before
handed = {secrets.HANDED: base64.b64encode(json.dumps({"HF_TOKEN": base64.b64encode(b"handed-synthetic").decode()}).encode()).decode(), "HF_TOKEN": "env-synthetic"}
self.assertEqual(cli.licences(bench, env=handed)[0].get("HF_TOKEN"), {"kind": "value", "value": "handed-synthetic"})

def test_a_benchmark_that_declares_no_provider_takes_nothing_from_the_environment(self):
from nullius.standard import cli, run
bench = {"key": "bench-a", "environments": {"e": {"needs_secrets": [{"name": "HF_TOKEN", "to": "driver"}]}}}
held = cli.licences(bench, env={"HF_TOKEN": "env-synthetic"})[0]
self.assertEqual(held.environ, {})
self.assertIsNone(held.get("HF_TOKEN"))
bare = run.Licences({"key": "bench-b"}, environ={"HF_TOKEN": "env-synthetic"})
self.assertEqual(bare.environ, {})
self.assertIsNone(bare.get("HF_TOKEN"))

def mode(self, p):
return stat.S_IMODE(os.stat(p).st_mode)

Expand Down
39 changes: 36 additions & 3 deletions app/tests/test_standard_environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

from nullius import secrets, settings
from nullius.runs import classify, parse
from nullius.standard import container, definitions, install, run
from nullius.standard import cli, container, definitions, install, run
from tests import docker_tier

HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "envbench")
Expand Down Expand Up @@ -327,10 +327,10 @@ def store(self, server_host="127.0.0.1"):
secrets.set("envbench", "FAKE_TOOL_LICENSE_FILE", LIC_FILE, kind="file", filename="tool.lic")
secrets.set("envbench", "FAKE_LICENSE_SERVER", f"{server_host}:{self.server.port}", kind="server")

def run_it(self, backend, unsandboxed=True, other=None, b=None):
def run_it(self, backend, unsandboxed=True, other=None, b=None, licences=None):
env = {"PATH": os.environ.get("PATH", os.defpath), "MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "sk-test-not-real"}
job_dir = run.run(b or bench(), other or probe_agent(), model="m", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"),
concurrency=3, env=env, say=lambda s: None, unsandboxed=unsandboxed, backend=backend)
concurrency=3, env=env, say=lambda s: None, unsandboxed=unsandboxed, backend=backend, licences=licences)
trials = {}
for name in os.listdir(job_dir):
p = os.path.join(job_dir, name, "result.json")
Expand All @@ -341,6 +341,22 @@ def run_it(self, backend, unsandboxed=True, other=None, b=None):
trials[r["standard"]["unit"]] = r
return job_dir, trials

def from_environment(self, server_host="127.0.0.1"):
"""The fixture with its driver's token declared a data provider's, given only as ``HF_TOKEN`` in a
headless run's environment (``standard.cli.licences``), the rest in its store; and the process's own
``HF_TOKEN`` set too, so no child can inherit it."""
b = bench()
for s in b["environments"]["licensed"]["needs_secrets"]:
if s["name"] == "FAKE_SCORER_TOKEN":
s["provider"] = "huggingface"
self.store(server_host)
secrets.delete("envbench", "FAKE_SCORER_TOKEN")
shell = mock.patch.dict(os.environ, {"HF_TOKEN": SECRETS["FAKE_SCORER_TOKEN"]})
shell.start()
self.addCleanup(shell.stop)
held, _ = cli.licences(b, env={"HF_TOKEN": SECRETS["FAKE_SCORER_TOKEN"]})
return b, held

def assertNoSecretIn(self, job_dir):
for root, _, files in os.walk(job_dir):
for n in files:
Expand Down Expand Up @@ -383,6 +399,15 @@ def test_with_them_each_goes_where_it_is_sent_and_nowhere_else(self):
self.assertEqual((plain["secret"], plain["licfile"], plain["server"]), ("no", "no", "no")) # another environment's stay there
self.assertNoSecretIn(job_dir)

def test_a_data_providers_token_from_the_environment_reaches_its_driver_alone(self):
b, held = self.from_environment()
job_dir, t = self.run_it("local", b=b, licences=held)
lic = t["licensed"]
self.assertEqual(lic["verifier_result"]["rewards"]["reward"], 1.0, lic.get("exception_info") or lic["_score"])
detail = lic["_score"]["detail"]
self.assertEqual((detail["scorer"], detail["hf"], detail["driver_has_scorer_token"]), ("no", "no", True)) # the driver's alone
self.assertNoSecretIn(job_dir) # redacted from every log

def test_another_benchmarks_licences_are_not_its_own(self):
for n, v in SECRETS.items():
secrets.set("otherbench", n, v)
Expand Down Expand Up @@ -436,6 +461,14 @@ def test_each_unit_runs_in_its_own_environment(self):
capture_output=True, text=True).stdout.split()
self.assertEqual(left, []) # the forwarder went with its unit

def test_a_data_providers_token_from_the_environment_reaches_its_driver_alone(self):
b, held = self.from_environment(server_host="host.docker.internal")
job_dir, t = self.run_it("docker", unsandboxed=False, b=b, licences=held)
lic = t["licensed"]
self.assertEqual(lic["verifier_result"]["rewards"]["reward"], 1.0, lic.get("exception_info") or lic["_score"])
self.assertEqual((lic["_score"]["detail"]["scorer"], lic["_score"]["detail"]["hf"]), ("no", "no")) # not in the agent's container
self.assertNoSecretIn(job_dir)


if __name__ == "__main__":
unittest.main()
1 change: 1 addition & 0 deletions docs/cli/run.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ nullius standard run locomo model --model gpt-4o-mini
| --- | --- |
| The pair must fit | Else refused, naming the missing port |
| Keys | Environment, then user `settings.json`; the agent never gets one |
| A data provider's token | `HF_TOKEN` from the environment, then Settings → Data providers, for a benchmark that declares Hugging Face; only its driver gets it. Not with `--from-run` |
| A control | No `--model`, no `--unsandboxed`; only a judge is called |
| Machine programs | An option needing them is refused without them |
| Progress | stderr, one line per stage (`[nullius] stage=install ...`, `stage=units n=10`, `stage=done`, `stage=refused`), also `progress.jsonl` |
Expand Down
2 changes: 1 addition & 1 deletion docs/settings/data-providers.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@

| Provider | |
| --- | --- |
| Hugging Face | A read token; accept each gated dataset's access conditions on Hugging Face first. Kept in `secrets/data-provider-huggingface/`; sent only to a Nullius benchmark's driver that declares it, never to an agent. Harbor runs use the `HF_TOKEN` of the environment Nullius started in, or a host's own. **Endpoint URL**, optional: a mirror such as `https://hf-mirror.com` in place of `https://huggingface.co`, saved as `HF_ENDPOINT` in `settings.json`; `HF_ENDPOINT` in the environment wins, also for a headless run. A run's drivers get it from their start, on either backend and on a host, and a dataset file pinned to huggingface.co is fetched from it, checked by the same digest. A gated dataset still needs the token |
| Hugging Face | A read token; accept each gated dataset's access conditions on Hugging Face first. Kept in `secrets/data-provider-huggingface/`; sent only to a Nullius benchmark's driver that declares it, never to an agent. A headless `nullius standard run` or `prove` reads `HF_TOKEN` from the environment first. Harbor runs use the `HF_TOKEN` of the environment Nullius started in, or a host's own. **Endpoint URL**, optional: a mirror such as `https://hf-mirror.com` in place of `https://huggingface.co`, saved as `HF_ENDPOINT` in `settings.json`; `HF_ENDPOINT` in the environment wins, also for a headless run. A run's drivers get it from their start, on either backend and on a host, and a dataset file pinned to huggingface.co is fetched from it, checked by the same digest. A gated dataset still needs the token |
| Docker Hub | A username and a read-only access token, given to `docker login --password-stdin` (`podman login ... docker.io`) once. The engine keeps them (Docker’s `~/.docker/config.json` or its credential helper, Podman’s `auth.json`); Nullius reads back the username only. **Remove** signs out. This machine only; a run host keeps its own login. Without a login, anonymous pull limits can stop a benchmark with an image per task part-way |
Loading