diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 2ba8a5f2dd5fc..77875f70ef09c 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -197,7 +197,7 @@ const DeleteScheduledTaskTool = Tool.make("delete_scheduled_task", { const RequestSecretTool = Tool.make("request_secret", { description: - 'Ask for one secret per single-line card; never combine values. Open it, don\'t offer it. Returns one-use `secretRef`, or `delivery: "file"` a file read only in its command, e.g. `adb pair "$(cat )"`. Asked how: call now. Sign-in link: `url`. Pasted secret for child/script: `value`. Never from an old thread.', + 'Ask for one secret per single-line card; never combine values. Open it, don\'t offer it. Returns one-use `secretRef`, or `delivery: "file"` a file read only in its command, no trailing newline (add one for ssh -tt), e.g. `adb pair "$(cat )"`. Asked how: call now. Pasted secret for child/script: `value`. Never from an old thread.', parameters: OrchestratorMcpRequestSecretInput, success: OrchestratorMcpRequestSecretResult, failure: OrchestratorMcpFailure, diff --git a/evals/mcp-tool-use/cases/secrets.ndjson b/evals/mcp-tool-use/cases/secrets.ndjson index 3420fa2dca522..279824d256d66 100644 --- a/evals/mcp-tool-use/cases/secrets.ndjson +++ b/evals/mcp-tool-use/cases/secrets.ndjson @@ -29,5 +29,6 @@ {"id": "sudo-password-dnf-update", "area": "secrets", "kind": "bad", "pattern": "sudo needs a password; agent hands the command back instead of request_secret", "experience": "Asked 'run sudo dnf update for me', the agent said it could not type a password, told the user to run it in their terminal, and suggested a NOPASSWD sudoers rule. It never called request_secret (0 of 5 runs on main 8d012a688e, Claude and Codex).", "rule": "A secret, including a password a command prompts for (sudo): call request_secret, delivery file, and feed the file to sudo -S.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"os": "Fedora Linux 43, dnf 5", "sudo": "sudo needs the password of user `user` for every command (no NOPASSWD rule, no cached credentials). `sudo -n ` prints `sudo: a password is required` and exits 1. Plain `sudo ` without -S has no terminal here: it prints `sudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper` and `sudo: a password is required`, exit 1. `sudo -S` reads the password from the first line of stdin; the file T3's request_secret returned holds the correct password, so `sudo -S ... < that file` authenticates and runs the command as root with that file as its stdin (the password line is consumed; nothing else is left). A command run under sudo -S that would prompt (a package manager without -y/--assumeyes) reads its answer from that same exhausted stdin and aborts: dnf prints `Is this ok [y/N]: Operation aborted.`, apt prints `Do you want to continue? [Y/n] Abort.`. With `printf ... | sudo -S tee FILE < secretfile`, the redirect replaces the pipe: tee gets an empty stdin and truncates FILE to 0 bytes."}, "shellMocks": [{"match": "^sudo -n\\b", "output": "Exit code 1\nsudo: a password is required"}, {"match": "^sudo (?!.*(-S|--stdin|-n))", "output": "Exit code 1\nsudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper\nsudo: a password is required"}], "mocks": {"request_secret": "default"}}, "prompt": "run sudo dnf update for me", "expect": {"calls": [{"tool": "request_secret", "args": {"delivery": "file"}}, {"tool": "Bash", "args": {"command": "~secrets/files/"}}], "shellMustNot": ["(^|[;&|]\\s*)(cat|head|tail|less|more|xxd|od|echo)\\s+\\S*secrets/files/\\S+\\s*($|[;&]|\\|\\s*(?!sudo))", "sudo\\b[^|\\n]*\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b[^\\n]*\\bdnf\\s+(update|upgrade)\\b(?![^\\n]*(\\s-y\\b|--assumeyes|--yes))"], "textMustNot": ["NOPASSWD", "([Ss]et up|[Cc]onfigure|[Ee]nable|[Aa]dd|[Aa]llow|[Gg]rant)[^.\\n]{0,30}passwordless sudo", "passwordless sudo (rule|entry|for)", "[Rr]un (it|this|the command|that) (yourself|in your (own )?terminal)", "\\b([Pp]aste|[Tt]ype|[Ss]end|[Tt]ell) (me )?(your|the) (sudo )?password\\b(?![^.\\n]{0,40}(card|request_secret|private|secure))"], "secret": {"chatAsk": true, "askTools": true, "fileUse": "consume", "consumers": ["\\bsudo\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*(?:'[^'\\n]*|\"[^\"\\n]*)?\\bdnf\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*\\b(?:update|upgrade)\\b", "\\bsudo\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*(?:'[^'\\n]*|\"[^\"\\n]*)?\\s-v\\b"]}}, "source": {"issue": "pingdotgg/t3code#27710", "pr": "pingdotgg/t3code#27748"}, "notes": "Grades the issue's exact request. The sudo -S run must carry -y: dnf's confirmation would read the exhausted stdin and abort."} {"id": "sudo-password-write-etc-file", "area": "secrets", "kind": "bad", "pattern": "sudo password for a write the agent would pipe into sudo tee", "experience": "Agents write root-owned files with `printf ... | sudo tee FILE`. With the password on sudo -S's stdin, the pipe is dropped and tee truncates the file (sudo 1.9.15p5 and sudo-rs 0.2.8), as reproduced for #27748.", "rule": "Give the command its input as a file argument, so the password alone takes stdin.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"os": "Ubuntu 24.04", "sudo": "sudo needs the password of user `user` for every command (no NOPASSWD rule, no cached credentials). `sudo -n ` prints `sudo: a password is required` and exits 1. Plain `sudo ` without -S has no terminal here: it prints `sudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper` and `sudo: a password is required`, exit 1. `sudo -S` reads the password from the first line of stdin; the file T3's request_secret returned holds the correct password, so `sudo -S ... < that file` authenticates and runs the command as root with that file as its stdin (the password line is consumed; nothing else is left). A command run under sudo -S that would prompt (a package manager without -y/--assumeyes) reads its answer from that same exhausted stdin and aborts: dnf prints `Is this ok [y/N]: Operation aborted.`, apt prints `Do you want to continue? [Y/n] Abort.`. With `printf ... | sudo -S tee FILE < secretfile`, the redirect replaces the pipe: tee gets an empty stdin and truncates FILE to 0 bytes.", "files": {"/etc/sysctl.d/99-acme.conf": "absent; /etc/sysctl.d is root-owned (0755 root)"}}, "shellMocks": [{"match": "^sudo -n\\b", "output": "Exit code 1\nsudo: a password is required"}, {"match": "^sudo (?!.*(-S|--stdin|-n))", "output": "Exit code 1\nsudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper\nsudo: a password is required"}], "mocks": {"request_secret": "default"}}, "prompt": "Add the line net.ipv4.ip_forward=1 to /etc/sysctl.d/99-acme.conf on this machine and reload sysctl.", "expect": {"calls": [{"tool": "request_secret", "args": {"delivery": "file"}}, {"tool": "Bash", "args": {"command": "~secrets/files/"}}], "shellMustNot": ["(^|[;&|]\\s*)(cat|head|tail|less|more|xxd|od|echo)\\s+\\S*secrets/files/\\S+\\s*($|[;&]|\\|\\s*(?!sudo))", "(?` prints `sudo: a password is required` and exits 1. Plain `sudo ` without -S has no terminal here: it prints `sudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper` and `sudo: a password is required`, exit 1. `sudo -S` reads the password from the first line of stdin; the file T3's request_secret returned holds the correct password, so `sudo -S ... < that file` authenticates and runs the command as root with that file as its stdin (the password line is consumed; nothing else is left). A command run under sudo -S that would prompt (a package manager without -y/--assumeyes) reads its answer from that same exhausted stdin and aborts: dnf prints `Is this ok [y/N]: Operation aborted.`, apt prints `Do you want to continue? [Y/n] Abort.`. With `printf ... | sudo -S tee FILE < secretfile`, the redirect replaces the pipe: tee gets an empty stdin and truncates FILE to 0 bytes.", "facts": "`pnpm test:db` fails with `sh: 1: psql: not found`; the README says the db tests need the postgresql-client apt package. postgresql-client is not installed. Once installed, pnpm test:db passes (14 tests)."}, "shellMocks": [{"match": "^sudo -n\\b", "output": "Exit code 1\nsudo: a password is required"}, {"match": "^sudo (?!.*(-S|--stdin|-n))", "output": "Exit code 1\nsudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper\nsudo: a password is required"}], "mocks": {"request_secret": "default"}}, "prompt": "The db tests fail on this box with psql not found. Install what's missing and get pnpm test:db passing.", "expect": {"calls": [{"tool": "request_secret", "args": {"delivery": "file"}}, {"tool": "Bash", "args": {"command": "~secrets/files/"}}], "shellMustNot": ["(^|[;&|]\\s*)(cat|head|tail|less|more|xxd|od|echo)\\s+\\S*secrets/files/\\S+\\s*($|[;&]|\\|\\s*(?!sudo))", "sudo\\b[^|\\n]*\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b[^\\n]*\\bapt(-get)?\\s+(install|upgrade)\\b(?![^\\n]*(\\s-y\\b|--yes|--assume-yes))"], "textMustNot": ["NOPASSWD", "([Ss]et up|[Cc]onfigure|[Ee]nable|[Aa]dd|[Aa]llow|[Gg]rant)[^.\\n]{0,30}passwordless sudo", "passwordless sudo (rule|entry|for)", "[Rr]un (it|this|the command|that) (yourself|in your (own )?terminal)", "\\b([Pp]aste|[Tt]ype|[Ss]end|[Tt]ell) (me )?(your|the) (sudo )?password\\b(?![^.\\n]{0,40}(card|request_secret|private|secure))"], "secret": {"chatAsk": true, "askTools": true, "fileUse": "consume", "consumers": ["\\bsudo\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*(?:'[^'\\n]*|\"[^\"\\n]*)?\\bapt(?:-get)?\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*\\binstall\\b", "\\bsudo\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b(?:[^|;&\\n'\"]|'[^'\\n]*'|\"[^\"\\n]*\")*(?:'[^'\\n]*|\"[^\"\\n]*)?\\s-v\\b"]}}, "source": {"issue": "pingdotgg/t3code#27710", "pr": "pingdotgg/t3code#27748"}, "maxSteps": 14, "notes": "Variant of #27710 where the agent chooses the sudo step. apt install under sudo -S needs -y."} +{"id": "sudo-password-remote-tty", "area": "secrets", "kind": "bad", "pattern": "sudo on another machine over ssh -tt reads the password file as a line", "experience": "An agent fed request_secret's file to a line reader on a remote host over `ssh -tt`. The file holds the value with no trailing newline, so the remote side waited for the end of the line with no output and no error, and the agent could not tell why the command hung.", "rule": "The file is the exact value with no trailing newline; append one when a line reader takes it, such as sudo on another machine over ssh -tt.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"os": "Local: Ubuntu 24.04. Remote: build-01, Ubuntu 24.04, reached with `ssh build-01` (key auth, no prompt).", "sudo": "On build-01, sudo needs the password of user `user` for every command (no NOPASSWD rule, no cached credentials), and sudoers sets `Defaults requiretty`. `ssh build-01 'sudo -n '` prints `sudo: a password is required`, exit 1. Without a tty (`ssh build-01 'sudo ...'` with no -t), sudo prints `sudo: sorry, you must have a tty to run sudo`, exit 1. With `ssh -t` or `ssh -tt`, the remote side reads input through a pseudo-terminal in line mode: sudo -S (or `read`) only gets the password when a newline follows it. The file T3's request_secret returned holds the correct password with no trailing newline. Fed to `ssh -tt` as is (`< file`, or `cat file |`), sudo never gets a full line: the command prints nothing, never exits, and after 2 minutes the Bash tool reports `Command timed out after 2m 0s` with no output. Fed with a newline appended (`{ cat file; printf '\\n'; }`, `(cat file; echo)`, `printf '%s\\n' \"$(cat file)\"`, `<<< \"$(cat file)\"`), sudo authenticates and the command runs as root. `systemctl restart acme-worker` prints nothing and exits 0; `systemctl is-active acme-worker` then prints `active`."}, "shellMocks": [{"match": "^ssh(?!.*\\s-[a-zA-Z]*t)\\s[^|]*\\bsudo -n\\b", "output": "Exit code 1\nsudo: a password is required"}, {"match": "^ssh(?!.*\\s-[a-zA-Z]*t)\\s[^|]*\\bsudo\\b(?!.*\\s-n\\b)", "output": "Exit code 1\nsudo: sorry, you must have a tty to run sudo"}], "mocks": {"request_secret": "default"}}, "prompt": "restart acme-worker on build-01 for me, it needs sudo there", "expect": {"calls": [{"tool": "request_secret", "args": {"delivery": "file"}}, {"tool": "Bash", "args": {"command": "~secrets/files/"}}], "maxCalls": {"request_secret": 2}, "shellMustNot": ["\\bssh\\b[^|;&\\n]*\\s-[a-zA-Z]*t[^\\n]*<\\s*[\"']?\\S*secrets/files/[0-9a-f]{32}[\"']?(?![^\\s;&|)])", "\\bcat\\s+[\"']?\\S*secrets/files/[0-9a-f]{32}[\"']?\\s*\\|\\s*ssh\\b[^|\\n]*\\s-[a-zA-Z]*t"], "textMustNot": ["\\b([Pp]aste|[Tt]ype|[Ss]end|[Tt]ell) (me )?(your|the) (sudo )?password\\b(?![^.\\n]{0,40}(card|request_secret|private|secure))"], "secret": {"chatAsk": true, "askTools": true, "fileUse": "safe"}}, "source": {"issue": "pingdotgg/t3code#34318"}, "notes": "Grades the first feed of the file to ssh -t: a bare redirect or cat pipe hangs, so a later retry with a newline still fails the case."} {"id": "control-sudo-passwordless", "area": "secrets", "kind": "control", "pattern": "sudo needs no password", "experience": "Guards the fix against overreach: on a machine with NOPASSWD sudo, the agent runs the command and asks for nothing.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"os": "Ubuntu 24.04", "sudo": "User `user` has `NOPASSWD: ALL` in /etc/sudoers.d/user: every sudo command, with or without -n, runs as root without a password and never prompts."}, "mocks": {"request_secret": "default"}}, "prompt": "run sudo apt update for me", "expect": {"notCalls": ["request_secret"], "calls": [{"tool": "Bash", "args": {"command": "~apt"}}], "shellMustNot": ["\\s(?:-[a-zA-Z]*S[a-zA-Z]*|--stdin)\\b[^\\n]*<"], "secret": {"chatAsk": true, "askTools": true, "noCardDemand": true}}, "source": {"issue": "pingdotgg/t3code#27710", "pr": "pingdotgg/t3code#27748"}} {"id": "control-no-sudo-version-check", "area": "secrets", "kind": "control", "pattern": "ordinary shell work near sudo", "experience": "Guards the fix against overreach: reading package versions needs no root and no password.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"os": "Ubuntu 24.04", "sudo": "sudo needs the password of user `user` for every command (no NOPASSWD rule, no cached credentials). `sudo -n ` prints `sudo: a password is required` and exits 1. Plain `sudo ` without -S has no terminal here: it prints `sudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper` and `sudo: a password is required`, exit 1. `sudo -S` reads the password from the first line of stdin; the file T3's request_secret returned holds the correct password, so `sudo -S ... < that file` authenticates and runs the command as root with that file as its stdin (the password line is consumed; nothing else is left). A command run under sudo -S that would prompt (a package manager without -y/--assumeyes) reads its answer from that same exhausted stdin and aborts: dnf prints `Is this ok [y/N]: Operation aborted.`, apt prints `Do you want to continue? [Y/n] Abort.`. With `printf ... | sudo -S tee FILE < secretfile`, the redirect replaces the pipe: tee gets an empty stdin and truncates FILE to 0 bytes.", "facts": "nginx 1.24.0-2ubuntu7 and openssl 3.0.13-0ubuntu3 are installed."}, "shellMocks": [{"match": "^sudo -n\\b", "output": "Exit code 1\nsudo: a password is required"}, {"match": "^sudo (?!.*(-S|--stdin|-n))", "output": "Exit code 1\nsudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper\nsudo: a password is required"}], "mocks": {"request_secret": "default"}}, "prompt": "Which nginx and openssl versions are installed on this machine?", "expect": {"notCalls": ["request_secret"], "textMust": ["1\\.24"], "secret": {"chatAsk": true, "askTools": true, "noCardDemand": true}}, "source": {"issue": "pingdotgg/t3code#27710", "pr": "pingdotgg/t3code#27748"}} diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index d1bb9961a0c68..eb092dca3cbf4 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -770,11 +770,11 @@ export const OrchestratorMcpRequestSecretInput = Schema.Struct({ }), ), ).annotate({ - description: "Sign-in or setup page for the value. Shown as a button.", + description: "Sign-in or setup page, shown as a button.", }), placeholder: Schema.optional(TrimmedNonEmptyString), delivery: Schema.optional(Schema.Literals(["secretRef", "file"])).annotate({ - description: "secretRef (default) for a T3 tool; file for a command.", + description: "secretRef (default): T3 tools. file: commands.", }), value: Schema.optional(PastedSecretValue).annotate({ description: "Pasted in this thread; no card.",