From 8f522824a302a621e94c8f3f75e74b538dad79f6 Mon Sep 17 00:00:00 2001 From: OmarAlex24 Date: Sun, 15 Nov 2026 19:37:38 +0000 Subject: [PATCH] fix(sandbox): background work an agent waits on keeps its sandbox on The keep-alive saw only thread statuses and updates, so a sandbox stopped while an agent waited on subagents, background commands or monitors, killing the agent before it woke. It now reads every thread's pending background work: work that holds a thread counts as busy, and work no agent waits on, such as a dev server, keeps the sandbox on up to the running limit. Each check logs how many threads had background work. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/sandbox/sandboxSetup.test.ts | 161 ++++++++++++++----- apps/server/src/sandbox/sandboxSetup.ts | 60 ++++--- docs/user/remote-access.md | 11 +- 3 files changed, 164 insertions(+), 68 deletions(-) diff --git a/apps/server/src/sandbox/sandboxSetup.test.ts b/apps/server/src/sandbox/sandboxSetup.test.ts index 4596e2d108925..df8d1a797dfdf 100644 --- a/apps/server/src/sandbox/sandboxSetup.test.ts +++ b/apps/server/src/sandbox/sandboxSetup.test.ts @@ -574,8 +574,8 @@ describe("sandboxKeepAliveScript", () => { const extendUrl = "https://boat.dev/api/v1/sandboxes/bx_abcdefgh"; // Stand-ins for what the official T3 Code, curl and systemd do on a sandbox. curl logs each - // call with the headers it read from files; the MCP server answers with the statuses in - // `threads` and the newest thread in `latest`, or not at all when `down` exists. + // call with the headers it read from files; the MCP server pages through the threads in + // `threads`, or does not answer when `down` exists. const stubs = (home: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -629,11 +629,12 @@ if (call.url.endsWith("/mcp")) { body = JSON.stringify({ jsonrpc: "2.0", id: 1, result: { protocolVersion: "2025-06-18" } }); if (head) fs.writeFileSync(head, "HTTP/1.1 200 OK\\r\\nMcp-Session-Id: sid-1\\r\\n\\r\\n"); } else if (data.includes("tools/call")) { - // Asked by status, the busy threads; otherwise the most recently updated one, if any. - const threads = JSON.parse(data).params.arguments.statuses === undefined - ? (fs.existsSync(home + "/latest") ? [JSON.parse(fs.readFileSync(home + "/latest", "utf8"))] : []) - : JSON.parse(fs.readFileSync(home + "/threads", "utf8")).map((status, index) => ({ threadId: "thread-" + index, status })); - body = JSON.stringify({ jsonrpc: "2.0", id: 2, result: { content: [{ type: "text", text: JSON.stringify({ threads }) }], structuredContent: { threads, total: threads.length }, isError: false } }); + // A page of the threads in \`threads\`, which the server lists newest-updated first. + const { cursor = 0, limit = 10 } = JSON.parse(data).params.arguments; + const all = fs.existsSync(home + "/threads") ? JSON.parse(fs.readFileSync(home + "/threads", "utf8")) : []; + const threads = all.slice(cursor, cursor + limit); + const result = { threads, nextCursor: cursor + limit < all.length ? cursor + limit : null, total: all.length }; + body = JSON.stringify({ jsonrpc: "2.0", id: 2, result: { content: [{ type: "text", text: JSON.stringify(result) }], structuredContent: result, isError: false } }); } else if (data.includes("notifications/initialized")) status = 202; } if (out) fs.writeFileSync(out, body); @@ -701,16 +702,16 @@ if (format) process.stdout.write(String(status)); `--user --quiet --collect --unit=t3-sandbox-keepalive sh ${dir}/loop.sh run\n`, ); - // A subagent counts: the server is asked for every busy thread, subagents included. - yield* fs.writeFileString(`${home}/threads`, JSON.stringify(["running"])); + // A subagent counts: the server is asked for every thread, subagents included. + yield* fs.writeFileString(`${home}/threads`, JSON.stringify([{ status: "running" }])); yield* once(home); const calls = yield* curlCalls(home); const listed = calls.find((call) => call.data?.includes("tools/call")); expect(JSON.parse(listed!.data!).params.arguments).toEqual({ allProjects: true, includeSubagents: true, - statuses: ["queued", "preparing", "starting", "running"], - limit: 1, + limit: 100, + cursor: 0, }); expect(listed!.headers).toContain(`Authorization: Bearer ${mcpToken}`); expect(listed!.headers).toContain("mcp-session-id: sid-1"); @@ -731,15 +732,11 @@ if (format) process.stdout.write(String(status)); expect(call.argv.join(" ")).not.toContain(mcpToken); } - // Idle, or waiting on the user (which the server filters out): Boat's deadline stands. - yield* fs.writeFileString(`${home}/threads`, JSON.stringify([])); - yield* once(home); - expect(yield* extends_(home)).toHaveLength(1); - - // A server that ignores the filter answers with any thread; only a busy one counts, and - // the CLI's own list decides. - yield* fs.writeFileString(`${home}/threads`, JSON.stringify(["completed"])); - yield* fs.writeFileString(`${home}/cli-threads`, '[{"status":"waiting"}]'); + // Idle, or waiting on the user: Boat's deadline stands. + yield* fs.writeFileString( + `${home}/threads`, + JSON.stringify([{ status: "waiting" }, { status: "completed" }]), + ); yield* once(home); expect(yield* extends_(home)).toHaveLength(1); @@ -780,27 +777,18 @@ if (format) process.stdout.write(String(status)); projectRoot: "/home/boat/work/hello", }), ); - yield* fs.writeFileString(`${home}/threads`, JSON.stringify([])); const lastCheck = (stamp: string) => fs.writeFileString(`${dir}/last-check`, `${stamp}\n`); // Nothing is busy now, but a turn ran from 18:03:54 to 18:07:22, after the 18:03:36 check. yield* lastCheck("20261113180336"); yield* fs.writeFileString( - `${home}/latest`, - JSON.stringify({ - threadId: "t", - status: "completed", - updatedAt: "2026-11-13T18:07:22.481Z", - }), + `${home}/threads`, + JSON.stringify([ + { threadId: "t", status: "completed", updatedAt: "2026-11-13T18:07:22.481Z" }, + { threadId: "u", status: "completed", updatedAt: "2026-11-13T17:00:00.000Z" }, + ]), ); yield* once(home); - const lists = (yield* curlCalls(home)).filter((call) => call.data?.includes("tools/call")); - // The newest thread by update, subagents included. - expect(JSON.parse(lists.at(-1)!.data!).params.arguments).toEqual({ - allProjects: true, - includeSubagents: true, - limit: 1, - }); expect(yield* extends_(home)).toEqual([ expect.objectContaining({ method: "PATCH", data: '{"ttlSeconds":2100}' }), ]); @@ -825,6 +813,100 @@ if (format) process.stdout.write(String(status)); ), ); + it.live("counts background work an agent will wake for, and a dev server to the limit", () => + withHome((home) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dir = `${home}/.t3/userdata/sandbox-keepalive`; + yield* stubs(home); + yield* runIn( + home, + sandboxKeepAliveScript({ + key, + extend: { method: "PATCH", url: extendUrl }, + ttlSeconds: 2100, + maxRunningMinutes: 60, + port: 3773, + projectRoot: "/home/boat/work/hello", + }), + ); + yield* fs.writeFileString(`${dir}/last-check`, "20261113180336\n"); + const log = fs.readFileString(`${dir}/log`).pipe(Effect.orElseSucceed(() => "")); + const startedAgo = (seconds: number) => + runIn(home, `echo $(( $(date +%s) - ${seconds} )) >"${dir}/started"`); + const old = "2026-11-13T17:00:00.000Z"; + const idle = (count: number) => + Array.from({ length: count }, (_, index) => ({ + threadId: `idle-${index}`, + status: "completed", + updatedAt: old, + })); + + // Turns ended long ago: one waits on a background test run, one on a subagent whose + // own thread has no runs. Both found past the first page of threads. + yield* fs.writeFileString( + `${home}/threads`, + JSON.stringify([ + ...idle(100), + { + threadId: "tests", + status: "completed", + updatedAt: old, + pendingBackgroundTasks: [{ taskId: "b1", kind: "command", awaited: true }], + }, + { + threadId: "parent", + status: "waiting", + updatedAt: old, + pendingBackgroundTasks: [{ taskId: "a1", kind: "subagent", childThreadId: "child" }], + }, + { threadId: "child", status: "idle", updatedAt: old }, + ]), + ); + yield* once(home); + expect(yield* extends_(home)).toEqual([ + expect.objectContaining({ method: "PATCH", data: '{"ttlSeconds":2100}' }), + ]); + expect(yield* log).toMatch(/ busy pending=2 extend 200\n$/); + + // A dev server no agent waits on runs on as it would on the user's computer, to the limit. + const devServer = JSON.stringify([ + { + threadId: "dev", + status: "completed", + updatedAt: old, + pendingBackgroundTasks: [{ taskId: "d1", kind: "command" }], + }, + ]); + yield* fs.writeFileString(`${home}/threads`, devServer); + yield* startedAgo(59 * 60); + yield* once(home); + expect(yield* extends_(home)).toHaveLength(2); + expect(yield* log).toMatch(/ background pending=1 extend 200\n$/); + yield* startedAgo(60 * 60); + yield* once(home); + expect(yield* extends_(home)).toHaveLength(2); + expect(yield* log).toMatch(/ background pending=1\n$/); + + // The CLI's list of top-level threads carries the same work where the MCP endpoint is down. + yield* fs.writeFileString(`${home}/down`, ""); + yield* fs.writeFileString( + `${home}/cli-threads`, + JSON.stringify([ + { + status: "waiting", + updatedAt: old, + pendingBackgroundTasks: [{ taskId: "a1", kind: "subagent" }], + }, + ]), + ); + yield* once(home); + expect(yield* extends_(home)).toHaveLength(3); + expect(yield* log).toMatch(/ busy pending=1 extend 200\n$/); + }), + ), + ); + it.live("keeps a sandbox up to its running limit while nothing says whether it works", () => withHome((home) => Effect.gen(function* () { @@ -904,18 +986,13 @@ if (format) process.stdout.write(String(status)); const lastCheck = (yield* fs.readFileString(`${dir}/last-check`)).trim(); expect(lastCheck >= stamp(before) && lastCheck <= stamp(after)).toBe(true); yield* fs.writeFileString(`${dir}/boat-auth`, `Authorization: Bearer ${key}\n`); - yield* fs.writeFileString(`${home}/threads`, JSON.stringify([])); yield* fs.writeFileString( - `${home}/latest`, - JSON.stringify({ - threadId: "t", - status: "completed", - updatedAt: iso(before - 60), - }), + `${home}/threads`, + JSON.stringify([{ threadId: "t", status: "completed", updatedAt: iso(before - 60) }]), ); yield* once(home); expect(yield* extends_(home)).toHaveLength(0); - expect(yield* fs.readFileString(`${dir}/log`)).toMatch(/ idle\n$/); + expect(yield* fs.readFileString(`${dir}/log`)).toMatch(/ idle pending=0\n$/); }), ), ); diff --git a/apps/server/src/sandbox/sandboxSetup.ts b/apps/server/src/sandbox/sandboxSetup.ts index e874b0aabfe96..720c8189b073a 100644 --- a/apps/server/src/sandbox/sandboxSetup.ts +++ b/apps/server/src/sandbox/sandboxSetup.ts @@ -718,13 +718,14 @@ const KEEP_ALIVE_DIR = `$HOME/.t3/userdata/sandbox-keepalive`; /** How often the sandbox checks for work; the provider's deadline is the idle window past a check. */ export const KEEP_ALIVE_INTERVAL_SECONDS = 300; -// A thread in one of these is working; `waiting` waits on the user, so it counts as idle. +// A thread in one of these is working; `waiting` waits on the user or on background work, which +// counts through the thread's pending background tasks instead. const BUSY_STATUSES = ["queued", "preparing", "starting", "running"] as const; -// A check extends when a thread is busy or was updated since the last check, so a short turn -// between two checks counts. The official server's MCP endpoint, through an `mcp-client` session -// the CLI mints, sees subagents; `t3 thread list` does not and is the fallback. With neither, the -// running limit applies. Tokens go in `curl -H @file`, never in argv. +// A check extends when a thread works, waits on its background work, or changed since the last +// check; unawaited work (a dev server) only up to the running limit, as unknown does. The official +// server's MCP endpoint, via an `mcp-client` session the CLI mints, sees subagents; `t3 thread +// list` is the fallback. Tokens go in `curl -H @file`, never in argv. const keepAliveLoop = (port: number) => `#!/bin/sh dir="${KEEP_ALIVE_DIR}" export PATH="$HOME/.local/bin:$PATH" @@ -766,17 +767,31 @@ worked_since() { case "$newest" in ''|*[!0-9]*) return 1 ;; esac [ "$newest" -ge "$since" ] } -# Prints busy, active or idle, or fails when the server cannot say. -mcp_ask() { - listed "$(mcp '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"t3_thread_list","arguments":{"allProjects":true,"includeSubagents":true,"statuses":${JSON.stringify(BUSY_STATUSES)},"limit":1}}}')" || return 1 - if ! grep -q '"threads":\\[\\]' "$dir/out"; then - # A server that ignores the status filter answers with any thread; only a busy one counts. - first="$(grep -o '"status":"[a-z_]*"' "$dir/out" | head -n 1 | cut -d'"' -f4)" - case "$first" in ${BUSY_STATUSES.join("|")}) echo busy; return 0 ;; *) return 1 ;; esac - fi - # The server lists threads newest-updated first. - listed "$(mcp '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"t3_thread_list","arguments":{"allProjects":true,"includeSubagents":true,"limit":1}}}')" || return 1 - if worked_since; then echo active; else echo idle; fi +# Prints busy, active, background or idle for the threads in $dir/out, with how many carry +# background work. A subagent or background task holds its thread; a command or monitor only when +# the agent awaits it. +judge() { + pending="$(grep -o '"pendingBackgroundTasks":\\[{' "$dir/out" | wc -l | tr -d ' ')" + if grep -Eq '"status":"(${BUSY_STATUSES.join("|")})"|"kind":"(subagent|background_task)"|"awaited":true' "$dir/out"; then + state=busy + elif worked_since; then state=active + elif [ "$pending" -gt 0 ]; then state=background + else state=idle; fi + echo "$state pending=$pending" +} +# Lists every thread into $dir/out, subagents included. The page cap stops a server that ignores +# the cursor. +mcp_list() { + : >"$dir/all" + cursor=0 + pages=0 + while [ -n "$cursor" ] && [ "$pages" -lt 20 ]; do + listed "$(mcp '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"t3_thread_list","arguments":{"allProjects":true,"includeSubagents":true,"limit":100,"cursor":'"$cursor"'}}}')" || return 1 + cat "$dir/out" >>"$dir/all" + cursor="$(grep -o '"nextCursor":[0-9]*' "$dir/out" | head -n 1 | cut -d: -f2)" + pages=$((pages + 1)) + done + mv "$dir/all" "$dir/out" } mcp_state() { [ -s "$dir/mcp-auth" ] || mint_session || return 1 @@ -792,16 +807,15 @@ mcp_state() { version="$(grep -o '"protocolVersion":"[^"]*"' "$dir/out" | head -n 1 | cut -d'"' -f4)" printf 'mcp-session-id: %s\\nmcp-protocol-version: %s\\n' "$sid" "\${version:-2025-06-18}" >"$dir/mcp-headers" mcp '{"jsonrpc":"2.0","method":"notifications/initialized"}' >/dev/null - state="$(mcp_ask)" || state="" + state="" + mcp_list && state="$(judge)" [ -n "$sid" ] && curl -sS -m 10 -o /dev/null -X DELETE -H @"$dir/mcp-auth" -H @"$dir/mcp-headers" "$origin/mcp" >/dev/null 2>&1 [ -n "$state" ] && echo "$state" } cli_state() { [ -n "\${PROJECT_ROOT:-}" ] || return 1 t3 thread list --project "$PROJECT_ROOT" --json >"$dir/out" 2>/dev/null || return 1 - if grep -Eq '"status":"(${BUSY_STATUSES.join("|")})"' "$dir/out"; then echo busy - elif worked_since; then echo active - else echo idle; fi + judge } extend() { curl -sS -m 30 -o /dev/null -w '%{http_code}' -X "$EXTEND_METHOD" -H @"$dir/boat-auth" \\ @@ -818,15 +832,15 @@ tick() { . "$dir/config" now="$(date -u +%Y%m%d%H%M%S)" state="$(mcp_state || cli_state || echo unknown)" - rm -f "$dir/out" "$dir/head" + rm -f "$dir/out" "$dir/head" "$dir/all" # A T3 that answers neither way (a changed CLI, say) must not stop a sandbox that may be working, # so it stays up as it would have before keep-alive: until the running limit. if [ "$state" = unknown ]; then if within_running_limit; then note "unknown extend $(extend)"; else note unknown; fi return 0 fi - if [ "$state" = idle ]; then - note idle + if [ "\${state%% *}" = idle ] || { [ "\${state%% *}" = background ] && ! within_running_limit; }; then + note "$state" echo "$now" >"$dir/last-check" return 0 fi diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index b44cec69848ec..c3aa71016b111 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -231,9 +231,14 @@ A thread's Boat sandbox has no running limit while its agents work. It stops on have been idle for the config's **Stop when idle** time (30 minutes by default). - Every 5 minutes the sandbox checks whether any thread on it, subagents included, is queued, - starting or running, or has had any activity since the last check, such as a short turn or your - answer to a question. If so, it moves Boat's auto-stop to the idle time plus 5 minutes from now. - A thread left waiting on you, for an approval or an answer, counts as idle. + starting or running, is waiting on background work it will pick up again (a subagent, or a + command or monitor it waits on), or has had any activity since the last check, such as a short + turn or your answer to a question. If so, it moves Boat's auto-stop to the idle time plus 5 + minutes from now. A thread left waiting on you, for an approval or an answer, counts as idle. +- Background commands no agent waits on, such as a dev server, keep the sandbox on too, but only + for 8 hours after the sandbox last started. + The check log in `~/.t3/userdata/sandbox-keepalive/log` shows each decision and how many threads + had background work. - The sandbox does this itself, so it keeps working while this machine sleeps or the app is closed. With no work, nothing moves the auto-stop, and Boat stops the sandbox when it passes. Its disk stays, and sending a message resumes it.