diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 25894773..3dd7d2f4 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -29,6 +29,13 @@ import { WorkspaceToolError, } from './workspace.js'; +import { + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES, +} from './protocol.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + const execFileAsync = promisify(execFile); test('reads a bounded range from a registered local workspace', async (t) => { @@ -2018,3 +2025,106 @@ test('rejects empty, duplicate, and unknown sandbox workspace registration', asy ); } }); + + +test('validates sandbox workspace requests before either executor is invoked', async () => { + const commands: WorkspaceToolRequest[] = []; + const delegated: WorkspaceToolRequest[] = []; + const tools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'primary' }], + }, + async execute(request) { + delegated.push(request); + throw new Error('Unexpected delegation'); + }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + async execute(request) { + commands.push(request); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'pwd', + }; + const malformed: unknown[] = [ + null, undefined, [], 'execute_command', {}, + { ...command, protocolVersion: 2 }, + { ...command, workspaceId: '' }, + { ...command, operation: 'unknown' }, + { ...command, command: undefined }, + { ...command, command: 123 }, + { ...command, command: ' ' }, + { ...command, command: 'echo\0secret' }, + { ...command, command: '\ud800' }, + { ...command, command: 'a'.repeat(BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + 1) }, + { ...command, command: 'é'.repeat(BRIDGE_WORKSPACE_COMMAND_MAX_BYTES / 2 + 1) }, + { ...command, cwd: '../outside' }, + { ...command, cwd: '/tmp' }, + { ...command, env: { UNSAFE: 'value' } }, + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: '../outside' }, + ]; + for (const [field, maximum] of [ + ['timeoutMs', BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS], + ['maxOutputBytes', BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES], + ] as const) { + for (const value of [0, -1, 1.5, NaN, Infinity, '1', null, maximum + 1]) { + malformed.push({ ...command, [field]: value }); + } + } + for (const request of malformed) { + await assert.rejects( + tools.execute(request as WorkspaceToolRequest), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'INVALID_REQUEST' && + error.mutationMayHaveCommitted === false, + ); + assert.deepEqual(commands, []); + assert.deepEqual(delegated, []); + } + for (const request of [ + command, + { ...command, timeoutMs: 1, maxOutputBytes: 1 }, + { + ...command, + command: 'é'.repeat(BRIDGE_WORKSPACE_COMMAND_MAX_BYTES / 2), + cwd: 'src', + timeoutMs: BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + maxOutputBytes: BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES, + }, + ]) { + await tools.execute(request); + assert.equal(commands.at(-1), request); + } + assert.equal(commands.length, 3); + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + tools.execute(command, controller.signal), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + error.mutationMayHaveCommitted === false, + ); + assert.equal(commands.length, 3); + assert.deepEqual(delegated, []); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 0fa3c8ba..5302593a 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1663,6 +1663,12 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { request: WorkspaceToolRequest, signal?: AbortSignal, ): Promise { + if (!isWorkspaceToolRequest(request)) { + throw new WorkspaceToolError( + 'Invalid workspace tool request', + 'INVALID_REQUEST', + ); + } if (request.operation !== 'execute_command') { return this.options.workspaceTools.execute(request, signal); } diff --git a/service/src/bridge/settlement-race.test.ts b/service/src/bridge/settlement-race.test.ts new file mode 100644 index 00000000..4de5d3f3 --- /dev/null +++ b/service/src/bridge/settlement-race.test.ts @@ -0,0 +1,272 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { randomUUID } from 'node:crypto'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import type * as t from '../types'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; + +function barrier(): { promise: Promise; release: () => void } { + let release!: () => void; + const promise = new Promise(resolve => { + release = resolve; + }); + return { promise, release }; +} + +// Optional real Redis run: use a test server. Keys are isolated per test and +// cleaned up by prefix, with separate dispatcher and worker connections. +const redisUrl = process.env.BRIDGE_TEST_REDIS_URL; +for (const backend of ['mock', 'redis'] as const) { + const suite = + backend === 'redis' && (redisUrl == null || redisUrl === '') + ? describe.skip + : describe; + suite(`settlement/close arbitration (${backend})`, () => { + let redis: Redis; + let workerRedis: Redis; + let admin: Redis | undefined; + let prefix: string; + let store: RedisBridgeStore; + let worker: RedisBridgeStore; + let originalEval: Redis['eval']; + let originalGet: Redis['get']; + const incarnationId = 'incarnation-settlement-race'; + const workerId = 'settlement-race'; + const markerPattern = + 'codeapi:bridge:v1:worker:settlement-race:workspace:*:quarantined'; + + beforeEach(() => { + prefix = `race-test:${randomUUID()}:`; + if (backend === 'redis') { + admin = new Redis(redisUrl!); + redis = new Redis(redisUrl!, { keyPrefix: prefix }); + workerRedis = new Redis(redisUrl!, { keyPrefix: prefix }); + } else { + redis = new RedisMock() as unknown as Redis; + workerRedis = redis; + } + store = new RedisBridgeStore(redis, 60, 100); + worker = new RedisBridgeStore(workerRedis, 60, 100); + originalEval = redis.eval.bind(redis) as Redis['eval']; + originalGet = redis.get.bind(redis) as Redis['get']; + }); + afterEach(async () => { + if (admin) { + const keys = await admin.keys(`${prefix}*`); + if (keys.length) await admin.del(...keys); + admin.disconnect(); + admin = undefined; + } else { + await redis.flushall(); + } + redis.disconnect(); + workerRedis.disconnect(); + }); + async function start(stateful = true): Promise<{ + controller: AbortController; + completion: Promise; + assignment: CodeBridgeAssignment; + finalizations: string[]; + }> { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: stateful, + sandboxProfile: 'nsjail', + runtimes: [], + }, + }); + const controller = new AbortController(); + const finalizations: string[] = []; + const completion = store.dispatch({ + workerId, + body: { language: 'bash' } as t.PayloadBody, + headers: {}, + ...(stateful ? { runtimeSessionId: 'race-workspace' } : {}), + deadlineAtMs: Date.now() + 10_000, + signal: controller.signal, + finalize: async settlement => { + finalizations.push(settlement.status); + return settlement; + }, + }); + void completion.catch(() => undefined); + const assignment = (await worker.lease(workerId, incarnationId, 1_000))!; + expect(assignment).toBeDefined(); + await worker.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + return { controller, completion, assignment, finalizations }; + } + function result(assignment: CodeBridgeAssignment): CodeBridgeSettlement { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled' as const, + result: { + language: 'bash', + version: '5.2', + session_id: 'race-result', + files: [], + }, + }; + } + async function markers(): Promise { + return admin + ? admin.keys(`${prefix}${markerPattern}`) + : redis.keys(markerPattern); + } + for (const stateful of [true, false]) { + test(`close rejects fulfillment already past preflight (stateful=${stateful})`, async () => { + const { controller, completion, assignment, finalizations } = + await start(stateful); + const entered = barrier(); + const resume = barrier(); + const workerEval = workerRedis.eval.bind(workerRedis); + workerRedis.eval = (async (...args: Parameters) => { + if ( + String(args[0]).includes( + 'local existing = redis.call(\'GET\', KEYS[2])', + ) + ) { + entered.release(); + await resume.promise; + } + return workerEval(...args); + }) as Redis['eval']; + const settling = worker.settle( + workerId, + assignment.assignmentId, + result(assignment), + ); + void settling.catch(() => undefined); + await entered.promise; + controller.abort(); + try { + await expect(completion).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + } finally { + resume.release(); + } + await expect(settling).rejects.toMatchObject({ + code: 'ASSIGNMENT_EXPIRED', + }); + expect( + await redis.get( + `codeapi:bridge:v1:assignment:${assignment.assignmentId}:settlement`, + ), + ).toBeNull(); + expect( + await redis.exists( + `codeapi:bridge:v1:assignment:${assignment.assignmentId}:deadline`, + ), + ).toBe(0); + expect(finalizations).toEqual([]); + expect(await markers()).toHaveLength(stateful ? 1 : 0); + if (stateful) { + await worker.settle(workerId, assignment.assignmentId, { + ...result(assignment), + status: 'rejected', + error: 'not executed', + }); + expect(await markers()).toHaveLength(0); + } + }); + } + test('settlement wins before close and commits despite caller abort', async () => { + const { controller, completion, assignment, finalizations } = + await start(); + const entered = barrier(); + const resume = barrier(); + redis.eval = (async (...args: Parameters) => { + if ( + String(args[0]).includes( + 'local settlement = redis.call(\'GET\', KEYS[2])', + ) + ) { + entered.release(); + await resume.promise; + } + return originalEval(...args); + }) as Redis['eval']; + controller.abort(); + await entered.promise; + try { + await worker.settle( + workerId, + assignment.assignmentId, + result(assignment), + ); + } finally { + resume.release(); + } + await expect(completion).resolves.toEqual(result(assignment)); + expect(finalizations).toEqual(['fulfilled']); + expect(await markers()).toHaveLength(0); + await expect( + worker.settle(workerId, assignment.assignmentId, result(assignment)), + ).resolves.toBeUndefined(); + }); + for (const failure of ['abort', 'timeout', 'error'] as const) { + test(`a poll ${failure} still closes fulfillment`, async () => { + const entered = barrier(); + let intercept = false; + redis.get = ((key: string) => { + if (intercept && key.endsWith(':settlement')) { + entered.release(); + return failure === 'error' + ? Promise.reject(new Error('poll unavailable')) + : new Promise(() => {}); + } + return originalGet(key); + }) as Redis['get']; + const { controller, completion, assignment } = await start(); + intercept = true; + await entered.promise; + if (failure === 'abort') controller.abort(); + const messages = { + abort: 'deadline', + error: 'poll unavailable', + timeout: 'poll timed out', + }; + await expect(completion).rejects.toThrow(messages[failure]); + redis.get = originalGet; + await expect( + worker.settle(workerId, assignment.assignmentId, result(assignment)), + ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + expect(await markers()).toHaveLength(1); + }); + } + test('an unconfirmed close preserves the workspace fence', async () => { + const { controller, completion } = await start(); + redis.eval = ((...args: Parameters) => { + if ( + String(args[0]).includes( + 'local settlement = redis.call(\'GET\', KEYS[2])', + ) + ) + return new Promise(() => {}); + return originalEval(...args); + }) as Redis['eval']; + controller.abort(); + await expect(completion).rejects.toThrow( + 'Bridge settlement close timed out', + ); + expect(await markers()).toHaveLength(1); + expect( + await redis.get('codeapi:bridge:v1:worker:settlement-race:lock'), + ).not.toBeNull(); + }); + }); +} diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index da3956d7..de302654 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -796,12 +796,7 @@ export class RedisBridgeStore { args.finalize == null ? settlement : await args.finalize(settlement, registration); - await this.commitPendingWorkspace( - assignment, - settlement, - args.deadlineAtMs, - args.signal, - ); + await this.commitPendingWorkspace(assignment, settlement); resultCommitted = true; return result; } catch (error) { @@ -1450,22 +1445,30 @@ export class RedisBridgeStore { deadlineAtMs: number, signal: AbortSignal, ): Promise { - while (!signal.aborted && Date.now() < deadlineAtMs) { - const raw = await boundedCommand( - this.redis.get(settlementKey(assignment.assignmentId)), - Math.max( - 1, - Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), - ), - 'Bridge settlement poll', - signal, - ); - if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement; - await delay(POLL_INTERVAL_MS, signal); + let pollError: unknown; + try { + while (!signal.aborted && Date.now() < deadlineAtMs) { + const raw = await boundedCommand( + this.redis.get(settlementKey(assignment.assignmentId)), + Math.max( + 1, + Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), + ), + 'Bridge settlement poll', + signal, + ); + if (raw != null) return JSON.parse(raw) as CodeBridgeSettlement; + await delay(POLL_INTERVAL_MS, signal); + } + } catch (error) { + // A failed/aborted poll does not cancel Redis work. Arbitrate with + // settlement before returning an error, even when the caller is gone. + pollError = error; } const closeKeys = [ assignmentKey(assignment.assignmentId), settlementKey(assignment.assignmentId), + assignmentDeadlineKey(assignment.assignmentId), ]; if (assignment.runtimeSessionId !== undefined) { closeKeys.push( @@ -1475,10 +1478,14 @@ export class RedisBridgeStore { ), ); } + // The deadline key is also the fulfillment gate checked by settle(). + // Keep acknowledged assignment metadata for late clean rejection recovery, + // but atomically revoke fulfillment when no settlement has won yet. const closeScript = [ 'local settlement = redis.call(\'GET\', KEYS[2])', 'if settlement then return settlement end', - 'if #KEYS == 3 and redis.call(\'GET\', KEYS[3]) == ARGV[1] then return nil end', + 'redis.call(\'DEL\', KEYS[3])', + 'if #KEYS == 4 and redis.call(\'GET\', KEYS[4]) == ARGV[1] then return nil end', 'redis.call(\'DEL\', KEYS[1])', 'return nil', ].join('\n'); @@ -1495,6 +1502,9 @@ export class RedisBridgeStore { if (finalSettlement != null) { return JSON.parse(String(finalSettlement)) as CodeBridgeSettlement; } + if (pollError != null && !signal.aborted && Date.now() < deadlineAtMs) { + throw pollError; + } throw new BridgeStoreError( 'ASSIGNMENT_EXPIRED', 'Bridge assignment exceeded its deadline', @@ -1619,8 +1629,6 @@ export class RedisBridgeStore { private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, - deadlineAtMs: number, - signal: AbortSignal, ): Promise { if ( assignment.runtimeSessionId === undefined || @@ -1646,12 +1654,10 @@ export class RedisBridgeStore { ), assignment.assignmentId, ), - Math.max( - 1, - Math.min(this.redisCommandTimeoutMs, deadlineAtMs - Date.now()), - ), + // Once settlement wins, caller cancellation must not prevent its + // workspace commit. Redis availability still has a bounded budget. + this.redisCommandTimeoutMs, 'Bridge workspace commit', - signal, ), ); if (committed !== 1) { diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index c1b9e2c9..b0121d2e 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from 'bun:test'; -import type { SandboxExecuteContext, SandboxTransportRequest } from './types'; +import type { SandboxBackendErrorCode, SandboxExecuteContext, SandboxTransportRequest } from './types'; +import { SandboxBackendError } from './types'; +import { publicExecutionFailure } from '../utils'; import type { RedisBridgeStore } from '../bridge/store'; import { BridgeStoreError } from '../bridge/store'; @@ -73,6 +75,63 @@ describe('RemoteBridgeSandboxBackend', () => { }); }); + const failures = { + WORKER_OFFLINE: ['BRIDGE_WORKER_OFFLINE', true, 503, 'Code environment is offline'], + WORKER_UNAUTHORIZED: ['BRIDGE_WORKER_UNAUTHORIZED', false, 403, 'Code environment is not authorized for this tenant'], + WORKER_BUSY: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], + ASSIGNMENT_EXPIRED: ['BRIDGE_DEADLINE_EXCEEDED', false, 504, 'Code environment execution timed out'], + ASSIGNMENT_FENCED: ['BRIDGE_ASSIGNMENT_FENCED', false, 409, 'Code environment assignment is fenced; inspect the execution before retrying'], + ASSIGNMENT_NOT_FOUND: ['BRIDGE_ASSIGNMENT_NOT_FOUND', false, 409, 'Code environment assignment is no longer available; inspect the execution before retrying'], + WORKER_FENCED: ['BRIDGE_WORKER_FENCED', false, 409, 'Code environment worker changed during execution; inspect the execution before retrying'], + WORKER_QUARANTINED: ['BRIDGE_WORKER_QUARANTINED', false, 409, 'Code environment is quarantined; recover the worker before retrying'], + WORKSPACE_QUARANTINED: ['BRIDGE_WORKSPACE_QUARANTINED', false, 409, 'Code environment workspace is quarantined; reset the workspace before retrying'], + WORKER_MISMATCH: ['BRIDGE_WORKER_MISMATCH', false, 409, 'Code environment does not support this execution; select a compatible worker'], + ASSIGNMENT_INVALID: ['BRIDGE_ASSIGNMENT_INVALID', false, 400, 'Code environment assignment is invalid'], + RESULT_INVALID: ['BRIDGE_RESULT_INVALID', false, 502, 'Code environment returned an invalid result'], + } satisfies Record; + + for (const [storeCode, [code, transient, status, message]] of Object.entries(failures)) { + test(`preserves ${storeCode} recovery through the backend and public response`, async () => { + const cause = new BridgeStoreError(storeCode as BridgeStoreError['code'], + 'worker vm-private at redis.internal\nprivate tenant-secret'); + const store = { + dispatch: async (): ReturnType => { throw cause; }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + const error: unknown = await backend.execute(request(), context()).catch((failure: unknown) => failure); + expect(error).toBeInstanceOf(SandboxBackendError); + if (!(error instanceof SandboxBackendError)) throw new Error('Expected backend failure'); + expect(error).toMatchObject({ code, transient, message: cause.message }); + expect(error.cause).toMatchObject({ message: cause.message }); + // The worker carries only the code/message through BullMQ, not transient. + const failure = publicExecutionFailure(new Error(`${error.code}: ${error.message}`)); + expect(failure).toEqual({ status, body: { error: code.toLowerCase(), message } }); + expect(JSON.stringify(failure)).not.toContain('vm-private'); + expect(JSON.stringify(failure)).not.toContain('redis.internal'); + expect(JSON.stringify(failure)).not.toContain('tenant-secret'); + }); + } + + test('preserves failures that are not bridge store errors', async () => { + const cause = new Error('result finalization failed'); + const backend = new RemoteBridgeSandboxBackend({ + dispatch: async (): ReturnType => { throw cause; }, + }, 'default-vm'); + await expect(backend.execute(request(), context())).rejects.toBe(cause); + }); + + test('keeps a rejected settlement non-transient', async () => { + const backend = new RemoteBridgeSandboxBackend({ + dispatch: async (): ReturnType => ({ + protocolVersion: 1, generation: 1, leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', status: 'rejected', error: 'sandbox rejected execution', + }), + }, 'default-vm'); + await expect(backend.execute(request(), context())).rejects.toMatchObject({ + code: 'BRIDGE_EXECUTION_FAILED', transient: false, + }); + }); + test('keeps an explicitly selected singleton on its unbound compatibility route', async () => { let dispatched: Parameters[0] | undefined; const store = { diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index b1a94ae0..719e7004 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -1,5 +1,6 @@ import type { SandboxBackend, + SandboxBackendErrorCode, SandboxExecuteContext, SandboxRawResponse, SandboxTransportRequest, @@ -11,6 +12,23 @@ import { bridgeStore } from '../bridge'; import { BridgeStoreError } from '../bridge/store'; import { SandboxBackendError } from './types'; +// Every store failure needs an explicit recovery classification. New store +// codes must not silently fall through to a retryable worker outage. +const bridgeErrorCodes = { + WORKER_OFFLINE: 'BRIDGE_WORKER_OFFLINE', + WORKER_UNAUTHORIZED: 'BRIDGE_WORKER_UNAUTHORIZED', + WORKER_BUSY: 'BRIDGE_WORKER_BUSY', + ASSIGNMENT_EXPIRED: 'BRIDGE_DEADLINE_EXCEEDED', + ASSIGNMENT_FENCED: 'BRIDGE_ASSIGNMENT_FENCED', + ASSIGNMENT_NOT_FOUND: 'BRIDGE_ASSIGNMENT_NOT_FOUND', + WORKER_FENCED: 'BRIDGE_WORKER_FENCED', + WORKER_QUARANTINED: 'BRIDGE_WORKER_QUARANTINED', + WORKSPACE_QUARANTINED: 'BRIDGE_WORKSPACE_QUARANTINED', + WORKER_MISMATCH: 'BRIDGE_WORKER_MISMATCH', + ASSIGNMENT_INVALID: 'BRIDGE_ASSIGNMENT_INVALID', + RESULT_INVALID: 'BRIDGE_RESULT_INVALID', +} satisfies Record; + export class RemoteBridgeSandboxBackend implements SandboxBackend { readonly name = 'remote-bridge' as const; @@ -63,32 +81,11 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { return settlement.result as SandboxRawResponse; } catch (error) { if (!(error instanceof BridgeStoreError)) throw error; - if (error.code === 'WORKER_UNAUTHORIZED') { - throw new SandboxBackendError( - 'BRIDGE_WORKER_UNAUTHORIZED', - error.message, - error, - ); - } - if (error.code === 'WORKER_BUSY') { - throw new SandboxBackendError( - 'BRIDGE_WORKER_BUSY', - error.message, - error, - ); - } - if (error.code === 'ASSIGNMENT_EXPIRED') { - throw new SandboxBackendError( - 'BRIDGE_DEADLINE_EXCEEDED', - error.message, - error, - ); - } throw new SandboxBackendError( - 'BRIDGE_WORKER_OFFLINE', + bridgeErrorCodes[error.code], error.message, error, - true, + error.code === 'WORKER_OFFLINE', ); } } diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index 96151dde..fbaa2d20 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -78,13 +78,21 @@ export type SandboxBackendErrorCode = | 'BRIDGE_WORKER_BUSY' | 'BRIDGE_EXECUTION_FAILED' | 'BRIDGE_DEADLINE_EXCEEDED' + | 'BRIDGE_ASSIGNMENT_FENCED' + | 'BRIDGE_ASSIGNMENT_NOT_FOUND' + | 'BRIDGE_WORKER_FENCED' + | 'BRIDGE_WORKER_QUARANTINED' + | 'BRIDGE_WORKSPACE_QUARANTINED' + | 'BRIDGE_WORKER_MISMATCH' + | 'BRIDGE_ASSIGNMENT_INVALID' + | 'BRIDGE_RESULT_INVALID' | 'MICROVM_LAUNCH_FAILED' | 'MICROVM_LAUNCH_THROTTLED' | 'MICROVM_UNHEALTHY' | 'MICROVM_FENCED' | 'MICROVM_DEADLINE_EXCEEDED'; -/** Lambda-only failure modes; the worker prefixes messages with the code so +/** Typed sandbox backend failure modes; the worker prefixes messages with the code so * the router can map them (e.g. RUNTIME_SESSION_BUSY -> 409). Axios errors * from the sandbox POST itself are rethrown raw by every backend. */ export class SandboxBackendError extends Error { diff --git a/service/src/utils.ts b/service/src/utils.ts index aae2d05e..6fec60b7 100644 --- a/service/src/utils.ts +++ b/service/src/utils.ts @@ -1,5 +1,66 @@ import axios from 'axios'; import type { AxiosError } from 'axios'; +import type { SandboxBackendErrorCode } from './sandbox-backend/types'; + +// Keep the public response exhaustive too: a new terminal backend code must +// not silently become an availability-related 503 after crossing BullMQ. +const bridgePublicFailures: Partial> = { + BRIDGE_WORKER_UNAUTHORIZED: { + status: 403, + message: 'Code environment is not authorized for this tenant', + }, + BRIDGE_WORKER_OFFLINE: { + status: 503, + message: 'Code environment is offline', + }, + BRIDGE_WORKER_BUSY: { + status: 409, + message: 'Code environment is busy', + }, + BRIDGE_EXECUTION_FAILED: { + status: 502, + message: 'Code environment execution failed', + }, + BRIDGE_DEADLINE_EXCEEDED: { + status: 504, + message: 'Code environment execution timed out', + }, + BRIDGE_ASSIGNMENT_FENCED: { + status: 409, + message: 'Code environment assignment is fenced; inspect the execution before retrying', + }, + BRIDGE_ASSIGNMENT_NOT_FOUND: { + status: 409, + message: 'Code environment assignment is no longer available; inspect the execution before retrying', + }, + BRIDGE_WORKER_FENCED: { + status: 409, + message: 'Code environment worker changed during execution; inspect the execution before retrying', + }, + BRIDGE_WORKER_QUARANTINED: { + status: 409, + message: 'Code environment is quarantined; recover the worker before retrying', + }, + BRIDGE_WORKSPACE_QUARANTINED: { + status: 409, + message: 'Code environment workspace is quarantined; reset the workspace before retrying', + }, + BRIDGE_WORKER_MISMATCH: { + status: 409, + message: 'Code environment does not support this execution; select a compatible worker', + }, + BRIDGE_ASSIGNMENT_INVALID: { + status: 400, + message: 'Code environment assignment is invalid', + }, + BRIDGE_RESULT_INVALID: { + status: 502, + message: 'Code environment returned an invalid result', + }, +} satisfies Record< + Extract, + { status: number; message: string } +>; export function applySystemReplacements(input: string): string { return input; @@ -135,13 +196,15 @@ export function publicExecutionFailure(error: unknown): { status: number; body: ); if (backendMatch) { const code = backendMatch[1]; + const bridgeFailure = bridgePublicFailures[code]; + if (bridgeFailure != null) { + return { + status: bridgeFailure.status, + body: { error: code.toLowerCase(), message: bridgeFailure.message }, + }; + } const statuses: Record = { RUNTIME_SESSION_BUSY: 409, - BRIDGE_WORKER_UNAUTHORIZED: 403, - BRIDGE_WORKER_OFFLINE: 503, - BRIDGE_WORKER_BUSY: 409, - BRIDGE_EXECUTION_FAILED: 502, - BRIDGE_DEADLINE_EXCEEDED: 504, SESSION_INPUT_TOO_LARGE: 413, SESSION_INPUT_UNAVAILABLE: 422, SESSION_INPUT_SOURCE_FAILED: 502, @@ -152,11 +215,6 @@ export function publicExecutionFailure(error: unknown): { status: number; body: const status = statuses[code] ?? (sessionInputFailure ? 500 : 503); const publicMessages: Record = { RUNTIME_SESSION_BUSY: 'Runtime session is busy', - BRIDGE_WORKER_UNAUTHORIZED: 'Code environment is not authorized for this tenant', - BRIDGE_WORKER_OFFLINE: 'Code environment is offline', - BRIDGE_WORKER_BUSY: 'Code environment is busy', - BRIDGE_EXECUTION_FAILED: 'Code environment execution failed', - BRIDGE_DEADLINE_EXCEEDED: 'Code environment execution timed out', MICROVM_LAUNCH_FAILED: 'Sandbox launch failed', MICROVM_LAUNCH_THROTTLED: 'Sandbox capacity is temporarily unavailable', MICROVM_UNHEALTHY: 'Sandbox runtime is unavailable',