diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index e6350a59c0..5703a07cfb 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -27,6 +27,7 @@ import { STATUS_NAMES, type WatchEvent, } from './client.js'; +import { ErrorInfoSchema, RetryInfoSchema } from './gen/google/rpc/error_details_pb.js'; import { OpenShell, ServiceAuthorizationMode as ProtoServiceAuthorizationMode, @@ -701,6 +702,30 @@ describe('watchLogs', () => { }); describe('create', () => { + it.each([ + ['MUTATION_LOCK_TIMEOUT', 3], + ['OTHER', 1], + ])('retries %s up to %i attempts', async (reason, attempts) => { + let calls = 0; + const sandbox = client({ + createSandbox: () => { + calls += 1; + if (calls < 3) { + throw new ConnectError('busy', Code.Unavailable, undefined, [ + { desc: ErrorInfoSchema, value: { reason } }, + { desc: RetryInfoSchema, value: { retryDelay: { nanos: 1_000_000 } } }, + ]); + } + return readySandbox('sb', 'sb-id'); + }, + }); + + const result = sandbox.create({ image: 'img' }); + if (attempts === 1) await expect(result).rejects.toMatchObject({ connectCode: Code.Unavailable }); + else await expect(result).resolves.toMatchObject({ name: 'sb' }); + expect(calls).toBe(attempts); + }); + it('accepts the existing SandboxRef shape without explicit access-path maps', () => { const ref: SandboxRef = { id: 'sb-id', diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index 89fa002be6..a493c1cd72 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -27,6 +27,7 @@ import type { PlatformEvent as WirePlatformEvent, } from './gen/openshell_pb.js'; import { + type CreateSandboxRequestSchema, type ExecSandboxInputSchema, OpenShell, ServiceAuthorizationMode as ProtoServiceAuthorizationMode, @@ -842,6 +843,12 @@ export function __setWatchLogsBackoffSleepForTests( watchLogsBackoffSleep = fn ?? defaultWatchLogsBackoffSleep; } +const CREATE_RETRY_MS = 60_000; + +function isLockTimeout(err: SdkError): boolean { + return err.connectCode === Code.Unavailable && err.errorInfo?.reason === 'MUTATION_LOCK_TIMEOUT'; +} + // Only Unavailable (connection drop, gateway restart) is retryable. Every // other status — including OUT_OF_RANGE — is terminal so the caller sees it // instead of the loop silently restarting from an empty cursor and hiding a gap. @@ -1230,6 +1237,21 @@ export class SandboxClient { return new SandboxClient(buildTransport(options)); } + // CreateSandbox, retried for up to 60s on MUTATION_LOCK_TIMEOUT. + private async createSandbox(request: MessageInitShape) { + const deadline = Date.now() + CREATE_RETRY_MS; + for (;;) { + try { + return await this.grpc.createSandbox(request); + } catch (e) { + const err = fromConnect(e); + const wait = (err.retryDelayMs ?? 1000) * (1 + Math.random() / 2); + if (!isLockTimeout(err) || Date.now() + wait > deadline) throw err; + await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + } + async create(spec: SandboxSpec): Promise { try { // Curated fields build the base spec; rawSpec then shallow-overrides at @@ -1248,7 +1270,7 @@ export class SandboxClient { }; if (spec.rawSpec) Object.assign(specInit, spec.rawSpec); - const resp = await this.grpc.createSandbox({ + const resp = await this.createSandbox({ workspaceScope: workspaceScope(spec), name: spec.name ?? '', labels: spec.labels ?? {}, @@ -1269,7 +1291,7 @@ export class SandboxClient { async createFromTemplate(spec: SandboxFromTemplateSpec): Promise { if (spec.workloadTemplate.trim() === '') throw new SdkError('invalid_config', 'workloadTemplate is required'); try { - const resp = await this.grpc.createSandbox({ + const resp = await this.createSandbox({ workspaceScope: workspaceScope(spec), name: spec.name ?? '', labels: spec.labels ?? {},