diff --git a/packages/api/src/exe-vm-backend.ts b/packages/api/src/exe-vm-backend.ts index 210b965..6b1c32f 100644 --- a/packages/api/src/exe-vm-backend.ts +++ b/packages/api/src/exe-vm-backend.ts @@ -43,12 +43,12 @@ export function tagsFromInventory(inventory: unknown): string[] { return [...tags].sort((a, b) => a.localeCompare(b)); } -function inventoryHasOwnedVm(body: string, vm: string): boolean { +function inventoryHasOwnedVm(body: string, vm: string): boolean | null { try { const value: unknown = JSON.parse(body); const visit = (item: unknown): boolean => Array.isArray(item) ? item.some(visit) : !!item && typeof item === "object" && ((item as any).name === vm || (item as any).vm_name === vm) && (item as any).comment === `Factorize VM ${vm}` || !!item && typeof item === "object" && Object.values(item as Record).some(visit); return visit(value); - } catch { return false; } + } catch { return null; } } export class ExeVmBackend implements ExecutionBackend { @@ -110,26 +110,26 @@ export class ExeVmBackend implements ExecutionBackend { const vm = vmNameFor(`verify-${verificationId}`); const tags = [...new Set(this.connection.tags)].map(tag => `--tag=${shellAtom(tag)}`).join(" "); const command = `new --name=${shellAtom(vm)} --no-email --comment=${shellAtom(`Factorize VM ${vm}`)}${tags ? ` ${tags}` : ""} --image=${shellAtom(image)}`; - let created: BackendCommandResult | undefined, vmMayExist = false; + let created: BackendCommandResult | undefined, createError: Error | undefined; try { created = await this.api(command); } catch { // The HTTPS API has a 30-second timeout. Reconcile by the deterministic // Factorize name and ownership marker before reporting an ambiguous create. const inventory = await this.api("ls --json").catch(() => null); - vmMayExist = Boolean(inventory?.ok && inventoryHasOwnedVm(inventory.body, vm)); - if (!vmMayExist) throw new ExeImageVerificationError("timeout", "Image creation timed out and no owned VM was found during reconciliation. Retry verification after checking exe.dev."); + const owned = inventory?.ok ? inventoryHasOwnedVm(inventory.body, vm) : null; + if (owned !== true) createError = new ExeImageVerificationError("timeout", owned === false ? "Image creation timed out and no owned VM was found during reconciliation. Retry verification after checking exe.dev." : "Image creation timed out and VM inventory could not reconcile the result. Verification will try ownership-safe cleanup and report if the VM remains."); } - if (created?.ok) vmMayExist = true; - else if (created && !created.ok) { + if (created && !created.ok) { const inventory = await this.api("ls --json").catch(() => null); - vmMayExist = Boolean(inventory?.ok && inventoryHasOwnedVm(inventory.body, vm)); - if (!vmMayExist) { + const owned = inventory?.ok ? inventoryHasOwnedVm(inventory.body, vm) : null; + if (owned !== true) { const kind = created.status === 403 ? "permission" : [429, 507].includes(created.status) ? "capacity" : [408, 504].includes(created.status) ? "timeout" : created.status === 503 && /capacity/i.test(created.body) ? "capacity" : "image_pull"; const detail = this.validationFailure("Image creation or pull failed. Check that the image is public, its reference is correct, and exe.dev has VM capacity and new permission", created); - throw new ExeImageVerificationError(kind, detail.message); + createError = owned === false ? new ExeImageVerificationError(kind, detail.message) : new ExeImageVerificationError("timeout", `${detail.message}. VM inventory was unavailable or invalid; verifying ownership-safe cleanup.`); } } try { + if (createError) throw createError; await progress("checking_ssh"); let ssh = await this.api(`ssh ${shellAtom(vm)} ${shellAtom("true")}`).catch(() => null); for (let attempt = 1; (!ssh || !ssh.ok) && attempt < 6; attempt++) ssh = await this.api(`ssh ${shellAtom(vm)} ${shellAtom("true")}`).catch(() => null); @@ -249,25 +249,24 @@ export class ExeVmBackend implements ExecutionBackend { private async deleteVm(vm: string): Promise { if (!isOwnedVmName(vm)) throw new Error("Refusing to delete a VM that is not owned by Factorize"); - const listed = await this.api("ls --json"); - if (!listed.ok) return listed; + let listed: BackendCommandResult | undefined; + for (let attempt = 0; attempt < 3; attempt++) { + listed = await this.api("ls --json").catch(error => ({ body: String(error), status: 0, exitCode: null, ok: false, requestBody: "ls --json" })); + if (listed.ok) break; + } + if (!listed?.ok) return listed ?? { body: "VM inventory is unavailable", status: 503, exitCode: null, ok: false, requestBody: "ls --json" }; let inventory: unknown; try { inventory = JSON.parse(listed.body); } catch { return { ...listed, ok: false, status: 502, body: "exe.dev returned an invalid VM inventory" }; } const objects: Record[] = []; - const visit = (value: unknown): void => { - if (Array.isArray(value)) return value.forEach(visit); - if (!value || typeof value !== "object") return; - objects.push(value as Record); - Object.values(value).forEach(visit); - }; + const visit = (value: unknown): void => { if (Array.isArray(value)) return value.forEach(visit); if (!value || typeof value !== "object") return; objects.push(value as Record); Object.values(value).forEach(visit); }; visit(inventory); const found = objects.find(item => item.name === vm || item.vm_name === vm); if (!found) return { ...listed, ok: true, status: 404, body: "VM is already absent" }; if (found.comment !== `Factorize VM ${vm}`) return { ...listed, ok: false, status: 409, body: "VM ownership marker does not match" }; let result: BackendCommandResult | undefined; for (let attempt = 0; attempt < 3; attempt++) { - result = await this.api(`rm ${shellAtom(vm)}`); + result = await this.api(`rm ${shellAtom(vm)}`).catch(error => ({ body: String(error), status: 0, exitCode: null, ok: false, requestBody: `rm ${vm}` })); if (result.ok || result.status === 404) return { ...result, ok: true }; } return result!; diff --git a/packages/api/test/execution-backend.test.ts b/packages/api/test/execution-backend.test.ts index e19fa1a..1e8b4e9 100644 --- a/packages/api/test/execution-backend.test.ts +++ b/packages/api/test/execution-backend.test.ts @@ -132,6 +132,19 @@ describe("ExeVmBackend", () => { expect(commands.at(-1)).toBe("rm 'factorize-verify-verification-1'"); }); + it("attempts ownership-safe cleanup when failed creation cannot be reconciled immediately", async () => { + const commands: string[] = []; let inventoryReads = 0; + vi.stubGlobal("fetch", vi.fn(async (_url: string, init: RequestInit) => { + const command = String(init.body); commands.push(command); + if (command.startsWith("new ")) return new Response("create failed", { status: 500 }); + if (command === "ls --json" && inventoryReads++ === 0) return new Response("unavailable", { status: 503 }); + if (command === "ls --json") return new Response('{"vms":[{"name":"factorize-verify-verification-1","comment":"Factorize VM factorize-verify-verification-1"}]}', { headers: { "X-Exe-Exit": "0" } }); + return new Response("removed", { headers: { "X-Exe-Exit": "0" } }); + })); + await expect(new ExeVmBackend(connection).verifyImage("verification-1", "ghcr.io/acme/agent:stable", "codex")).rejects.toThrow("inventory was unavailable"); + expect(commands.at(-1)).toBe("rm 'factorize-verify-verification-1'"); + }); + it.each(["ssh", "agent"] as const)("distinguishes %s prerequisite failure and still cleans up", async failedCheck => { const commands: string[] = []; vi.stubGlobal("fetch", vi.fn(async (_url: string, init: RequestInit) => {