Add public VM images and explicit verification

packages/api-client/src/schema.d.ts

@@ -1158,6 +1158,52 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/api/v1/vm-image-verifications": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /**
+         * Verify a public VM image
+         * @description Starts a tenant-scoped background verification. It creates a temporary Factorize-owned VM, checks SSH and the selected harness prerequisites, then deletes the VM. It never executes a job prompt.
+         *
+         *     Requires the jobs:write scope.
+         *
+         *     Accepts a scoped bearer token or interactive owner session.
+         */
+        post: operations["post_api_v1_vm_image_verifications"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
+    "/api/v1/vm-image-verifications/{verificationId}": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /**
+         * Get VM image verification progress
+         * @description Requires the jobs:read scope.
+         *
+         *     Accepts a scoped bearer token or interactive owner session.
+         */
+        get: operations["get_api_v1_vm_image_verifications_verificationId"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/api/v1/jobs": {
         parameters: {
             query?: never;
@@ -1905,6 +1951,7 @@ export interface components {
             concurrencyLimit: number;
             executionConfigId: string;
             tags?: string[];
+            vmImage?: string;
             model?: string;
             effort?: string;
             /** @default [] */
@@ -4621,6 +4668,7 @@ export interface operations {
                             workspace?: string;
                             cwd?: string;
                             agentKind: string;
+                            image?: string;
                         };
                         executionConfigId: string;
                         agentKind: string;
@@ -4778,6 +4826,95 @@ export interface operations {
             503: components["responses"]["Error503"];
         };
     };
+    post_api_v1_vm_image_verifications: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody: {
+            content: {
+                "application/json": {
+                    connectionId: string;
+                    image: string;
+                    tags: string[];
+                    /** @enum {string} */
+                    agentKind: "codex" | "claude" | "pi";
+                };
+            };
+        };
+        responses: {
+            /** @description Verification operation */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": {
+                        /** Format: uuid */
+                        id: string;
+                        /** @enum {string} */
+                        status: "queued" | "creating" | "checking_ssh" | "checking_agent" | "cleaning_up" | "succeeded" | "failed" | "cleanup_failed";
+                        phase: string;
+                        errorKind: string | null;
+                        error: string | null;
+                        createdAt: string;
+                        updatedAt: string;
+                    };
+                };
+            };
+            400: components["responses"]["Error400"];
+            401: components["responses"]["Error401"];
+            403: components["responses"]["Error403"];
+            404: components["responses"]["Error404"];
+            409: components["responses"]["Error409"];
+            415: components["responses"]["Error415"];
+            429: components["responses"]["Error429"];
+            500: components["responses"]["Error500"];
+            503: components["responses"]["Error503"];
+        };
+    };
+    get_api_v1_vm_image_verifications_verificationId: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                verificationId: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Verification operation */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": {
+                        /** Format: uuid */
+                        id: string;
+                        /** @enum {string} */
+                        status: "queued" | "creating" | "checking_ssh" | "checking_agent" | "cleaning_up" | "succeeded" | "failed" | "cleanup_failed";
+                        phase: string;
+                        errorKind: string | null;
+                        error: string | null;
+                        createdAt: string;
+                        updatedAt: string;
+                    };
+                };
+            };
+            400: components["responses"]["Error400"];
+            401: components["responses"]["Error401"];
+            403: components["responses"]["Error403"];
+            404: components["responses"]["Error404"];
+            409: components["responses"]["Error409"];
+            429: components["responses"]["Error429"];
+            500: components["responses"]["Error500"];
+            503: components["responses"]["Error503"];
+        };
+    };
     get_api_v1_jobs: {
         parameters: {
             query?: never;
@@ -4811,6 +4948,7 @@ export interface operations {
                             workspace?: string;
                             cwd?: string;
                             agentKind: string;
+                            image?: string;
                         };
                         executionConfigId: string;
                         agentKind: string;
@@ -5004,6 +5142,7 @@ export interface operations {
                             workspace?: string;
                             cwd?: string;
                             agentKind: string;
+                            image?: string;
                         };
                         executionConfigId: string;
                         agentKind: string;
@@ -5369,6 +5508,7 @@ export interface operations {
                             workspace?: string;
                             cwd?: string;
                             agentKind: string;
+                            image?: string;
                         };
                         executionConfigId: string;
                         agentKind: string;
@@ -5547,6 +5687,7 @@ export interface operations {
                     concurrencyLimit?: number;
                     executionConfigId: string;
                     tags?: string[];
+                    vmImage?: string;
                     model?: string;
                     effort?: string;
                     /** @default [] */
@@ -5677,6 +5818,7 @@ export interface operations {
                             workspace?: string;
                             cwd?: string;
                             agentKind: string;
+                            image?: string;
                         };
                         executionConfigId: string;
                         agentKind: string;

packages/api/migrations/0024_vm_image_verifications.sql

@@ -0,0 +1,15 @@
+CREATE TABLE app.vm_image_verifications (
+  tenant_id uuid NOT NULL REFERENCES app.tenants(id) ON DELETE CASCADE,
+  id uuid NOT NULL,
+  job_id uuid,
+  status text NOT NULL CHECK (status IN ('queued','creating','checking_ssh','checking_agent','cleaning_up','succeeded','failed','cleanup_failed')),
+  phase text NOT NULL,
+  config jsonb NOT NULL,
+  error_kind text,
+  error text,
+  vm_name text,
+  created_at timestamptz NOT NULL DEFAULT now(),
+  updated_at timestamptz NOT NULL DEFAULT now(),
+  PRIMARY KEY (tenant_id,id)
+);
+CREATE INDEX vm_image_verifications_status_idx ON app.vm_image_verifications(status,updated_at);

packages/api/src/api-contract.ts

@@ -14,6 +14,8 @@ import {
   jobConditionsTestSchema,
   listRunsSchema,
   scopeSchema,
+  vmImageVerificationInput,
+  vmImageVerificationResponse,
 } from "./flow-schemas";
 import { traceReplaySchema } from "./trace-replay-schemas";
 import document from "./api-document.json";
@@ -661,6 +663,8 @@ export const API_ROUTES: RouteContract[] = [
   route("DELETE", "/api/v1/skills/{skillId}", "skills:write", { summary: "Delete an unused global skill", responses: { "200": { description: "Deleted" } } }, (service, { params, query }) => service.deleteSkill(params.skillId, query.expectedRevision), { query: z.object({ expectedRevision: z.coerce.number().int().positive() }) }),
   route("GET", "/api/v1/skills/{skillId}/jobs", "skills:read", { summary: "List jobs using a skill", responses: { "200": { description: "Jobs", content: { "application/json": { schema: jsonSchema(skillJobsResponse, "output") } } } } }, (service, { params }) => service.listSkillJobs(params.skillId)),
   route("POST", "/api/v1/jobs/{jobId}/project", "flows:write", { summary: "Move a job to a project", responses: { "200": { description: "Moved", content: { "application/json": { schema: jsonSchema(jobResponse, "output") } } } } }, (service, { params, body }) => service.moveJob(params.jobId, body.projectId, body.expectedUpdatedAt), { body: z.object({ projectId: z.guid(), expectedUpdatedAt: z.iso.datetime().optional() }) }),
+  route("POST", "/api/v1/vm-image-verifications", "flows:write", { summary: "Verify a public VM image", description: "Starts a tenant-scoped background verification. It creates a temporary Factorize-owned VM, checks SSH and the selected harness prerequisites, then deletes the VM. It never executes a job prompt.", responses: { "200": { description: "Verification operation", content: { "application/json": { schema: jsonSchema(vmImageVerificationResponse, "output") } } } } }, (service, { body }) => service.startVmImageVerification(body), { body: vmImageVerificationInput }),
+  route("GET", "/api/v1/vm-image-verifications/{verificationId}", "flows:read", { summary: "Get VM image verification progress", responses: { "200": { description: "Verification operation", content: { "application/json": { schema: jsonSchema(vmImageVerificationResponse, "output") } } } } }, (service, { params }) => service.getVmImageVerification(params.verificationId)),
   route(
     "GET",
     "/api/v1/jobs",

packages/api/src/exe-vm-backend.ts

@@ -24,6 +24,9 @@ function harnessCommand(harness: NonNullable<LaunchRequest["harness"]>): string
 }
 
 export class ExeIntegrationError extends Error {}
+export class ExeImageVerificationError extends ExeIntegrationError {
+  constructor(public kind: "image_pull" | "permission" | "capacity" | "timeout", message: string) { super(message); }
+}
 
 export interface ExePermissionTest { ok: boolean; missingPermissions: string[]; tags: string[]; checks: BackendCommandResult[] }
 export interface ExeAgentValidation { models: string[]; command: BackendCommandResult }
@@ -40,11 +43,19 @@ export function tagsFromInventory(inventory: unknown): string[] {
   return [...tags].sort((a, b) => a.localeCompare(b));
 }
 
+function inventoryHasOwnedVm(body: string, vm: string): boolean {
+  try {
+    const value: unknown = JSON.parse(body);
+    const visit = (item: unknown): boolean => Array.isArray(item) ? item.some(visit) : !!item && typeof item === "object" && ((item as any).name === vm || (item as any).vm_name === vm) && (item as any).comment === `Factorize VM ${vm}` || !!item && typeof item === "object" && Object.values(item as Record<string, unknown>).some(visit);
+    return visit(value);
+  } catch { return false; }
+}
+
 export class ExeVmBackend implements ExecutionBackend {
   readonly kind = "exe-vm";
   readonly capabilities = capabilities;
 
-  constructor(private readonly connection: Pick<ExeConnection, "apiToken" | "tags"> | ExeRunConnection) {}
+  constructor(private readonly connection: Pick<ExeConnection, "apiToken" | "tags"> & { image?: string } | ExeRunConnection) {}
 
   private async api(command: string): Promise<BackendCommandResult> {
     const response = await fetch(API, { method: "POST", headers: { Authorization: `Bearer ${this.connection.apiToken}`, "Content-Type": "text/plain" }, body: command });
@@ -95,14 +106,54 @@ export class ExeVmBackend implements ExecutionBackend {
     }
   }
 
+  async verifyImage(verificationId: string, image: string, agentKind: "codex" | "claude" | "pi", progress: (phase: string) => Promise<void> = async () => {}): Promise<void> {
+    const vm = vmNameFor(`verify-${verificationId}`);
+    const tags = [...new Set(this.connection.tags)].map(tag => `--tag=${shellAtom(tag)}`).join(" ");
+    const command = `new --name=${shellAtom(vm)} --no-email --comment=${shellAtom(`Factorize VM ${vm}`)}${tags ? ` ${tags}` : ""} --image=${shellAtom(image)}`;
+    let created: BackendCommandResult | undefined, vmMayExist = false;
+    try { created = await this.api(command); }
+    catch {
+      // The HTTPS API has a 30-second timeout. Reconcile by the deterministic
+      // Factorize name and ownership marker before reporting an ambiguous create.
+      const inventory = await this.api("ls --json").catch(() => null);
+      vmMayExist = Boolean(inventory?.ok && inventoryHasOwnedVm(inventory.body, vm));
+      if (!vmMayExist) throw new ExeImageVerificationError("timeout", "Image creation timed out and no owned VM was found during reconciliation. Retry verification after checking exe.dev.");
+    }
+    if (created?.ok) vmMayExist = true;
+    else if (created && !created.ok) {
+      const inventory = await this.api("ls --json").catch(() => null);
+      vmMayExist = Boolean(inventory?.ok && inventoryHasOwnedVm(inventory.body, vm));
+      if (!vmMayExist) {
+        const kind = created.status === 403 ? "permission" : [429, 507].includes(created.status) ? "capacity" : [408, 504].includes(created.status) ? "timeout" : created.status === 503 && /capacity/i.test(created.body) ? "capacity" : "image_pull";
+        const detail = this.validationFailure("Image creation or pull failed. Check that the image is public, its reference is correct, and exe.dev has VM capacity and new permission", created);
+        throw new ExeImageVerificationError(kind, detail.message);
+      }
+    }
+    try {
+      await progress("checking_ssh");
+      let ssh = await this.api(`ssh ${shellAtom(vm)} ${shellAtom("true")}`).catch(() => null);
+      for (let attempt = 1; (!ssh || !ssh.ok) && attempt < 6; attempt++) ssh = await this.api(`ssh ${shellAtom(vm)} ${shellAtom("true")}`).catch(() => null);
+      if (!ssh?.ok) throw new ExeIntegrationError(`ssh: VM was created from the image, but SSH access failed (${ssh?.status ?? "timeout"}). Ensure the image boots an exe.dev-compatible system with SSH enabled.`);
+      await progress("checking_agent");
+      const check = `command -v ${shellAtom(agentKind)} >/dev/null 2>&1 && command -v curl >/dev/null 2>&1 && command -v jq >/dev/null 2>&1`;
+      const agent = await this.api(`ssh ${shellAtom(vm)} ${shellAtom(check)}`).catch(() => null);
+      if (!agent?.ok) throw new ExeIntegrationError(`agent: SSH works, but the selected ${agentKind} harness prerequisites are missing. Use an image with ${agentKind}, curl, and jq installed.`);
+    } finally {
+      await progress("cleaning_up");
+      const removed = await this.deleteVm(vm);
+      if (!removed.ok) throw new ExeIntegrationError(`cleanup: Temporary VM ${vm} could not be deleted. It will be retried; remove it in exe.dev if it remains.`);
+    }
+  }
+
   async launch(request: LaunchRequest): Promise<LaunchReceipt> {
     const connection = this.connection as ExeRunConnection;
     if (!connection.agentKind) throw new Error("The job is missing its agent configuration");
     if (!request.harness) throw new Error("The run is missing its agent harness launch plan");
     if (request.traceSources) validateTraceSources(request.traceSources);
     const vm = vmNameFor(request.runId);
     const tags = [...new Set(this.connection.tags)].map(tag => `--tag=${shellAtom(tag)}`).join(" ");
-    const created = await this.api(`new --name=${shellAtom(vm)} --no-email --comment=${shellAtom(`Factorize VM ${vm}`)} ${tags}`.trim());
+    const image = connection.image;
+    const created = await this.api(`new --name=${shellAtom(vm)} --no-email --comment=${shellAtom(`Factorize VM ${vm}`)} ${tags}${image ? ` --image=${shellAtom(image)}` : ""}`.trim());
     if (!created.ok) {
       await this.deleteVm(vm).catch(() => undefined);
       throw new Error(`exe.dev VM creation failed (${created.status}): ${created.body.slice(0, 500)}`);

packages/api/src/exe.ts

@@ -33,6 +33,7 @@ export interface ExeRunConnection extends ExeConnection {
   model?: string;
   effort?: string;
   efforts?: string[];
+  image?: string;
 }
 
 /** Single shell atom; prevents user-controlled values from becoming syntax. */

packages/api/src/flow-schemas.ts

@@ -43,11 +43,19 @@ export const jobInputSchema = z.object({
   concurrencyLimit: z.number().int().min(1).max(50).default(1),
   executionConfigId: z.string().min(1),
   tags: z.array(z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/)).max(20).optional(),
+  vmImage: z.string().trim().max(512).regex(/^(?:(?:[a-z0-9]+(?:[._-][a-z0-9]+)*\/)?[a-z0-9]+(?:[._-][a-z0-9]+)*(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?(?:@sha256:[a-f0-9]{64})?)?$/, "Enter a public Docker/OCI image reference.").optional(),
   model: z.string().trim().max(120).optional(),
   effort: z.string().trim().max(40).optional(),
   triggers: z.array(triggerSchema).max(50).default([]),
   skillIds: z.array(z.guid()).max(50).optional(),
 }).strict();
+export const vmImageVerificationInput = z.object({
+  connectionId: z.string().min(1).max(200),
+  image: z.string().trim().min(1).max(512).regex(/^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*\/)?[a-z0-9]+(?:[._-][a-z0-9]+)*(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[A-Za-z0-9_][A-Za-z0-9_.-]{0,127})?(?:@sha256:[a-f0-9]{64})?$/),
+  tags: z.array(z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/)).max(20),
+  agentKind: z.enum(["codex", "claude", "pi"]),
+}).strict();
+export const vmImageVerificationResponse = z.object({ id: z.guid(), status: z.enum(["queued", "creating", "checking_ssh", "checking_agent", "cleaning_up", "succeeded", "failed", "cleanup_failed"]), phase: z.string(), errorKind: z.string().nullable(), error: z.string().nullable(), createdAt: z.string(), updatedAt: z.string() });
 export const jobIdSchema = z.object({ jobId: z.string().min(1) });
 export const manualInvocationSchema = z.object({
   prompt: z.string().max(50_000).default(""),