diff --git a/packages/api-client/src/schema.d.ts b/packages/api-client/src/schema.d.ts index 9385438..5f7cd2c 100644 --- a/packages/api-client/src/schema.d.ts +++ b/packages/api-client/src/schema.d.ts @@ -1550,6 +1550,30 @@ export interface paths { patch: operations["patch_api_v1_runs_runId"]; trace?: never; }; + "/api/v1/runs/{runId}/report-error": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Report an error for an active run + * @description Records a sanitized reason and authenticated reporter without changing run execution or releasing its concurrency slot. The agent may continue work and should then exit normally. An identical retry succeeds; a different replacement reason returns 409. Terminal runs reject reports. + * + * Requires the runs:write scope. + * + * Accepts a scoped bearer token or interactive owner session. + */ + post: operations["post_api_v1_runs_runId_report_error"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/runs/{runId}/trace": { parameters: { query?: never; @@ -1697,6 +1721,17 @@ export interface components { created_at: string; updated_at: string; started_at: string | null; + failure_source: ("system" | "agent") | null; + agent_error: { + reason: string; + reported_at: string; + reporter: { + user_id: string; + /** @enum {string} */ + kind: "api_key" | "oauth" | "session"; + id: string; + }; + } | null; elapsed_time?: ({ duration: number; } | { @@ -1732,6 +1767,17 @@ export interface components { created_at: string; updated_at: string; started_at: string | null; + failure_source: ("system" | "agent") | null; + agent_error: { + reason: string; + reported_at: string; + reporter: { + user_id: string; + /** @enum {string} */ + kind: "api_key" | "oauth" | "session"; + id: string; + }; + } | null; elapsed_time?: ({ duration: number; } | { @@ -6206,6 +6252,17 @@ export interface operations { artifact_state: "pending" | "collecting" | "stored" | "partial" | "failed"; finalizing: boolean; trace_revision: string; + failure_source: ("system" | "agent") | null; + agent_error: { + reason: string; + reported_at: string; + reporter: { + user_id: string; + /** @enum {string} */ + kind: "api_key" | "oauth" | "session"; + id: string; + }; + } | null; }; }; }; @@ -6346,6 +6403,52 @@ export interface operations { 503: components["responses"]["Error503"]; }; }; + post_api_v1_runs_runId_report_error: { + parameters: { + query?: never; + header?: never; + path: { + runId: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + reason: string; + }; + }; + }; + responses: { + /** @description Report saved or identical retry acknowledged */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @constant */ + reported: true; + reason: string; + retry: boolean; + }; + }; + }; + /** @description Invalid reason or request body */ + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + /** @description Missing runs:write authorization */ + 403: components["responses"]["Error403"]; + /** @description Run not found within the tenant */ + 404: components["responses"]["Error404"]; + /** @description Run is terminal or a different reason was already reported */ + 409: components["responses"]["Error409"]; + 415: components["responses"]["Error415"]; + 429: components["responses"]["Error429"]; + 500: components["responses"]["Error500"]; + 503: components["responses"]["Error503"]; + }; + }; get_api_v1_runs_runId_trace: { parameters: { query?: { diff --git a/packages/api/migrations/0025_agent_run_error.sql b/packages/api/migrations/0025_agent_run_error.sql new file mode 100644 index 0000000..a79fe92 --- /dev/null +++ b/packages/api/migrations/0025_agent_run_error.sql @@ -0,0 +1,17 @@ +ALTER TABLE app.runs + ADD COLUMN failure_source text, + ADD COLUMN agent_error_reason text, + ADD COLUMN agent_error_reported_at timestamptz, + ADD COLUMN agent_error_reporter_user_id uuid, + ADD COLUMN agent_error_reporter_kind text, + ADD COLUMN agent_error_reporter_id text; + +UPDATE app.runs SET failure_source='system' WHERE state='failed'; + +ALTER TABLE app.runs ADD CONSTRAINT runs_failure_source_check + CHECK (failure_source IS NULL OR failure_source IN ('system','agent')); +ALTER TABLE app.runs ADD CONSTRAINT runs_failure_source_state_check + CHECK (failure_source IS NULL OR state='failed'); +ALTER TABLE app.runs ADD CONSTRAINT runs_agent_error_report_check + CHECK ((agent_error_reason IS NULL AND agent_error_reported_at IS NULL AND agent_error_reporter_user_id IS NULL AND agent_error_reporter_kind IS NULL AND agent_error_reporter_id IS NULL) + OR (agent_error_reason IS NOT NULL AND agent_error_reported_at IS NOT NULL AND agent_error_reporter_user_id IS NOT NULL AND agent_error_reporter_kind IS NOT NULL AND agent_error_reporter_id IS NOT NULL)); diff --git a/packages/api/src/api-contract.ts b/packages/api/src/api-contract.ts index 5bf8f39..2581278 100644 --- a/packages/api/src/api-contract.ts +++ b/packages/api/src/api-contract.ts @@ -12,6 +12,8 @@ import { jobInputSchema, manualInvocationSchema, renameRunSchema, + reportRunErrorSchema, + reportRunErrorResponse, jobConditionsTestSchema, listRunsSchema, scopeSchema, @@ -155,6 +157,7 @@ function route( // Legacy route labels are translated by resource and effect; they are never // accepted as newly issued scopes. function scopeForRoute(path: string, legacy: string): string | string[] { + if (legacy === "runs:write" && path.endsWith("/report-error")) return "runs:write"; if (legacy === "runs:write" && path === "/api/v1/runs/{runId}" ) return "runs:write"; if (legacy === "runs:read" && path === "/api/v1/search") return "jobs:read+runs:read"; if (legacy === "flows:read") { @@ -894,6 +897,17 @@ export const API_ROUTES: RouteContract[] = [ description: "Changes only the run display name. Supply expectedName for optimistic concurrency; omission supports last-writer-wins clients.", responses: { "200": { description: "Saved run display name", content: { "application/json": { schema: { type: "object", required: ["run_name"], properties: { run_name: { type: "string" } }, additionalProperties: false } } } }, "404": { description: "Run not found within the tenant" }, "409": { description: "stale_run_name: reload before retrying" } }, }, (service, { params, body }) => service.renameRun(params.runId, body), { body: renameRunSchema, parameters: z.object({ runId: z.uuid() }) }), + route("POST", "/api/v1/runs/{runId}/report-error", "runs:write", { + summary: "Report an error for an active run", + description: "Records a sanitized reason and authenticated reporter without changing run execution or releasing its concurrency slot. The agent may continue work and should then exit normally. An identical retry succeeds; a different replacement reason returns 409. Terminal runs reject reports.", + responses: { + "200": { description: "Report saved or identical retry acknowledged", content: { "application/json": { schema: jsonSchema(reportRunErrorResponse, "output") } } }, + "400": { description: "Invalid reason or request body" }, + "403": { description: "Missing runs:write authorization" }, + "404": { description: "Run not found within the tenant" }, + "409": { description: "Run is terminal or a different reason was already reported" }, + }, + }, (service, { params, body }) => service.reportRunError(params.runId, body), { body: reportRunErrorSchema, maxBodyBytes: 4096, parameters: z.object({ runId: z.uuid() }) }), route( "GET", "/api/v1/runs/{runId}/trace", diff --git a/packages/api/src/device-oauth.ts b/packages/api/src/device-oauth.ts index bdfc880..ca86273 100644 --- a/packages/api/src/device-oauth.ts +++ b/packages/api/src/device-oauth.ts @@ -225,7 +225,7 @@ export async function decideDevice(env: Env, oauth: OAuthHelpers, session: { ten const authorize = async (record: DeviceRecord | null, write: (record: DeviceRecord) => Promise) => { if (!record || record.status !== "pending" || record.expiresAt <= Math.floor(Date.now() / 1000)) return null; if (decision === "deny") { record.status = "denied"; await write(record); return { status: "denied" as const }; } - const props: OAuthProps = { tenantId: session.tenantId, userId: session.userId, sessionVersion: session.sessionVersion, scopes: record.scope, authMethod: "oauth" }; + const props: OAuthProps = { tenantId: session.tenantId, userId: session.userId, sessionVersion: session.sessionVersion, scopes: record.scope, authMethod: "oauth" }; const authRequest: AuthRequest = { responseType: "code", clientId: record.clientId, redirectUri: record.redirectUri, scope: record.scope, state: "", codeChallenge: record.codeChallenge, codeChallengeMethod: "S256", resource: record.resource, issuer: env.APP_ORIGIN }; const result = await oauth.completeAuthorization({ request: authRequest, userId: session.userId, metadata: { tenantId: session.tenantId }, scope: record.scope, props, revokeExistingGrants: false }); record.authorizationCode = new URL(result.redirectTo).searchParams.get("code") ?? undefined; diff --git a/packages/api/src/flow-schemas.ts b/packages/api/src/flow-schemas.ts index 7b544b8..4ca7753 100644 --- a/packages/api/src/flow-schemas.ts +++ b/packages/api/src/flow-schemas.ts @@ -16,6 +16,8 @@ export const listRunsSchema = z.object({ }).refine(value => !value.direction || value.sort, { message: "direction requires sort" }); export const runIdSchema = z.object({ runId: z.string().min(1) }); export const renameRunSchema = z.object({ name: z.string().trim().min(1).max(120), expectedName: z.string().max(120).optional() }).strict(); +export const reportRunErrorSchema = z.object({ reason: z.string().trim().min(1).max(2000) }).strict(); +export const reportRunErrorResponse = z.object({ reported: z.literal(true), reason: z.string(), retry: z.boolean() }); const webhookCommonSchema = z.object({ conditions: conditionsSchema.optional() }); export const webhookTriggerConfigSchema = z.discriminatedUnion("provider", [ diff --git a/packages/api/src/flow-service.ts b/packages/api/src/flow-service.ts index 4aa7d57..4755a33 100644 --- a/packages/api/src/flow-service.ts +++ b/packages/api/src/flow-service.ts @@ -5,7 +5,7 @@ import { z } from "zod"; import { runPageResponse, fullRunResponse } from "./run-read-contracts"; import { triggerContextCatalog } from "./trigger-context"; import type { Env, OAuthProps } from "./types"; -import { canonicalizeVmImage, renameRunSchema, vmImageVerificationInput, type JobInput, type ManualInvocationInput } from "./flow-schemas"; +import { canonicalizeVmImage, renameRunSchema, reportRunErrorSchema, vmImageVerificationInput, type JobInput, type ManualInvocationInput } from "./flow-schemas"; import { nextOccurrence, validateScheduleConfig } from "./schedule"; import { databaseFor } from "./postgres/database"; import { IdentityRepository } from "./postgres/identity-repository"; @@ -35,6 +35,8 @@ import { ProjectRepository } from "./postgres/project-repository"; import { effortOptions, selectableExeTags, type ExeConnection } from "./exe"; import { ExeImageVerificationError, ExeVmBackend } from "./exe-vm-backend"; import { SkillRepository, SkillConflict, SkillReferenced } from "./postgres/skill-repository"; +import { RunRepository } from "./postgres/run-repository"; +import { safeDiagnosticText } from "./harness-diagnostics"; import { skillInput, skillUpdateInput, skillPage, skillResponse, skillJobsResponse, previewSkillInput, composePrompt } from "./skills"; export class ServiceError extends Error { @@ -157,6 +159,19 @@ export class ApiService { async search(query: string) { await this.authorize("jobs:read"); await this.authorize("runs:read"); return publicValue(searchResponse, await new OperationsRepository(databaseFor(this.env), this.auth.tenantId).search(query)); } async triggerContextMetadata() { await this.authorize("jobs:read"); return triggerContextCatalog; } async getRunStatus(runId: string) { await this.authorize("runs:read"); const status = await new RunQueryRepository(databaseFor(this.env), this.auth.tenantId).status(runId); if (!status) throw new ServiceError(404, "not_found", "Run not found"); return status; } + async reportRunError(runId: string, input: unknown) { + await this.authorize("runs:write"); + const parsed = reportRunErrorSchema.parse(input), reason = safeDiagnosticText(parsed.reason, 1000); + if (!reason.trim()) throw new ServiceError(400, "invalid_request", "Reason must contain safe text."); + const reporterKind = this.auth.authMethod === "oauth" ? "oauth" : this.auth.authMethod === "access_token" ? "api_key" : "session"; + const reporterId = reporterKind === "oauth" ? this.auth.oauthGrantId && this.auth.oauthClientId ? `${this.auth.oauthClientId}:${this.auth.oauthGrantId}` : undefined : reporterKind === "api_key" ? this.auth.accessTokenId : this.auth.userId; + if (!reporterId) throw new ServiceError(403, "invalid_token", "Authenticated reporter identity is unavailable."); + const result = await new RunRepository(databaseFor(this.env)).reportAgentError({ tenantId: this.auth.tenantId, runId, reason, userId: this.auth.userId, reporterKind, reporterId }); + if (result === "missing") throw new ServiceError(404, "not_found", "Run not found"); + if (result === "terminal") throw new ServiceError(409, "terminal_run", "Run is already terminal and cannot accept an error report."); + if (result === "conflict") throw new ServiceError(409, "error_already_reported", "A different error reason was already reported for this run."); + return { reported: true, reason, retry: result === "identical" }; + } async renameRun(runId: string, input: unknown) { await this.authorize("runs:write"); const parsed = renameRunSchema.parse(input); @@ -183,11 +198,12 @@ export class ApiService { run.trace_projection = projection.rows[0] ?? null; run.trace_generation = cursor.rows[0]?.generation ?? null; run.harness_log = artifacts.find(item => item.kind === "terminal_log") ?? null; // Enumerate public fields rather than publishing future database columns. + const agentError = run.agent_error_reason ? { reason: run.agent_error_reason, reported_at: run.agent_error_reported_at, reporter: { user_id: run.agent_error_reporter_user_id, kind: run.agent_error_reporter_kind, id: run.agent_error_reporter_id } } : null; return publicValue(fullRunResponse, JSON.parse(JSON.stringify({ id: run.id, tenant_id: run.tenant_id, job_id: run.job_id, invocation_id: run.invocation_id, execution_handle: run.execution_handle ? { backendKind: run.execution_handle.backendKind, id: run.execution_handle.id } : null, provider: run.provider, issue_id: run.issue_id, issue_url: run.issue_url, issue_title: run.issue_title, run_name: run.run_name, agent_name: run.agent_name, workspace_name: run.workspace_name, agent_kind: run.agent_kind, - state: run.state, execution_backend_kind: run.execution_backend_kind, backend_kind: run.backend_kind, + state: run.state, failure_source: run.failure_source ?? null, agent_error: agentError, execution_backend_kind: run.execution_backend_kind, backend_kind: run.backend_kind, execution_capabilities: run.execution_capabilities, capabilities: run.capabilities, destination_url: run.destination_url, artifact_state: run.artifact_state, artifact_error: run.artifact_error, claim_released: run.claim_released, vm_cleanup_attempt: run.vm_cleanup_attempt, cleanup_next_at: run.cleanup_next_at, vm_cleanup_complete: run.vm_cleanup_complete, @@ -205,7 +221,7 @@ export class ApiService { new ArtifactRepository(database, this.auth.tenantId).list(runId), ]); const primary = run.trace_sources?.primary ?? null; - return { runId: run.id, jobId: run.job_id, state: run.state, provider: run.provider, backendKind: run.execution_backend_kind, + return { runId: run.id, jobId: run.job_id, state: run.state, failureSource: run.failure_source, agentError: run.agent_error, provider: run.provider, backendKind: run.execution_backend_kind, execution: run.execution_diagnostics, traceSources: run.trace_sources, trace: { ...evidence, provider: primary?.provider ?? artifacts.find(item => item.kind === "native_session")?.provider ?? run.agent_kind ?? null, primarySource: primary?.kind ?? "native_session", sourceVersion: primary?.formatVersion ?? null, artifacts: artifacts.filter(item => ["execution_stream", "native_session"].includes(item.kind)), diff --git a/packages/api/src/postgres/run-query-repository.ts b/packages/api/src/postgres/run-query-repository.ts index 34c6bff..0d5cfaf 100644 --- a/packages/api/src/postgres/run-query-repository.ts +++ b/packages/api/src/postgres/run-query-repository.ts @@ -34,7 +34,9 @@ export class RunQueryRepository { } values.push(limit + 1); const order = sort ? `CASE WHEN ${sortExpressions[sort]} IS NULL THEN 1 ELSE 0 END ASC, ${sortExpressions[sort]} ${direction} NULLS LAST, r.id ${direction}` : "r.created_at DESC,r.id DESC"; - const result = await this.database.pool.query(`SELECT r.id,r.job_id,j.project_id,j.name job_name,p.name project_name,r.issue_id,r.issue_url,r.issue_title,r.run_name,r.agent_name,r.workspace_name,r.agent_kind,r.state,r.provider,r.execution_backend_kind backend_kind,r.execution_capabilities capabilities,r.destination_url,r.artifact_state,r.artifact_error,r.created_at,r.updated_at,jr.started_at, + const result = await this.database.pool.query(`SELECT r.id,r.job_id,j.project_id,j.name job_name,p.name project_name,r.issue_id,r.issue_url,r.issue_title,r.run_name,r.agent_name,r.workspace_name,r.agent_kind,r.state,r.failure_source, + CASE WHEN r.agent_error_reason IS NULL THEN NULL ELSE jsonb_build_object('reason',r.agent_error_reason,'reported_at',r.agent_error_reported_at,'reporter',jsonb_build_object('user_id',r.agent_error_reporter_user_id,'kind',r.agent_error_reporter_kind,'id',r.agent_error_reporter_id)) END agent_error, + r.provider,r.execution_backend_kind backend_kind,r.execution_capabilities capabilities,r.destination_url,r.artifact_state,r.artifact_error,r.created_at,r.updated_at,jr.started_at, CASE WHEN r.state IN ('succeeded','failed','stopped') AND jr.started_at IS NOT NULL AND jr.updated_at >= jr.started_at THEN json_build_object('duration', round(EXTRACT(EPOCH FROM (jr.updated_at - jr.started_at)))) WHEN r.state NOT IN ('succeeded','failed','stopped') THEN e.elapsed_time @@ -66,7 +68,8 @@ export class RunQueryRepository { FROM app.runs r JOIN app.jobs j ON j.tenant_id=r.tenant_id AND j.id=r.job_id JOIN app.job_runs jr ON jr.tenant_id=r.tenant_id AND jr.id=r.id JOIN app.invocations i ON i.tenant_id=r.tenant_id AND i.id=r.invocation_id WHERE r.tenant_id=$1 AND r.id=$2`, [this.tenantId, runId])).rows[0] ?? null; } async status(runId: string) { - const row = (await this.database.pool.query(`SELECT r.id,r.job_id,j.project_id,pr.name project_name,jr.state,r.run_name,j.name job_name,r.destination_url, + const row = (await this.database.pool.query(`SELECT r.id,r.job_id,j.project_id,pr.name project_name,jr.state,r.run_name,j.name job_name,r.destination_url,r.failure_source, + CASE WHEN r.agent_error_reason IS NULL THEN NULL ELSE jsonb_build_object('reason',r.agent_error_reason,'reported_at',r.agent_error_reported_at,'reporter',jsonb_build_object('user_id',r.agent_error_reporter_user_id,'kind',r.agent_error_reporter_kind,'id',r.agent_error_reporter_id)) END agent_error, r.created_at,greatest(r.updated_at,jr.updated_at) updated_at,jr.started_at,r.artifact_state, (jr.state IN ('succeeded','failed','stopped') AND (r.artifact_state IN ('pending','collecting') OR NOT r.vm_cleanup_complete)) finalizing, ${traceRevisionSql} trace_revision diff --git a/packages/api/src/postgres/run-repository.ts b/packages/api/src/postgres/run-repository.ts index 3bc5e6a..c795281 100644 --- a/packages/api/src/postgres/run-repository.ts +++ b/packages/api/src/postgres/run-repository.ts @@ -80,11 +80,32 @@ export class RunRepository { await this.database.pool.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'stop_attempt','Scheduler attempted agent termination')", [run.tenantId, crypto.randomUUID(), run.id]); } + async reportAgentError(input: { tenantId: string; runId: string; reason: string; userId: string; reporterKind: string; reporterId: string }): Promise<"missing" | "terminal" | "conflict" | "reported" | "identical"> { + return this.database.transaction(async client => { + // Keep the lifecycle lock order used by terminal(), stop(), and rename(). + const jobRun = (await client.query<{ state: ExecutionState }>("SELECT state FROM app.job_runs WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [input.tenantId, input.runId])).rows[0]; + if (!jobRun) return "missing"; + const run = (await client.query<{ agent_error_reason: string | null }>("SELECT agent_error_reason FROM app.runs WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [input.tenantId, input.runId])).rows[0]; + if (!run) return "missing"; + if (!["reserved", "starting", "running", "blocked", "stopping"].includes(jobRun.state)) return "terminal"; + if (run.agent_error_reason !== null) return run.agent_error_reason === input.reason ? "identical" : "conflict"; + await client.query(`UPDATE app.runs SET agent_error_reason=$3,agent_error_reported_at=now(),agent_error_reporter_user_id=$4, + agent_error_reporter_kind=$5,agent_error_reporter_id=$6,updated_at=now() WHERE tenant_id=$1 AND id=$2`, + [input.tenantId, input.runId, input.reason, input.userId, input.reporterKind, input.reporterId]); + await client.query("INSERT INTO app.run_activity(tenant_id,id,run_id,action,detail) VALUES ($1,$2,$3,'agent_error_reported',$4)", [input.tenantId, crypto.randomUUID(), input.runId, input.reason]); + return "reported"; + }); + } + async terminal(run: PersistedRun, state: "succeeded" | "failed" | "stopped", artifactState: "stored" | "partial" | "failed", error?: string, cleanupComplete = false): Promise { await this.database.transaction(async client => { - const result = await client.query<{ state: ExecutionState }>("UPDATE app.job_runs SET state=CASE WHEN state='stopping' THEN 'stopped' ELSE $3 END,next_poll_at=NULL,launch_lease_expires_at=NULL,updated_at=now() WHERE tenant_id=$1 AND id=$2 AND state IN ('starting','running','blocked','stopping') RETURNING state", [run.tenantId, run.id, state]); - if (!result.rows[0]) return; - await client.query("UPDATE app.runs SET state=$3,artifact_state=$4,artifact_error=$5,claim_released=true,vm_cleanup_complete=$6,updated_at=now() WHERE tenant_id=$1 AND id=$2", [run.tenantId, run.id, result.rows[0].state, artifactState, error ? safeDiagnosticText(error) : null, cleanupComplete]); + const current = (await client.query<{ state: ExecutionState }>("SELECT state FROM app.job_runs WHERE tenant_id=$1 AND id=$2 AND state IN ('starting','running','blocked','stopping') FOR UPDATE", [run.tenantId, run.id])).rows[0]; + if (!current) return; + const saved = (await client.query<{ agent_error_reason: string | null }>("SELECT agent_error_reason FROM app.runs WHERE tenant_id=$1 AND id=$2 FOR UPDATE", [run.tenantId, run.id])).rows[0]; + const terminalState = current.state === "stopping" ? "stopped" : state === "succeeded" && saved?.agent_error_reason ? "failed" : state; + const failureSource = terminalState === "failed" ? state === "failed" ? "system" : "agent" : null; + await client.query("UPDATE app.job_runs SET state=$3,next_poll_at=NULL,launch_lease_expires_at=NULL,updated_at=now() WHERE tenant_id=$1 AND id=$2", [run.tenantId, run.id, terminalState]); + await client.query("UPDATE app.runs SET state=$3,failure_source=$4,artifact_state=$5,artifact_error=$6,claim_released=true,vm_cleanup_complete=$7,updated_at=now() WHERE tenant_id=$1 AND id=$2", [run.tenantId, run.id, terminalState, failureSource, artifactState, error ? safeDiagnosticText(error) : null, cleanupComplete]); await client.query("DELETE FROM app.active_claims WHERE tenant_id=$1 AND run_id=$2", [run.tenantId, run.id]); // Automation runs before polling, so terminal events need another pass. await client.query("INSERT INTO app.wake_hints(key,not_before) VALUES ('global',now()) ON CONFLICT (key) DO UPDATE SET not_before=least(app.wake_hints.not_before,excluded.not_before),updated_at=now()"); diff --git a/packages/api/src/postgres/schema.ts b/packages/api/src/postgres/schema.ts index ed270cc..30698e5 100644 --- a/packages/api/src/postgres/schema.ts +++ b/packages/api/src/postgres/schema.ts @@ -71,7 +71,7 @@ export const jobRuns = app.table("job_runs", { export const runs = app.table("runs", { executionConfig: jsonb("execution_config").$type().notNull(), - tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), invocationId: uuid("invocation_id").notNull(), provider: text("provider").notNull(), issueId: text("issue_id").notNull(), issueUrl: text("issue_url"), issueTitle: text("issue_title").notNull().default(""), runName: text("run_name").notNull().default(""), agentName: text("agent_name").notNull(), workspaceName: text("workspace_name").notNull().default(""), agentKind: text("agent_kind").notNull().default(""), state: text("state").notNull(), executionBackendKind: text("execution_backend_kind").notNull().default(""), executionHandle: jsonb("execution_handle").$type>(), executionCapabilities: jsonb("execution_capabilities").$type().notNull().default([]), destinationUrl: text("destination_url"), artifactState: text("artifact_state").$type().notNull().default("pending"), artifactError: text("artifact_error"), traceSources: jsonb("trace_sources").$type(), executionDiagnostics: jsonb("execution_diagnostics").$type(), claimReleased: boolean("claim_released").notNull().default(false), vmCleanupAttempt: integer("vm_cleanup_attempt").notNull().default(0), cleanupNextAt: timestamp("cleanup_next_at", { withTimezone: true }), vmCleanupComplete: boolean("vm_cleanup_complete").notNull().default(false), createdAt: createdAt(), updatedAt: updatedAt(), + tenantId: uuid("tenant_id").notNull(), id: uuid("id").notNull(), jobId: uuid("job_id").notNull(), invocationId: uuid("invocation_id").notNull(), provider: text("provider").notNull(), issueId: text("issue_id").notNull(), issueUrl: text("issue_url"), issueTitle: text("issue_title").notNull().default(""), runName: text("run_name").notNull().default(""), agentName: text("agent_name").notNull(), workspaceName: text("workspace_name").notNull().default(""), agentKind: text("agent_kind").notNull().default(""), state: text("state").notNull(), executionBackendKind: text("execution_backend_kind").notNull().default(""), executionHandle: jsonb("execution_handle").$type>(), executionCapabilities: jsonb("execution_capabilities").$type().notNull().default([]), destinationUrl: text("destination_url"), artifactState: text("artifact_state").$type().notNull().default("pending"), artifactError: text("artifact_error"), failureSource: text("failure_source").$type<"system" | "agent">(), agentErrorReason: text("agent_error_reason"), agentErrorReportedAt: timestamp("agent_error_reported_at", { withTimezone: true }), agentErrorReporterUserId: uuid("agent_error_reporter_user_id"), agentErrorReporterKind: text("agent_error_reporter_kind"), agentErrorReporterId: text("agent_error_reporter_id"), traceSources: jsonb("trace_sources").$type(), executionDiagnostics: jsonb("execution_diagnostics").$type(), claimReleased: boolean("claim_released").notNull().default(false), vmCleanupAttempt: integer("vm_cleanup_attempt").notNull().default(0), cleanupNextAt: timestamp("cleanup_next_at", { withTimezone: true }), vmCleanupComplete: boolean("vm_cleanup_complete").notNull().default(false), createdAt: createdAt(), updatedAt: updatedAt(), }, t => [primaryKey({ columns: [t.tenantId, t.id] }), unique().on(t.tenantId, t.invocationId), foreignKey({ columns: [t.tenantId, t.jobId], foreignColumns: [jobs.tenantId, jobs.id] }).onDelete("cascade"), foreignKey({ columns: [t.tenantId, t.invocationId], foreignColumns: [invocations.tenantId, invocations.id] }).onDelete("cascade"), index("runs_order").on(t.tenantId, t.createdAt.desc(), t.id.desc()), index("runs_state_order").on(t.tenantId, t.state, t.createdAt.desc(), t.id.desc()), index("runs_cleanup_due").on(t.cleanupNextAt).where(sql`not ${t.vmCleanupComplete}`), index("runs_issue_trgm").using("gin", sql`${t.issueId} gin_trgm_ops`), index("runs_name_trgm").using("gin", sql`${t.runName} gin_trgm_ops`)]); export const runArtifacts = app.table("run_artifacts", { tenantId: uuid("tenant_id").notNull(), runId: uuid("run_id").notNull(), id: uuid("id").notNull(), kind: text("kind").$type().notNull(), objectKey: text("object_key").notNull(), provider: text("provider").notNull(), format: text("format").notNull(), formatVersion: text("format_version"), cliVersion: text("cli_version"), nativeSessionId: text("native_session_id"), sourceGeneration: text("source_generation"), sourcePath: text("source_path"), mediaType: text("media_type"), harnessVersion: text("harness_version"), byteSize: bigint("byte_size", { mode: "number" }).notNull(), sha256: text("sha256").notNull(), state: text("state").$type().notNull(), createdAt: createdAt(), diff --git a/packages/api/src/protected-api.ts b/packages/api/src/protected-api.ts index 7ce3cb3..2df05a1 100644 --- a/packages/api/src/protected-api.ts +++ b/packages/api/src/protected-api.ts @@ -7,7 +7,7 @@ import { createMcpHandler, McpServer } from "@modelcontextprotocol/server"; import { createMcpHonoApp } from "@modelcontextprotocol/hono"; import { z } from "zod"; import { ApiService, ServiceError } from "./flow-service"; -import { jobConditionsTestSchema, jobIdSchema, jobInputSchema, listRunsSchema, manualInvocationSchema, renameRunSchema, runIdSchema } from "./flow-schemas"; +import { jobConditionsTestSchema, jobIdSchema, jobInputSchema, listRunsSchema, manualInvocationSchema, renameRunSchema, reportRunErrorSchema, runIdSchema } from "./flow-schemas"; import type { Env, OAuthProps } from "./types"; import { executeAuth, validBrowserOrigin } from "./auth-api"; import { matchApiOperation } from "./api-contract"; @@ -111,6 +111,7 @@ function createMcpServer(service: ApiService): McpServer { tool("get_run", "Get a run with its rendered prompt, full triggered context, invocation metadata, execution metadata, output, and activity. For manual invocations, invocation.idempotency_key is the client-reusable value; invocation.claim_key is internal.", runIdSchema, ({ runId }: any) => service.getRun(runId)); tool("rename_run", "Rename a run without changing its execution. Supply expectedName to reject stale edits; names are trimmed and must contain 1 to 120 characters.", runIdSchema.extend(renameRunSchema.shape), ({ runId, ...input }: any) => service.renameRun(runId, input)); tool("get_run_diagnostics", "Get credential-safe launch, prompt delivery, output capture, claim release, cleanup, and activity diagnostics for a run.", runIdSchema, ({ runId }: any) => service.getRunDiagnostics(runId)); + tool("report_run_error", "Record an error reason for an active run and continue execution. Identical retries are idempotent; a different reason conflicts. Finish remaining work, then exit normally. Terminal runs reject reports.", runIdSchema.extend(reportRunErrorSchema.shape).strict(), ({ runId, reason }: any) => service.reportRunError(runId, { reason })); tool("stop_run", "Stop an active run", runIdSchema, ({ runId }: any) => service.stopRun(runId)); tool("kill_run", "Force a run in any non-terminal state to stop immediately, release its concurrency slot, and clean up its backend asynchronously.", runIdSchema, ({ runId }: any) => service.killRun(runId)); tool("list_jobs", "List jobs with their current run count and maximum concurrency", z.object({}), () => service.listJobs()); @@ -169,5 +170,19 @@ async function mcp(request: Request, service: ApiService, env: Env, ctx: Executi } export class ProtectedApiHandler extends WorkerEntrypoint { - fetch(request: Request): Promise { return protectedApiFetch(request, this.env, this.ctx.props, this.ctx); } + async fetch(request: Request): Promise { + let auth = this.ctx.props; + const isReportRequest = new URL(request.url).pathname.endsWith("/report-error") + || new URL(request.url).pathname === "/mcp" && await request.clone().json().then(body => (body as any)?.params?.name === "report_run_error").catch(() => false); + if (isReportRequest && auth.authMethod === "oauth" && (!auth.oauthGrantId || !auth.oauthClientId)) { + const authorization = request.headers.get("Authorization"), credential = authorization?.startsWith("Bearer ") ? authorization.slice(7) : ""; + if (credential && this.env.OAUTH_PROVIDER) { + // The SDK validates the bearer before dispatch. Unwrap only for stable grant/client IDs; + // the credential itself is never logged, returned, or persisted. + const token = await this.env.OAUTH_PROVIDER.unwrapToken(credential); + if (token?.userId === auth.userId && token.grant?.clientId) auth = { ...auth, oauthGrantId: token.grantId, oauthClientId: token.grant.clientId }; + } + } + return protectedApiFetch(request, this.env, auth, this.ctx); + } } diff --git a/packages/api/src/run-read-contracts.ts b/packages/api/src/run-read-contracts.ts index e657a8d..0e2a4d2 100644 --- a/packages/api/src/run-read-contracts.ts +++ b/packages/api/src/run-read-contracts.ts @@ -1,11 +1,12 @@ import { z } from "zod"; export const executionState = z.enum(["queued", "reserved", "starting", "running", "blocked", "stopping", "succeeded", "failed", "stopped"]); +export const agentErrorResponse = z.object({ reason: z.string(), reported_at: z.string(), reporter: z.object({ user_id: z.string(), kind: z.enum(["api_key", "oauth", "session"]), id: z.string() }) }); export const runStatusResponse = z.object({ id: z.string(), job_id: z.string(), project_id: z.string().optional(), project_name: z.string().optional(), state: executionState, run_name: z.string(), job_name: z.string(), destination_url: z.string().nullable(), created_at: z.string(), updated_at: z.string(), started_at: z.string().nullable(), artifact_state: z.enum(["pending", "collecting", "stored", "partial", "failed"]), - finalizing: z.boolean(), trace_revision: z.string(), + finalizing: z.boolean(), trace_revision: z.string(), failure_source: z.enum(["system", "agent"]).nullable(), agent_error: agentErrorResponse.nullable(), }); export const traceEventResponse = z.object({ sequence: z.number().int(), id: z.string(), parentId: z.string().optional(), @@ -18,6 +19,7 @@ export const revisionTracePage = z.object({ items: z.array(traceEventResponse), export const traceRevisionSql = "jsonb_build_array(c.generation,p.source_kind,p.artifact_sha256,p.parser_version,p.updated_at)::text"; export const runSummaryResponse = z.object({ id: z.string(), job_id: z.string(), project_id: z.string().optional(), project_name: z.string().optional(), job_name: z.string(), issue_id: z.string(), issue_url: z.string().nullable(), issue_title: z.string(), run_name: z.string(), agent_name: z.string(), workspace_name: z.string(), agent_kind: z.string(), state: z.enum([...executionState.options, "done", "ignored"]), provider: z.string(), backend_kind: z.string(), capabilities: z.array(z.string()), destination_url: z.string().nullable(), artifact_state: z.enum(["pending", "collecting", "stored", "partial", "failed"]), artifact_error: z.string().nullable(), created_at: z.string(), updated_at: z.string(), started_at: z.string().nullable(), + failure_source: z.enum(["system", "agent"]).nullable(), agent_error: agentErrorResponse.nullable(), elapsed_time: z.union([ z.object({ duration: z.number() }), z.object({ low: z.number(), high: z.number() }), diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 1199c96..494f42b 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -40,6 +40,9 @@ export type OAuthProps = { sessionVersion: number; scopes: string[]; accessTokenId?: string; + /** Stable identifier for the OAuth authorization grant; never a bearer token. */ + oauthGrantId?: string; + oauthClientId?: string; authMethod?: "session" | "oauth" | "access_token"; }; diff --git a/packages/api/test/api-boundary.test.ts b/packages/api/test/api-boundary.test.ts index 5a8b1b4..73fe70c 100644 --- a/packages/api/test/api-boundary.test.ts +++ b/packages/api/test/api-boundary.test.ts @@ -175,7 +175,7 @@ describe("Worker OpenAPI request/response boundaries", () => { }); it("documents the job name in run detail for clients with only runs:read", async () => { - vi.spyOn(ApiService.prototype, "getRun").mockResolvedValue({ id: "run-1", tenant_id: "tenant", execution_handle: null, job_id: "job-1", issue_id: "", issue_url: null, issue_title: "", run_name: "", agent_name: "", workspace_name: "", agent_kind: "codex", state: "queued", provider: "manual", backend_kind: "exe-vm", capabilities: [], destination_url: null, artifact_state: "pending", artifact_error: null, created_at: "2026-10-09T12:00:00.000Z", updated_at: "2026-10-09T12:00:00.000Z", started_at: null, job_name: "Job name", invocation_id: "invocation", execution_backend_kind: "exe-vm", execution_capabilities: [], claim_released: false, vm_cleanup_attempt: 0, cleanup_next_at: null, vm_cleanup_complete: false, occurrence: {}, invocation_source: "manual", invocation_claim_key: "manual:key", invocation_trigger_id: null, invocation_created_at: "2026-10-09T12:00:00.000Z", trace_sources: null, trace_projection: null, trace_generation: null, execution_diagnostics: null, harness_log: null, prompt: "Prompt", context: {}, invocation: { id: "invocation", source: "manual", claim_key: "manual:key", trigger_id: null, context: {}, occurrence: {}, created_at: "2026-10-09T12:00:00.000Z" }, activity: [] }); + vi.spyOn(ApiService.prototype, "getRun").mockResolvedValue({ id: "run-1", tenant_id: "tenant", execution_handle: null, job_id: "job-1", issue_id: "", issue_url: null, issue_title: "", run_name: "", agent_name: "", workspace_name: "", agent_kind: "codex", state: "queued", failure_source: null, agent_error: null, provider: "manual", backend_kind: "exe-vm", capabilities: [], destination_url: null, artifact_state: "pending", artifact_error: null, created_at: "2026-10-09T12:00:00.000Z", updated_at: "2026-10-09T12:00:00.000Z", started_at: null, job_name: "Job name", invocation_id: "invocation", execution_backend_kind: "exe-vm", execution_capabilities: [], claim_released: false, vm_cleanup_attempt: 0, cleanup_next_at: null, vm_cleanup_complete: false, occurrence: {}, invocation_source: "manual", invocation_claim_key: "manual:key", invocation_trigger_id: null, invocation_created_at: "2026-10-09T12:00:00.000Z", trace_sources: null, trace_projection: null, trace_generation: null, execution_diagnostics: null, harness_log: null, prompt: "Prompt", context: {}, invocation: { id: "invocation", source: "manual", claim_key: "manual:key", trigger_id: null, context: {}, occurrence: {}, created_at: "2026-10-09T12:00:00.000Z" }, activity: [] }); const result = await assertResponse(await request("GET", "/api/v1/runs/run-1", undefined, { auth: { ...auth, scopes: ["runs:read"] } }), "GET", "/api/v1/runs/{runId}", 200); expect(result.job_name).toBe("Job name"); }); diff --git a/packages/api/test/report-run-error-boundary.test.ts b/packages/api/test/report-run-error-boundary.test.ts new file mode 100644 index 0000000..31fbe58 --- /dev/null +++ b/packages/api/test/report-run-error-boundary.test.ts @@ -0,0 +1,123 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ApiService, ServiceError } from "../src/flow-service"; +import { ProtectedApiHandler, protectedApiFetch } from "../src/protected-api"; +import { RunRepository } from "../src/postgres/run-repository"; + +vi.mock("cloudflare:workers", () => ({ WorkerEntrypoint: class { constructor(public ctx: any, public env: any) {} } })); +afterEach(() => vi.restoreAllMocks()); + +const runId = "00000000-0000-4000-8000-000000000001"; +async function call(transport: "REST" | "MCP", input: Record, auth: any = { tenantId: "tenant", userId: "owner", authMethod: "oauth", oauthGrantId: "grant-1", scopes: ["runs:write"] }) { + const url = transport === "REST" ? `https://example.com/api/v1/runs/${runId}/report-error` : "https://example.com/mcp"; + const body = transport === "REST" ? input : { jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "report_run_error", arguments: { runId, ...input } } }; + return protectedApiFetch(new Request(url, { method: "POST", headers: { Host: "example.com", Origin: "https://example.com", "Content-Type": "application/json", Accept: "application/json, text/event-stream" }, body: JSON.stringify(body) }), { APP_ORIGIN: "https://example.com", DATABASE: {} } as any, auth, {} as any); +} + +describe.each(["REST", "MCP"] as const)("%s report run error boundary", transport => { + it("routes a bounded validated reason through the same service and requires runs:write", async () => { + vi.spyOn(ApiService.prototype as any, "authorize").mockImplementation(function(this: any, scope: any) { if (!this.auth.scopes.includes(scope)) throw new ServiceError(403, "insufficient_scope", "Missing scope"); }); + const report = vi.spyOn(ApiService.prototype, "reportRunError").mockImplementation(async function(this: any) { await this.authorize("runs:write"); return { reported: true, reason: "Problem", retry: false }; }); + const response = await call(transport, { reason: " Problem " }); + expect(response.status).toBe(200); + if (transport === "REST") expect(await response.json()).toEqual({ reported: true, reason: "Problem", retry: false }); + else expect(await response.text()).toContain('"reason":"Problem"'); + expect(report).toHaveBeenCalledWith(runId, { reason: "Problem" }); + + const unauthorized = await call(transport, { reason: "Problem" }, { tenantId: "tenant", userId: "owner", scopes: ["runs:read"] }); + if (transport === "REST") expect(unauthorized.status).toBe(403); + else expect(await unauthorized.text()).toContain("insufficient_scope"); + const callsBeforeInvalid = report.mock.calls.length; + const invalid = await call(transport, { reason: " ", token: "must-not-be-accepted" }); + if (transport === "REST") expect(invalid.status).toBe(400); + else expect(await invalid.text()).toContain("Input validation error"); + expect(report).toHaveBeenCalledTimes(callsBeforeInvalid); + }); + + it("maps terminal and replacement conflicts without overwriting the first report", async () => { + vi.spyOn(ApiService.prototype, "reportRunError").mockRejectedValueOnce(new ServiceError(409, "terminal_run", "terminal")); + const terminal = await call(transport, { reason: "Problem" }); + if (transport === "REST") expect(terminal.status).toBe(409); + else expect(await terminal.text()).toContain("terminal_run"); + vi.spyOn(ApiService.prototype, "reportRunError").mockRejectedValueOnce(new ServiceError(409, "error_already_reported", "conflict")); + const conflict = await call(transport, { reason: "Different" }); + if (transport === "REST") expect(conflict.status).toBe(409); + else expect(await conflict.text()).toContain("error_already_reported"); + }); +}); + +describe("report persistence lock and retry behavior", () => { + it("locks job_runs before runs, records provenance once, and distinguishes identical and conflicting retries", async () => { + let existing: string | null = null; + const query = vi.fn(async (sql: string, values: unknown[]) => { + if (sql.includes("SELECT state FROM app.job_runs")) return { rows: [{ state: "running" }] }; + if (sql.includes("SELECT agent_error_reason")) return { rows: [{ agent_error_reason: existing }] }; + if (sql.startsWith("UPDATE app.runs SET agent_error_reason")) { existing = values[2] as string; return { rows: [] }; } + return { rows: [] }; + }); + const repo = new RunRepository({ transaction: (work: any) => work({ query }) } as any); + const input = { tenantId: "tenant", runId, reason: "safe reason", userId: "owner", reporterKind: "oauth", reporterId: "grant-stable" }; + expect(await repo.reportAgentError(input)).toBe("reported"); + expect(await repo.reportAgentError(input)).toBe("identical"); + expect(await repo.reportAgentError({ ...input, reason: "replacement" })).toBe("conflict"); + expect(query.mock.calls[0][0]).toContain("FROM app.job_runs"); + expect(query.mock.calls[1][0]).toContain("FROM app.runs"); + expect(query.mock.calls[2][1]).toEqual(["tenant", runId, "safe reason", "owner", "oauth", "grant-stable"]); + expect(query.mock.calls.filter(([sql]) => String(sql).startsWith("UPDATE app.runs SET agent_error_reason"))).toHaveLength(1); + }); + + it.each([ + ["succeeded", "running", "reported problem", "failed", "agent"], + ["failed", "running", "reported problem", "failed", "system"], + ["succeeded", "stopping", "reported problem", "stopped", null], + ] as const)("finalizes observer %s from %s with report precedence", async (observation, existingState, reason, expectedState, expectedSource) => { + const calls: Array<[string, unknown[]]> = []; + const query = vi.fn(async (sql: string, values: unknown[] = []) => { + calls.push([sql, values]); + if (sql.includes("FROM app.job_runs") && sql.includes("FOR UPDATE")) return { rows: [{ state: existingState }] }; + if (sql.includes("SELECT agent_error_reason")) return { rows: [{ agent_error_reason: reason }] }; + return { rows: [] }; + }); + const repo = new RunRepository({ transaction: (work: any) => work({ query }) } as any); + await repo.terminal({ tenantId: "tenant", id: runId } as any, observation, "partial", "artifact failed", true); + const runUpdate = calls.find(([sql]) => sql.startsWith("UPDATE app.runs SET state="))!; + expect(runUpdate[0]).toContain("failure_source=$4"); + expect(runUpdate[1].slice(2, 7)).toEqual([expectedState, expectedSource, "partial", "artifact failed", true]); + }); +}); + +describe("report authenticated provenance and sanitization", () => { + it.each([ + [{ authMethod: "oauth", oauthGrantId: "stable-grant", oauthClientId: "client" }, "oauth", "client:stable-grant"], + [{ authMethod: "access_token", accessTokenId: "stable-token" }, "api_key", "stable-token"], + ] as const)("derives safe %s provenance from the authenticated request", async (identity, kind, identifier) => { + vi.spyOn(ApiService.prototype as any, "authorize").mockResolvedValue(undefined); + const report = vi.spyOn(RunRepository.prototype, "reportAgentError").mockResolvedValue("reported"); + const auth = { tenantId: "tenant", userId: "owner", scopes: ["runs:write"], ...identity } as any; + const result = await new ApiService({ DATABASE: {} } as any, auth).reportRunError(runId, { reason: "token=do-not-store" }); + expect(result.reason).not.toContain("do-not-store"); + expect(report).toHaveBeenCalledWith(expect.objectContaining({ tenantId: "tenant", runId, userId: "owner", reporterKind: kind, reporterId: identifier })); + }); + + it("accepts an identical retry and rejects missing OAuth grant provenance", async () => { + vi.spyOn(ApiService.prototype as any, "authorize").mockResolvedValue(undefined); + vi.spyOn(RunRepository.prototype, "reportAgentError").mockResolvedValue("identical"); + const auth = { tenantId: "tenant", userId: "owner", scopes: ["runs:write"], authMethod: "oauth", oauthGrantId: "stable-grant", oauthClientId: "client" } as any; + await expect(new ApiService({ DATABASE: {} } as any, auth).reportRunError(runId, { reason: "same" })).resolves.toMatchObject({ retry: true }); + const missing = { ...auth, oauthGrantId: undefined }; + await expect(new ApiService({ DATABASE: {} } as any, missing).reportRunError(runId, { reason: "same" })).rejects.toMatchObject({ status: 403 }); + }); + + it("uses the validated OAuth token summary for legacy grant reporter identity", async () => { + const credential = "never-persist-this-token-value", unwrapToken = vi.fn().mockResolvedValue({ userId: "owner", grantId: "real-grant-id", grant: { clientId: "real-client-id" } }); + const report = vi.spyOn(ApiService.prototype, "reportRunError").mockImplementation(async function(this: any) { + expect(this.auth).toMatchObject({ oauthGrantId: "real-grant-id", oauthClientId: "real-client-id" }); + return { reported: true, reason: "safe", retry: false }; + }); + const handler = new ProtectedApiHandler({ props: { tenantId: "tenant", userId: "owner", scopes: ["runs:write"], authMethod: "oauth" } } as any, { APP_ORIGIN: "https://example.com", OAUTH_PROVIDER: { unwrapToken } } as any); + const response = await handler.fetch(new Request(`https://example.com/api/v1/runs/${runId}/report-error`, { method: "POST", headers: { Authorization: `Bearer ${credential}`, "Content-Type": "application/json" }, body: JSON.stringify({ reason: "safe" }) })); + expect(response.status).toBe(200); + expect(unwrapToken).toHaveBeenCalledWith(credential); + expect(report).toHaveBeenCalledOnce(); + expect(JSON.stringify(await response.json())).not.toContain(credential); + }); +}); diff --git a/packages/api/test/report-run-error.postgres.test.ts b/packages/api/test/report-run-error.postgres.test.ts new file mode 100644 index 0000000..eecd39a --- /dev/null +++ b/packages/api/test/report-run-error.postgres.test.ts @@ -0,0 +1,54 @@ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { loadingFixture } from "./helpers/loading-fixture"; +import { RunRepository } from "../src/postgres/run-repository"; + +const url = process.env.AUTH_TEST_DATABASE_URL; +vi.mock("@cloudflare/workers-oauth-provider", () => ({ OAuthProvider: class {}, AuthorizationError: class extends Error {}, getOAuthApi: vi.fn() })); +vi.mock("cloudflare:workers", () => ({ WorkerEntrypoint: class {}, DurableObject: class {} })); +describe.skipIf(!url)("agent run error persistence (PostgreSQL)", () => { + let f: Awaited>; + beforeAll(async () => { f = await loadingFixture(url!); }); + afterAll(async () => f?.cleanup()); + + async function activate() { + await f.db.pool.query("UPDATE app.job_runs SET state='running',started_at=now() WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + await f.db.pool.query("UPDATE app.runs SET state='running',failure_source=NULL,agent_error_reason=NULL,agent_error_reported_at=NULL,agent_error_reporter_user_id=NULL,agent_error_reporter_kind=NULL,agent_error_reporter_id=NULL,claim_released=false,vm_cleanup_complete=false WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + } + + it("isolates tenants and atomically keeps an accepted report when observer success finalizes", async () => { + const runs = new RunRepository(f.db); + await activate(); + const provenance = { userId: f.auth.userId, reporterKind: "oauth", reporterId: "stable-grant" }; + expect(await runs.reportAgentError({ tenantId: crypto.randomUUID(), runId: f.runId, reason: "safe failure", ...provenance })).toBe("missing"); + + const [report, terminal] = await Promise.all([ + runs.reportAgentError({ tenantId: f.tenantId, runId: f.runId, reason: "safe failure", ...provenance }), + runs.terminal({ tenantId: f.tenantId, id: f.runId } as any, "succeeded", "stored", undefined, true), + ]); + const final = (await f.db.pool.query("SELECT jr.state job_state,r.state,r.failure_source,r.agent_error_reason,r.agent_error_reporter_user_id,r.agent_error_reporter_kind,r.agent_error_reporter_id FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE r.tenant_id=$1 AND r.id=$2", [f.tenantId, f.runId])).rows[0]; + if (report === "reported") { + expect(final).toMatchObject({ job_state: "failed", state: "failed", failure_source: "agent", agent_error_reason: "safe failure", agent_error_reporter_user_id: f.auth.userId, agent_error_reporter_kind: "oauth", agent_error_reporter_id: "stable-grant" }); + expect(await runs.reportAgentError({ tenantId: f.tenantId, runId: f.runId, reason: "safe failure", ...provenance })).toBe("terminal"); + } else { + expect(report).toBe("terminal"); + expect(final).toMatchObject({ job_state: "succeeded", state: "succeeded", failure_source: null, agent_error_reason: null }); + } + }); + + it("keeps system diagnostics authoritative on observer failure and preserves reports when stopped", async () => { + const runs = new RunRepository(f.db), provenance = { userId: f.auth.userId, reporterKind: "api_key", reporterId: "stable-token" }; + await activate(); + expect(await runs.reportAgentError({ tenantId: f.tenantId, runId: f.runId, reason: "agent finding", ...provenance })).toBe("reported"); + await runs.terminal({ tenantId: f.tenantId, id: f.runId } as any, "failed", "partial", "artifact upload failed", true); + let final = (await f.db.pool.query("SELECT jr.state job_state,r.state,r.failure_source,r.agent_error_reason,r.agent_error_reporter_id,r.artifact_error FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE r.tenant_id=$1 AND r.id=$2", [f.tenantId, f.runId])).rows[0]; + expect(final).toMatchObject({ job_state: "failed", state: "failed", failure_source: "system", agent_error_reason: "agent finding", agent_error_reporter_id: "stable-token", artifact_error: "artifact upload failed" }); + + await activate(); + expect(await runs.reportAgentError({ tenantId: f.tenantId, runId: f.runId, reason: "keep on stop", ...provenance })).toBe("reported"); + await f.db.pool.query("UPDATE app.job_runs SET state='stopping' WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + await f.db.pool.query("UPDATE app.runs SET state='stopping' WHERE tenant_id=$1 AND id=$2", [f.tenantId, f.runId]); + await runs.terminal({ tenantId: f.tenantId, id: f.runId } as any, "succeeded", "stored", undefined, true); + final = (await f.db.pool.query("SELECT jr.state job_state,r.state,r.failure_source,r.agent_error_reason FROM app.job_runs jr JOIN app.runs r USING (tenant_id,id) WHERE r.tenant_id=$1 AND r.id=$2", [f.tenantId, f.runId])).rows[0]; + expect(final).toEqual({ job_state: "stopped", state: "stopped", failure_source: null, agent_error_reason: "keep on stop" }); + }); +}); diff --git a/packages/app/src/features/runs/detail.tsx b/packages/app/src/features/runs/detail.tsx index 5c1b93d..67cabc1 100644 --- a/packages/app/src/features/runs/detail.tsx +++ b/packages/app/src/features/runs/detail.tsx @@ -58,7 +58,7 @@ export function RunDetail() { {status.isPending &&

Loading run…

}{status.error &&

{status.error.message}

} {status.data ? {status.data.job_name} : Projects} {status.data && <> - { event.preventDefault(); setNameError(""); rename.mutate(); }}> { setNameDraft(event.target.value); setNameError(""); }} onKeyDown={event => { if (event.key === "Escape") { event.preventDefault(); cancelRename(); } }} />{nameError && {nameError}} : } description={

{status.data.state}{status.data.finalizing ? " · Finalizing trace and artifacts" : ""}

}>{["starting", "running", "blocked", "stopping"].includes(status.data.state) && } + { event.preventDefault(); setNameError(""); rename.mutate(); }}> { setNameDraft(event.target.value); setNameError(""); }} onKeyDown={event => { if (event.key === "Escape") { event.preventDefault(); cancelRename(); } }} />{nameError && {nameError}} : } description={

{status.data.state === "failed" && status.data.failure_source ? status.data.failure_source === "agent" ? "Agent failed" : "System failed" : status.data.state}{status.data.finalizing ? " · Finalizing trace and artifacts" : ""}{(status.data.agent_error as any)?.reason && {(status.data.agent_error as any).reason}}

}>{["starting", "running", "blocked", "stopping"].includes(status.data.state) && } } {!status.data && } {stop.error &&

{stop.error.message}

} diff --git a/packages/app/src/features/runs/expanded-detail.tsx b/packages/app/src/features/runs/expanded-detail.tsx index 7dcea54..e18ed2e 100644 --- a/packages/app/src/features/runs/expanded-detail.tsx +++ b/packages/app/src/features/runs/expanded-detail.tsx @@ -7,6 +7,6 @@ export function ExpandedRunDetail({ runId }: { runId: string }) { const [details, setDetails] = useState(false), [diagnostics, setDiagnostics] = useState(false); const run = useQuery({ ...runDetailQuery(runId), enabled: details }), evidence = useQuery({ ...diagnosticsQuery(runId), enabled: diagnostics }); return setDetails(e.currentTarget.open)}>Prompt, context and provenance - {run.isPending &&

Loading run details…

}{run.error &&

{run.error.message}

}{run.data && <>

Prompt

{run.data.prompt}

Activity

    {run.data.activity.map((item, index) =>
  • {item.created_at} · {item.action}: {item.detail}
    • )}
} + {run.isPending &&

Loading run details…

}{run.error &&

{run.error.message}

}{run.data && <>{run.data.agent_error &&

Agent reported error

{(run.data.agent_error as any).reason}

Reported {new Date((run.data.agent_error as any).reported_at).toLocaleString()}

}{run.data.execution_diagnostics?.detail &&

Execution observer diagnostics

{run.data.execution_diagnostics.detail}

}

Prompt

{run.data.prompt}

Activity

    {run.data.activity.map((item, index) =>
  • {item.created_at} · {item.action}: {item.detail}
    • )}
} setDiagnostics(e.currentTarget.open)}>Diagnostics and artifacts{evidence.isPending &&

Loading diagnostics…

}{evidence.error &&

{evidence.error.message}

}{evidence.data && 
{JSON.stringify(evidence.data, null, 2)}
}; } diff --git a/packages/app/src/features/runs/history.tsx b/packages/app/src/features/runs/history.tsx index 1738cec..ed4e237 100644 --- a/packages/app/src/features/runs/history.tsx +++ b/packages/app/src/features/runs/history.tsx @@ -25,8 +25,8 @@ export function RunHistory({ jobId }: { jobId: string }) { {runs.data && <>{columns.slice(1, 2).map(([column, label]) => sortColumn(column)} />)}{columns.slice(2).map(([column, label]) => sortColumn(column)} />)}{runs.data.items.map(run => - - + + )}{runs.data.items.length === 0 && }
Job runs{search.sort ? "" : ", newest first"}
Elapsed
{run.run_name || run.issue_title || "Run"}{run.run_name || run.issue_title || "Run"}{(run.agent_error as any)?.reason &&

{(run.agent_error as any).reason}

}		 {run.elapsed_time ? formatElapsedTime(run.elapsed_time) : "—"}{run.agent_kind}
{search.states?.length ? <>

No runs match the selected statuses.

: "No runs found."}
} diff --git a/packages/app/src/features/runs/list.tsx b/packages/app/src/features/runs/list.tsx index f5f6c1c..0e67868 100644 --- a/packages/app/src/features/runs/list.tsx +++ b/packages/app/src/features/runs/list.tsx @@ -27,5 +27,5 @@ export function RunList({ jobId, limit = 30 }: { jobId?: string; limit?: number const pagination = jobId ? : ; const projectOptions = (projects.data ?? []).map(project => ({ id: project.id, label: project.name })), jobOptions = (jobs.data?.items ?? []).map(job => ({ id: job.id, label: job.name })); const emptyMessage = search.states?.length && !search.projects?.length && !search.jobs?.length ? "No runs match the selected statuses." : "No runs match the selected filters."; - return
{runs.isPending &&

Loading runs…

}{runs.error &&

{runs.error.message}

}{runs.data && <>{statusHeader}{!jobId ? changeSelection("projects", id, checked)} onClear={() => clearFilters("projects")} /> : }{!jobId ? changeSelection("jobs", id, checked)} onClear={() => clearFilters("jobs")} /> : }{sortableHeader("run", "Run")}{sortableHeader("created", "Created")}{runs.data.items.map(run => )}{runs.data.items.length === 0 && }
Job runs{search.sort ? "" : ", newest first"}
ProjectJobElapsed
{run.project_id && run.project_name ?
changeSelection("projects", run.project_id!, !(search.projects ?? []).includes(run.project_id!))} />{run.project_name}
: —}
changeSelection("jobs", run.job_id, !(search.jobs ?? []).includes(run.job_id))} />{run.job_name}
{run.run_name || "Run"}{run.elapsed_time ? formatElapsedTime(run.elapsed_time) : "—"}
{search.states?.length || search.projects?.length || search.jobs?.length ? <>

{emptyMessage}

: jobId ? "No runs found." : "No runs yet"}
}{pagination}
; + return
{runs.isPending &&

Loading runs…

}{runs.error &&

{runs.error.message}

}{runs.data && <>{statusHeader}{!jobId ? changeSelection("projects", id, checked)} onClear={() => clearFilters("projects")} /> : }{!jobId ? changeSelection("jobs", id, checked)} onClear={() => clearFilters("jobs")} /> : }{sortableHeader("run", "Run")}{sortableHeader("created", "Created")}{runs.data.items.map(run => )}{runs.data.items.length === 0 && }
Job runs{search.sort ? "" : ", newest first"}
ProjectJobElapsed
{run.project_id && run.project_name ?
changeSelection("projects", run.project_id!, !(search.projects ?? []).includes(run.project_id!))} />{run.project_name}
: —}
changeSelection("jobs", run.job_id, !(search.jobs ?? []).includes(run.job_id))} />{run.job_name}
{run.run_name || "Run"}{(run.agent_error as any)?.reason &&

{(run.agent_error as any).reason}

}		{run.elapsed_time ? formatElapsedTime(run.elapsed_time) : "—"}
{search.states?.length || search.projects?.length || search.jobs?.length ? <>

{emptyMessage}

: jobId ? "No runs found." : "No runs yet"}
}{pagination}
; } diff --git a/packages/app/src/shared/ui.tsx b/packages/app/src/shared/ui.tsx index 9836b8c..7746b55 100644 --- a/packages/app/src/shared/ui.tsx +++ b/packages/app/src/shared/ui.tsx @@ -60,8 +60,9 @@ const runStatusStyles: Record; } export function Button({ variant = "secondary", className = "", ...props }: ComponentProps<"button"> & { variant?: keyof typeof buttonClasses }) { diff --git a/packages/docs/llms-full.txt b/packages/docs/llms-full.txt index d5d0111..2db39cc 100644 --- a/packages/docs/llms-full.txt +++ b/packages/docs/llms-full.txt @@ -168,6 +168,8 @@ Check that the intended trigger fired, the repository/task appears in the render For missing output or execution failures, call `get_run_diagnostics` and follow [failed-run troubleshooting](https://docs.factorize.sh/troubleshooting/runs). +Failed runs keep the existing `failed` state filter and expose `failure_source` as `system` or `agent`. Historical failed runs read as system failures. An agent-reported reason appears in run detail and history; execution observer diagnostics remain available separately, and artifact/cleanup errors are separate diagnostics. + ## Stop and retry runs Source: https://docs.factorize.sh/work/stop-retry @@ -177,6 +179,8 @@ Ask: “Inspect this run, stop it if it is still active, and verify its final st For a timed-out manual submission, repeat `invoke_job` with the **same** `idempotencyKey`. The response contains the original `invocationId`, `runId`, and `duplicate: true`. Retain the key you submitted: current `get_run` does not return `invocation.idempotency_key`. Never send its internal `invocation.claim_key` or a `manual:` prefix back as the key. If the key was lost, inspect existing runs before requesting another. +To report a problem without stopping the agent, call `report_run_error` (or `POST /api/v1/runs/{runId}/report-error`) with a reason, continue any useful remaining work, and then exit normally. The first reason is retained: identical retries are safe, while a different replacement reason returns a conflict. Terminal runs reject reports. A report never changes polling, evidence collection, cleanup, or slot accounting. + An intentional new run after failure needs a new key. First verify that the prior run is terminal, inspect whether it made external changes, and fix the cause. A retry may repeat those external effects: invocation deduplication is not transactional rollback of the agent's work. For `queue_full`, wait for capacity and reuse the key for the intended request. See [failed runs](https://docs.factorize.sh/troubleshooting/runs). ## Schedule a repository review @@ -426,6 +430,7 @@ The required scope inventory below is authoritative. Legacy `flows:*` and `runs: | `get_run` | runs:read | Required string `runId` | | `rename_run` | runs:write | Required `runId` and `name`; optional `expectedName` | | `get_run_diagnostics` | runs:read | Required string `runId` | +| `report_run_error` | runs:write | Required `runId` and `reason`; records an error without stopping execution | | `invoke_job` | runs:invoke | Required string `jobId`; optional `name`, `prompt`, `data`, `idempotencyKey` | | `stop_run` | runs:cancel | Required string `runId` | | `kill_run` | runs:cancel | Required string `runId`; currently the same stop operation | @@ -541,6 +546,8 @@ Allowed state filters: `queued`, `starting`, `running`, `done`, `blocked`, `fail Retain the submitted key for retries. Current `get_run` returns only internal `invocation.claim_key`, not `invocation.idempotency_key`; never pass that internal field back. `get_run_diagnostics({"runId":"RUN_ID"})` returns a diagnostic object with `runId`, `jobId`, `state`, `execution`, `trace`, `artifact`, `harnessLog`, `activity` and timestamps. These cover launch/prompt/output/cleanup and trace evidence, not credentials. Missing runs return `not_found`. +`report_run_error({"runId":"RUN_ID","reason":"The deployment check found a regression."})` records a bounded, sanitized reason and the authenticated reporter while the run remains active. It does not stop execution or release concurrency. Report the problem, finish any remaining work, then exit normally so Factorize can collect evidence and clean up. An identical retry is idempotent; a different reason returns `error_already_reported`, and a terminal run returns `terminal_run`. Reporter identity comes from the authenticated API key or OAuth grant; credentials are never accepted in the input or returned. A report followed by observer success makes the final run `failed` with source `agent`; observer failure remains a `system` failure and its diagnostics are retained alongside the report. Stopping takes precedence and keeps the run `stopped`. + `rename_run({"runId":"RUN_ID","name":"Release review","expectedName":"First review"})` → `{"run_name":"Release review"}`. Names are trimmed and must contain 1–120 characters. `expectedName` is optional; when supplied, a changed current name returns `stale_run_name` so the caller can reload and reconcile. Missing or cross-tenant runs return `not_found`. `stop_run({"runId":"RUN_ID"})` → `{"id":"RUN_ID","state":"stopping"}`. Current `kill_run({"runId":"RUN_ID"})` returns the same result and uses the same operation. Both accept starting/running/blocked runs; queued or terminal runs return `operation_failed`. Neither guarantees immediate termination/slot release. Poll `get_run` and inspect cleanup. See [stop/retry](https://docs.factorize.sh/work/stop-retry). diff --git a/packages/docs/mcp/tools.mdx b/packages/docs/mcp/tools.mdx index 6f93b67..190c98f 100644 --- a/packages/docs/mcp/tools.mdx +++ b/packages/docs/mcp/tools.mdx @@ -32,6 +32,7 @@ The required scope inventory below is authoritative. Legacy `flows:*` and `runs: | `get_run` | runs:read | Required string `runId` | | `rename_run` | runs:write | Required `runId` and `name`; optional `expectedName` | | `get_run_diagnostics` | runs:read | Required string `runId` | +| `report_run_error` | runs:write | Required `runId` and `reason`; records an error without stopping execution | | `invoke_job` | runs:invoke | Required string `jobId`; optional `name`, `prompt`, `data`, `idempotencyKey` | | `stop_run` | runs:cancel | Required string `runId` | | `kill_run` | runs:cancel | Required string `runId`; currently the same stop operation | @@ -147,6 +148,8 @@ Allowed state filters: `queued`, `starting`, `running`, `done`, `blocked`, `fail Retain the submitted key for retries. Current `get_run` returns only internal `invocation.claim_key`, not `invocation.idempotency_key`; never pass that internal field back. `get_run_diagnostics({"runId":"RUN_ID"})` returns a diagnostic object with `runId`, `jobId`, `state`, `execution`, `trace`, `artifact`, `harnessLog`, `activity` and timestamps. These cover launch/prompt/output/cleanup and trace evidence, not credentials. Missing runs return `not_found`. +`report_run_error({"runId":"RUN_ID","reason":"The deployment check found a regression."})` records a bounded, sanitized reason and the authenticated reporter while the run remains active. It does not stop execution or release concurrency. Report the problem, finish any remaining work, then exit normally so Factorize can collect evidence and clean up. An identical retry is idempotent; a different reason returns `error_already_reported`, and a terminal run returns `terminal_run`. Reporter identity comes from the authenticated API key or OAuth grant; credentials are never accepted in the input or returned. A report followed by observer success makes the final run `failed` with source `agent`; observer failure remains a `system` failure and its diagnostics are retained alongside the report. Stopping takes precedence and keeps the run `stopped`. + `rename_run({"runId":"RUN_ID","name":"Release review","expectedName":"First review"})` → `{"run_name":"Release review"}`. Names are trimmed and must contain 1–120 characters. `expectedName` is optional; when supplied, a changed current name returns `stale_run_name` so the caller can reload and reconcile. Missing or cross-tenant runs return `not_found`. `stop_run({"runId":"RUN_ID"})` → `{"id":"RUN_ID","state":"stopping"}`. Current `kill_run({"runId":"RUN_ID"})` returns the same result and uses the same operation. Both accept starting/running/blocked runs; queued or terminal runs return `operation_failed`. Neither guarantees immediate termination/slot release. Poll `get_run` and inspect cleanup. See [stop/retry](/work/stop-retry). diff --git a/packages/docs/openapi.yaml b/packages/docs/openapi.yaml index 0679c2a..413e24e 100644 --- a/packages/docs/openapi.yaml +++ b/packages/docs/openapi.yaml @@ -110,6 +110,45 @@ components: type: - string - "null" + failure_source: + anyOf: + - type: string + enum: + - system + - agent + - type: "null" + agent_error: + anyOf: + - type: object + properties: + reason: + type: string + reported_at: + type: string + reporter: + type: object + properties: + user_id: + type: string + kind: + type: string + enum: + - api_key + - oauth + - session + id: + type: string + required: + - user_id + - kind + - id + additionalProperties: false + required: + - reason + - reported_at + - reporter + additionalProperties: false + - type: "null" elapsed_time: anyOf: - anyOf: @@ -152,6 +191,8 @@ components: - created_at - updated_at - started_at + - failure_source + - agent_error additionalProperties: true RunPage: type: object @@ -233,6 +274,45 @@ components: type: - string - "null" + failure_source: + anyOf: + - type: string + enum: + - system + - agent + - type: "null" + agent_error: + anyOf: + - type: object + properties: + reason: + type: string + reported_at: + type: string + reporter: + type: object + properties: + user_id: + type: string + kind: + type: string + enum: + - api_key + - oauth + - session + id: + type: string + required: + - user_id + - kind + - id + additionalProperties: false + required: + - reason + - reported_at + - reporter + additionalProperties: false + - type: "null" elapsed_time: anyOf: - anyOf: @@ -275,6 +355,8 @@ components: - created_at - updated_at - started_at + - failure_source + - agent_error additionalProperties: false nextCursor: type: @@ -9964,6 +10046,45 @@ paths: type: boolean trace_revision: type: string + failure_source: + anyOf: + - type: string + enum: + - system + - agent + - type: "null" + agent_error: + anyOf: + - type: object + properties: + reason: + type: string + reported_at: + type: string + reporter: + type: object + properties: + user_id: + type: string + kind: + type: string + enum: + - api_key + - oauth + - session + id: + type: string + required: + - user_id + - kind + - id + additionalProperties: false + required: + - reason + - reported_at + - reporter + additionalProperties: false + - type: "null" required: - id - job_id @@ -9977,6 +10098,8 @@ paths: - artifact_state - finalizing - trace_revision + - failure_source + - agent_error additionalProperties: false "400": $ref: "#/components/responses/Error400" @@ -10285,6 +10408,87 @@ paths: required: - name additionalProperties: false + /api/v1/runs/{runId}/report-error: + post: + summary: Report an error for an active run + description: |- + Records a sanitized reason and authenticated reporter without changing run execution or releasing its concurrency slot. The agent may continue work and should then exit normally. An identical retry succeeds; a different replacement reason returns 409. Terminal runs reject reports. + + Requires the runs:write scope. + + Accepts a scoped bearer token or interactive owner session. + responses: + "200": + description: Report saved or identical retry acknowledged + content: + application/json: + schema: + type: object + properties: + reported: + type: boolean + const: true + reason: + type: string + retry: + type: boolean + required: + - reported + - reason + - retry + additionalProperties: false + "400": + $ref: "#/components/responses/Error400" + description: Invalid reason or request body + "401": + $ref: "#/components/responses/Error401" + "403": + $ref: "#/components/responses/Error403" + description: Missing runs:write authorization + "404": + $ref: "#/components/responses/Error404" + description: Run not found within the tenant + "409": + $ref: "#/components/responses/Error409" + description: Run is terminal or a different reason was already reported + "415": + $ref: "#/components/responses/Error415" + "429": + $ref: "#/components/responses/Error429" + "500": + $ref: "#/components/responses/Error500" + "503": + $ref: "#/components/responses/Error503" + operationId: post_api_v1_runs_runId_report_error + tags: + - runs + security: + - bearerAuth: [] + - cookieAuth: [] + x-required-scope: runs:write + x-owner-session-required: false + parameters: + - name: runId + in: path + required: true + schema: + type: string + format: uuid + pattern: ^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$ + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + reason: + type: string + minLength: 1 + maxLength: 2000 + required: + - reason + additionalProperties: false /api/v1/runs/{runId}/trace: get: summary: Get paginated run trace events diff --git a/packages/docs/work/results.mdx b/packages/docs/work/results.mdx index c70c721..7b8308b 100644 --- a/packages/docs/work/results.mdx +++ b/packages/docs/work/results.mdx @@ -10,3 +10,5 @@ Run pages return `items` and `nextCursor`; send that cursor for the next page. O Check that the intended trigger fired, the repository/task appears in the rendered prompt, and the browser/REST trace supports the reported result. A queued submission is not a successful execution. Browser run pages are at `https://app.factorize.sh/job-runs/RUN_ID`. Agent output stays in Factorize and is not automatically copied into Linear. For missing output or execution failures, call `get_run_diagnostics` and follow [failed-run troubleshooting](/troubleshooting/runs). + +Failed runs keep the existing `failed` state filter and expose `failure_source` as `system` or `agent`. Historical failed runs read as system failures. An agent-reported reason appears in run detail and history; execution observer diagnostics remain available separately, and artifact/cleanup errors are separate diagnostics. diff --git a/packages/docs/work/stop-retry.mdx b/packages/docs/work/stop-retry.mdx index ea99f0a..0b99402 100644 --- a/packages/docs/work/stop-retry.mdx +++ b/packages/docs/work/stop-retry.mdx @@ -9,4 +9,6 @@ Ask: “Inspect this run, stop it if it is still active, and verify its final st For a timed-out manual submission, repeat `invoke_job` with the **same** `idempotencyKey`. The response contains the original `invocationId`, `runId`, and `duplicate: true`. Retain the key you submitted: current `get_run` does not return `invocation.idempotency_key`. Never send its internal `invocation.claim_key` or a `manual:` prefix back as the key. If the key was lost, inspect existing runs before requesting another. +To report a problem without stopping the agent, call `report_run_error` (or `POST /api/v1/runs/{runId}/report-error`) with a reason, continue any useful remaining work, and then exit normally. The first reason is retained: identical retries are safe, while a different replacement reason returns a conflict. Terminal runs reject reports. A report never changes polling, evidence collection, cleanup, or slot accounting. + An intentional new run after failure needs a new key. First verify that the prior run is terminal, inspect whether it made external changes, and fix the cause. A retry may repeat those external effects: invocation deduplication is not transactional rollback of the agent's work. For `queue_full`, wait for capacity and reuse the key for the intended request. See [failed runs](/troubleshooting/runs).