Skip to content
Merged
99 changes: 99 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -827,6 +827,105 @@ it.effect("record reads filter notification sources without dropping matching me
}).pipe(Effect.provide(layerTest)),
);

it.effect("shells flag a latest run triggered by a provider background wake", () =>
Effect.gen(function* () {
const store = yield* ProjectionStore.ProjectionStoreV2;
const threadId = yield* addRolledBackRecoveryCandidate("background-wake");
const initial = yield* store.getThreadProjection(threadId);
const at = initial.updatedAt;
let ordinal = 1;
for (const [id, source, outcome, expected] of [
["background-task", { kind: "background_task" }, "updated", true],
["monitor", { kind: "monitor" }, "updated", true],
["command", { kind: "command" }, "updated", true],
["command-finished", { kind: "command" }, "completed", false],
["monitor-stopped", { kind: "monitor" }, "cancelled", false],
["subagent", { kind: "subagent" }, "updated", false],
["delegated", { kind: "delegated_task", taskIds: [] }, "updated", false],
["steered", { kind: "background_task" }, "updated", false],
] as const) {
const messageId = MessageId.make(`message:background-wake:${id}`);
yield* store.apply({
id: EventId.make(`event:background-wake:${id}:message`),
type: "message.updated",
threadId,
occurredAt: at,
payload: {
id: messageId,
threadId,
runId: null,
nodeId: null,
createdBy: "agent",
creationSource: "server",
role: "user",
text: id,
attachments: [],
streaming: false,
createdAt: at,
updatedAt: at,
notification: { source, outcome, summary: "Background activity updated" },
...(id === "delegated"
? {
delegatedCompletion: {
parentRunId: initial.runs[0]!.id,
generation: 1,
taskIds: [],
},
}
: {}),
},
});
ordinal += 1;
yield* store.apply({
id: EventId.make(`event:background-wake:${id}:run`),
type: "run.created",
threadId,
occurredAt: at,
payload: {
...initial.runs[0]!,
id: RunId.make(`run:background-wake:${id}`),
ordinal,
userMessageId: messageId,
status: "completed",
completedAt: at,
},
});
if (id === "steered") {
yield* store.apply({
id: EventId.make("event:background-wake:steered:follow-up"),
type: "message.updated",
threadId,
occurredAt: at,
payload: {
id: MessageId.make("message:background-wake:steered:follow-up"),
threadId,
runId: RunId.make("run:background-wake:steered"),
nodeId: null,
createdBy: "user",
creationSource: "web",
role: "user",
text: "also fix X",
attachments: [],
streaming: false,
createdAt: at,
updatedAt: at,
},
});
}
const shells = [
yield* store.getThreadShell(threadId),
ProjectionStore.threadShellFromProjection(yield* store.getThreadProjection(threadId)),
];
assert.deepEqual(
shells.map((shell) => shell?.latestRunBackgroundWake),
[expected, expected],
id,
);
yield* assertShellIndexCoversShells();
}
}).pipe(Effect.provide(layerTest)),
);

it.effect("SQL queue readiness skips held runs without bypassing active work", () =>
Effect.gen(function* () {
const store = yield* ProjectionStore.ProjectionStoreV2;
Expand Down
50 changes: 50 additions & 0 deletions apps/server/src/orchestration-v2/ProjectionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1032,6 +1032,7 @@ type ShellThreadRow = {
readonly latest_run_started_at: string | null;
readonly latest_run_completed_at: string | null;
readonly latest_run_stop_requested: number;
readonly latest_run_background_wake: number;
readonly active_run_id: string | null;
readonly active_run_status: string | null;
readonly activity_run_status: string | null;
Expand All @@ -1043,6 +1044,7 @@ type ShellThreadRow = {
readonly blocking_run_started_at: string | null;
readonly blocking_run_completed_at: string | null;
readonly blocking_run_stop_requested: number;
readonly blocking_run_background_wake: number;
readonly blocking_failure_payload_json: string | null;
readonly pending_request_payload_json: string | null;
readonly pending_secret_request_payload_json: string | null;
Expand Down Expand Up @@ -1658,6 +1660,8 @@ export function threadShellFromProjection(
projection.turnItems.some(
(item) => item.runId === latestRun.id && item.type === "run_interrupt_request",
),
latestRunBackgroundWake:
latestRun != null && isBackgroundWakeRunForShell(latestRun, projection),
activeRunId: activeRun?.id ?? null,
blockedOnThreadWait: deriveBlockedOnThreadWait({ activeRun, turnItems: projection.turnItems }),
activityRunStatus: activityRun?.status ?? null,
Expand Down Expand Up @@ -1753,6 +1757,25 @@ function isSilentRunForShell(
);
}

const BACKGROUND_WAKE_SOURCE_KINDS = new Set(["background_task", "command", "monitor"]);

function isBackgroundWakeRunForShell(
run: OrchestrationV2ThreadProjection["runs"][number],
projection: Pick<OrchestrationV2ThreadProjection, "messages">,
): boolean {
const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
return (
message?.delegatedCompletion === undefined &&
// Only progress wakes; a terminal one ("Command … finished") is the work reporting its end.
message?.notification?.outcome === "updated" &&
BACKGROUND_WAKE_SOURCE_KINDS.has(message.notification.source.kind) &&
// A follow-up the user steered into the wake makes the run theirs.
!projection.messages.some(
(candidate) => candidate.runId === run.id && candidate.createdBy === "user",
)
);
}

function isInterruptibleRunForShell(run: OrchestrationV2ThreadProjection["runs"][number]): boolean {
return run.status === "preparing" || run.status === "starting" || run.status === "running";
}
Expand All @@ -1775,6 +1798,7 @@ type ShellThreadState = {
readonly latestRunStartedAt: DateTime.Utc | null;
readonly latestRunCompletedAt: DateTime.Utc | null;
readonly latestRunStopRequested: boolean;
readonly latestRunBackgroundWake: boolean;
readonly activeRunId: RunId | null;
readonly blockedOnThreadWait: boolean;
readonly activityRunStatus: ShellActivityRunStatus | null;
Expand Down Expand Up @@ -1935,6 +1959,7 @@ function shellFromState(input: {
latestRunStartedAt: input.state.latestRunStartedAt,
latestRunCompletedAt: input.state.latestRunCompletedAt,
latestRunStopRequested: input.state.latestRunStopRequested,
latestRunBackgroundWake: input.state.latestRunBackgroundWake,
activeRunId: input.state.activeRunId,
blockedOnThreadWait: input.state.blockedOnThreadWait,
activityRunStatus: input.state.activityRunStatus,
Expand Down Expand Up @@ -6267,6 +6292,25 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
LIMIT 1
)`;

// Mirrors isBackgroundWakeRunForShell. Stored sources keep the legacy encoding, where a
// command is background_command and a subagent is background_task with work "subagent".
const isBackgroundWakeShellRun = (alias: string) => {
const run = sql.literal(alias);
return sql`EXISTS (
SELECT 1 FROM orchestration_v2_projection_messages wake
WHERE wake.message_id = json_extract(${run}.payload_json, '$.userMessageId')
AND json_extract(wake.payload_json, '$.notification.source.kind')
IN ('background_task', 'background_command', 'command', 'monitor')
AND json_extract(wake.payload_json, '$.notification.source.work') IS NULL
AND json_extract(wake.payload_json, '$.notification.outcome') = 'updated'
AND json_extract(wake.payload_json, '$.delegatedCompletion') IS NULL
) AND NOT EXISTS (
SELECT 1 FROM orchestration_v2_projection_messages steer
WHERE steer.run_id = ${run}.run_id
AND json_extract(steer.payload_json, '$.createdBy') = 'user'
)`;
};

const selectShellThreadRows = (
threadId?: ThreadId,
{
Expand Down Expand Up @@ -6299,6 +6343,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
WHERE stop_request.thread_id = t.thread_id AND stop_request.run_id = presented.run_id
AND stop_request.type = 'run_interrupt_request'
) AS latest_run_stop_requested,
${isBackgroundWakeShellRun("presented")} AS latest_run_background_wake,
active.run_id AS active_run_id,
active.status AS active_run_status,
(
Expand Down Expand Up @@ -6356,6 +6401,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
WHERE stop_request.thread_id = t.thread_id AND stop_request.run_id = blocked.run_id
AND stop_request.type = 'run_interrupt_request'
) AS blocking_run_stop_requested,
${isBackgroundWakeShellRun("blocked")} AS blocking_run_background_wake,
(
SELECT item.payload_json
FROM orchestration_v2_projection_turn_items item
Expand Down Expand Up @@ -6928,6 +6974,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
? null
: DateTime.makeUnsafe(row.latest_run_completed_at);
let latestRunStopRequested = row.latest_run_stop_requested === 1;
let latestRunBackgroundWake = row.latest_run_background_wake === 1;
if (row.blocking_run_id !== null && row.blocking_failure_payload_json !== null) {
const blockingFailure = yield* decodeTurnItemPayload(
row.blocking_failure_payload_json,
Expand All @@ -6941,6 +6988,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
terminalFailureItem = blockingFailure;
latestRunId = RunId.make(row.blocking_run_id);
latestRunStopRequested = row.blocking_run_stop_requested === 1;
latestRunBackgroundWake = row.blocking_run_background_wake === 1;
latestRunStatus = "failed";
latestRunRequestedAt =
row.blocking_run_requested_at === null
Expand Down Expand Up @@ -6981,6 +7029,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
latestRunStartedAt,
latestRunCompletedAt,
latestRunStopRequested,
latestRunBackgroundWake,
activeRunId: row.active_run_id === null ? null : RunId.make(row.active_run_id),
blockedOnThreadWait: deriveBlockedOnThreadWait({
activeRun:
Expand Down Expand Up @@ -7251,6 +7300,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
activityRunStatus: state.activityRunStatus,
latestRunCompletedAt: state.latestRunCompletedAt,
latestRunStopRequested: state.latestRunStopRequested,
latestRunBackgroundWake: state.latestRunBackgroundWake,
pendingRuntimeRequest: state.pendingRuntimeRequest,
pendingBackgroundTasks: state.pendingBackgroundTasks,
})),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import * as Option from "effect/Option";
import { Atom, AtomRegistry } from "effect/reactivity";
import { expect, it, vi } from "vite-plus/test";

import { stubGlobal } from "../../test/render";
import { fakeTimers, stubGlobal } from "../../test/render";
import { RELEASED_COMPLETION_GRACE_MS } from "./ThreadNotificationCoordinator.logic";

const state = vi.hoisted(() => ({
mode: "notifications",
Expand Down Expand Up @@ -267,6 +268,69 @@ it("presents completion notifications and sound as soon as the shell completes",
expect(state.sound).toHaveBeenCalledWith("completion", expect.any(Function));
});

it.each([false, true])(
"presents one completion when a monitor ends with a final wake (disconnected: %s)",
async (disconnect) => {
await using app = setup();
using _timers = fakeTimers();
state.mode = "notifications-and-sound";
app.start();
const watching = (
turnId: string,
turnState: string,
completedAt: string | null,
latestRunBackgroundWake: boolean,
pendingBackgroundTasks: ReadonlyArray<{ taskId: string; kind: "monitor" }>,
) => ({
status: "live",
snapshot: Option.some({
threads: [
{
...toV2ThreadShell({
...thread,
latestTurn: { turnId, state: turnState, completedAt },
}),
latestRunBackgroundWake,
pendingBackgroundTasks,
},
],
}),
});
const monitor = [{ taskId: "watch", kind: "monitor" as const }];
const titles = () => TestNotification.sent.map(({ title }) => title);
state.shells.set(
"one",
watching("turn", "completed", "2026-09-13T07:00:01.000Z", false, monitor),
);
expect(titles()).toEqual(["Thread completed"]);

state.shells.set("one", watching("wake", "running", null, true, monitor));
state.shells.set(
"one",
watching("wake", "completed", "2026-09-13T07:00:09.000Z", true, monitor),
);
expect(titles()).toEqual(["Thread completed"]);

// The adapter drains the roster before the final wake starts.
state.shells.set("one", watching("wake", "completed", "2026-09-13T07:00:09.000Z", true, []));
expect(titles()).toEqual(["Thread completed"]);
if (disconnect) {
state.shells.set("one", {
...watching("wake", "completed", "2026-09-13T07:00:09.000Z", true, []),
status: "cached",
});
vi.advanceTimersByTime(RELEASED_COMPLETION_GRACE_MS);
expect(titles()).toEqual(["Thread completed"]);
}

state.shells.set("one", watching("final", "running", null, false, []));
vi.advanceTimersByTime(RELEASED_COMPLETION_GRACE_MS);
state.shells.set("one", watching("final", "completed", "2026-09-13T07:00:11.000Z", false, []));
expect(titles()).toEqual(["Thread completed", "Thread completed"]);
expect(state.sound).toHaveBeenCalledTimes(2);
},
);

it.each(["hasPendingApprovals", "hasPendingUserInput"] as const)(
"notifies for offline completion and %s once across reconnects",
async (flag) => {
Expand Down
Loading
Loading