From 309e80c0c7f4feaf53a27957f861541540b14b3f Mon Sep 17 00:00:00 2001 From: Tomasz Torz Date: Thu, 29 Oct 2026 19:13:41 +0100 Subject: [PATCH] feat(notifications): your phone stays quiet while you're at your computer The phone stays quiet while a web or desktop client connected to the environment has had input in the last 2 minutes and can alert there itself. Locking, sleeping, quitting or 2 minutes idle brings phone alerts back. Clients report an optional userIdleMs with their activity lease; the server's BackgroundPolicy.hasAttendedClient decides, and AgentAwarenessRelay publishes the state with notify: false inside the signed proof until the result changes (a new phase or a new run). The relay skips quiet states for APNs and FCM alerts while cards and Live Activities still update. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/ipc/DesktopIpcHandlers.ts | 2 + apps/desktop/src/ipc/channels.ts | 2 + .../src/ipc/methods/userPresence.test.ts | 114 +++++++++++++ apps/desktop/src/ipc/methods/userPresence.ts | 63 +++++++ apps/desktop/src/preload.ts | 6 + .../src/background/BackgroundPolicy.test.ts | 22 +++ .../server/src/background/BackgroundPolicy.ts | 21 +++ .../src/relay/AgentAwarenessRelay.test.ts | 75 ++++++++- apps/server/src/relay/AgentAwarenessRelay.ts | 41 ++++- .../ThreadNotificationCoordinator.tsx | 6 + .../lib/backgroundActivityReporter.test.ts | 60 +++++++ .../web/src/lib/backgroundActivityReporter.ts | 76 ++++++++- apps/web/src/threadNotifications.test.ts | 11 ++ apps/web/src/threadNotifications.ts | 10 ++ docs/internals/t3-connect.md | 5 + docs/operations/android-notifications.md | 2 + docs/user/mobile-notifications.md | 4 +- .../agentActivity/AgentActivityPublisher.ts | 2 + .../src/agentActivity/ApnsDeliveries.test.ts | 158 ++++++++++++++++++ .../relay/src/agentActivity/ApnsDeliveries.ts | 9 +- .../src/agentActivity/FcmDeliveries.test.ts | 53 ++++++ .../agentActivity/agentActivityAggregate.ts | 3 + .../src/agentActivity/agentActivityAlerts.ts | 20 ++- packages/contracts/src/background.ts | 4 + packages/contracts/src/ipc.ts | 6 + packages/contracts/src/relay.ts | 3 + 26 files changed, 754 insertions(+), 24 deletions(-) create mode 100644 apps/desktop/src/ipc/methods/userPresence.test.ts create mode 100644 apps/desktop/src/ipc/methods/userPresence.ts diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 35333ac3d33a4..b1837fc09eb3b 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -3,6 +3,7 @@ import * as Effect from "effect/Effect"; import { receiveProviderAuthCallback, cancelProviderAuthCallback } from "./methods/providerAuth.ts"; import * as DesktopIpc from "./DesktopIpc.ts"; import { installNotificationBadge } from "./methods/notificationBadge.ts"; +import { installUserPresence } from "./methods/userPresence.ts"; import { getClientSettings, setClientSettings } from "./methods/clientSettings.ts"; import { getConnectionCatalog, setConnectionCatalog } from "./methods/connectionCatalog.ts"; import { @@ -82,6 +83,7 @@ import { export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers")(function* () { const ipc = yield* DesktopIpc.DesktopIpc; yield* installNotificationBadge(); + yield* installUserPresence(); yield* PreviewIpc.installPreviewEventForwarding(); yield* ipc.handle(AppActivationIpc.setReady); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index bba4f6449b8c1..0b219b4c32ce3 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -1,5 +1,7 @@ export const PICK_FOLDER_CHANNEL = "desktop:pick-folder"; export const SET_NOTIFICATION_BADGE_CHANNEL = "desktop:set-notification-badge"; +export const GET_USER_IDLE_MS_CHANNEL = "desktop:get-user-idle-ms"; +export const USER_AWAY_CHANGED_CHANNEL = "desktop:user-away-changed"; export const PICK_PROJECT_FAVICON_CHANNEL = "desktop:pick-project-favicon"; export const PICK_THEME_FILES_CHANNEL = "desktop:pick-theme-files"; export const SET_THEME_CHANNEL = "desktop:set-theme"; diff --git a/apps/desktop/src/ipc/methods/userPresence.test.ts b/apps/desktop/src/ipc/methods/userPresence.test.ts new file mode 100644 index 0000000000000..688d2b66015c9 --- /dev/null +++ b/apps/desktop/src/ipc/methods/userPresence.test.ts @@ -0,0 +1,114 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; +import { expect, vi } from "vite-plus/test"; +import { it } from "@effect/vitest"; + +const native = vi.hoisted(() => ({ + isDestroyed: () => false, + webContents: { send: vi.fn() }, +})); +vi.mock("electron", () => ({ BrowserWindow: { getAllWindows: () => [native] } })); + +import * as ElectronPowerMonitor from "../../electron/ElectronPowerMonitor.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; +import { installUserPresence } from "./userPresence.ts"; + +function withUserPresence( + body: (machine: { + readonly readIdleMs: () => Effect.Effect; + readonly set: (next: { + idleSeconds?: number; + idleState?: ElectronPowerMonitor.ElectronIdleState; + }) => void; + readonly fire: (eventName: string) => void; + }) => Effect.Effect, + initialIdleSeconds = 42, +) { + native.webContents.send.mockClear(); + const handlers = new Map(); + const powerEvents = new Map void>(); + const machine = { + idleSeconds: initialIdleSeconds, + idleState: "active" as ElectronPowerMonitor.ElectronIdleState, + }; + return Effect.scoped( + Effect.gen(function* () { + yield* installUserPresence(); + yield* body({ + readIdleMs: () => + Effect.promise(async () => + handlers.get("desktop:get-user-idle-ms")!({ sender: { id: 1 } }, undefined), + ), + set: (next) => void Object.assign(machine, next), + fire: (eventName) => powerEvents.get(eventName)!(), + }); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + Layer.mock(ElectronPowerMonitor.ElectronPowerMonitor, { + getSystemIdleTime: Effect.sync(() => machine.idleSeconds), + getSystemIdleState: () => Effect.sync(() => machine.idleState), + onSimpleEvent: (eventName, listener) => + Effect.sync(() => void powerEvents.set(eventName, listener)), + }), + DesktopIpc.layer({ + handle: (channel, handler) => void handlers.set(channel, handler), + removeHandler: (channel) => void handlers.delete(channel), + on: vi.fn(), + removeAllListeners: vi.fn(), + }), + ), + ), + ); +} + +it.effect("reports the system idle time and treats a locked or sleeping computer as away", () => + withUserPresence(({ readIdleMs, set, fire }) => + Effect.gen(function* () { + expect(yield* readIdleMs()).toBe(42_000); + set({ idleState: "locked" }); + fire("lock-screen"); + expect(yield* readIdleMs()).toBeNull(); + set({ idleState: "active" }); + fire("suspend"); + expect(yield* readIdleMs()).toBeNull(); + fire("resume"); + expect(yield* readIdleMs()).toBe(42_000); + expect(native.webContents.send).toHaveBeenCalledTimes(3); + expect(native.webContents.send).toHaveBeenCalledWith("desktop:user-away-changed"); + }), + ), +); + +it.effect("tells windows as soon as someone comes back after a minute away", () => + withUserPresence(({ set }) => + Effect.gen(function* () { + for (const [idleSeconds, sends] of [ + [30, 0], + [0, 0], + [90, 0], + [1, 1], + ] as const) { + set({ idleSeconds }); + yield* TestClock.adjust("5 seconds"); + expect(native.webContents.send).toHaveBeenCalledTimes(sends); + } + }), + ), +); + +it.effect("leaves presence to window input until the idle clock has moved", () => + withUserPresence( + ({ readIdleMs, set }) => + Effect.gen(function* () { + expect(yield* readIdleMs()).toBeUndefined(); + set({ idleSeconds: 6 }); + expect(yield* readIdleMs()).toBe(6_000); + set({ idleSeconds: 0 }); + expect(yield* readIdleMs()).toBe(0); + }), + 0, + ), +); diff --git a/apps/desktop/src/ipc/methods/userPresence.ts b/apps/desktop/src/ipc/methods/userPresence.ts new file mode 100644 index 0000000000000..6d513c98696dd --- /dev/null +++ b/apps/desktop/src/ipc/methods/userPresence.ts @@ -0,0 +1,63 @@ +import * as Electron from "electron"; +import * as Effect from "effect/Effect"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; + +import * as ElectronPowerMonitor from "../../electron/ElectronPowerMonitor.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; +import { GET_USER_IDLE_MS_CHANNEL, USER_AWAY_CHANGED_CHANNEL } from "../channels.ts"; + +// Input in other apps reaches no renderer, so the OS idle clock decides; sample it +// to tell windows the moment someone comes back, not on their next heartbeat. +const AWAY_SECONDS = 60; + +export const installUserPresence = Effect.fn("desktop.ipc.installUserPresence")(function* () { + const ipc = yield* DesktopIpc.DesktopIpc; + const powerMonitor = yield* ElectronPowerMonitor.ElectronPowerMonitor; + let suspended = false; + const broadcastAwayChange = () => { + for (const window of Electron.BrowserWindow.getAllWindows()) { + if (!window.isDestroyed()) window.webContents.send(USER_AWAY_CHANGED_CHANNEL); + } + }; + yield* powerMonitor.onSimpleEvent("suspend", () => { + suspended = true; + broadcastAwayChange(); + }); + yield* powerMonitor.onSimpleEvent("resume", () => { + suspended = false; + broadcastAwayChange(); + }); + yield* powerMonitor.onSimpleEvent("lock-screen", broadcastAwayChange); + yield* powerMonitor.onSimpleEvent("unlock-screen", broadcastAwayChange); + let lastIdleSeconds = 0; + yield* powerMonitor.getSystemIdleTime.pipe( + Effect.tap((idleSeconds) => + Effect.sync(() => { + if (lastIdleSeconds >= AWAY_SECONDS && idleSeconds < lastIdleSeconds) { + broadcastAwayChange(); + } + lastIdleSeconds = idleSeconds; + }), + ), + Effect.repeat(Schedule.spaced("5 seconds")), + Effect.forkScoped, + ); + // Chromium reports 0 forever where it has no idle source (Wayland outside GNOME + // and KDE), which would read as someone always at the computer. Until the clock + // moves once, input in the window decides instead. + let idleClockMoved = false; + yield* ipc.handle( + DesktopIpc.makeIpcMethod({ + channel: GET_USER_IDLE_MS_CHANNEL, + payload: Schema.Undefined, + result: Schema.NullishOr(Schema.Number), + handler: Effect.fn("desktop.ipc.getUserIdleMs")(function* () { + if (suspended || (yield* powerMonitor.getSystemIdleState(1)) === "locked") return null; + const idleSeconds = yield* powerMonitor.getSystemIdleTime; + if (idleSeconds > 0) idleClockMoved = true; + return idleClockMoved ? idleSeconds * 1_000 : undefined; + }), + }), + ); +}); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 965b7db5d6d40..a63e1ef64b82f 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -91,6 +91,12 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.on(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); return () => ipcRenderer.removeListener(IpcChannels.SET_NOTIFICATION_BADGE_CHANNEL, handler); }, + getUserIdleMs: () => ipcRenderer.invoke(IpcChannels.GET_USER_IDLE_MS_CHANNEL), + onUserAwayChange: (listener) => { + const handler = () => listener(); + ipcRenderer.on(IpcChannels.USER_AWAY_CHANGED_CHANNEL, handler); + return () => ipcRenderer.removeListener(IpcChannels.USER_AWAY_CHANGED_CHANNEL, handler); + }, onTrackpadScrollEnd: (listener) => { const handler = () => listener(); ipcRenderer.on(IpcChannels.TRACKPAD_SCROLL_END_CHANNEL, handler); diff --git a/apps/server/src/background/BackgroundPolicy.test.ts b/apps/server/src/background/BackgroundPolicy.test.ts index d53fb35f57a0d..bef9bdab00201 100644 --- a/apps/server/src/background/BackgroundPolicy.test.ts +++ b/apps/server/src/background/BackgroundPolicy.test.ts @@ -10,6 +10,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerSettings from "../serverSettings.ts"; import * as BackgroundPolicy from "./BackgroundPolicy.ts"; @@ -69,6 +70,27 @@ function layerFor( } describe("BackgroundPolicy", () => { + it.effect("counts a computer as attended until its last input is two minutes old", () => + Effect.gen(function* () { + const policy = yield* BackgroundPolicy.BackgroundPolicy; + const session = AuthSessionId.make("session-1"); + const rpcClient = RpcClientId.make(1); + + yield* policy.reportClientActivity(session, rpcClient, makeReport()); + assert.equal(yield* policy.hasAttendedClient, false); + + yield* policy.reportClientActivity(session, rpcClient, makeReport({ userIdleMs: 100_000 })); + assert.equal(yield* policy.hasAttendedClient, true); + yield* TestClock.adjust("20 seconds"); + assert.equal(yield* policy.hasAttendedClient, false); + + yield* policy.reportClientActivity(session, rpcClient, makeReport({ userIdleMs: 0 })); + assert.equal(yield* policy.hasAttendedClient, true); + yield* policy.removeRpcClient(session, rpcClient); + assert.equal(yield* policy.hasAttendedClient, false); + }).pipe(Effect.provide(layerFor(nominalHostPower))), + ); + it.effect("removes all leases for a disconnected websocket connection", () => Effect.gen(function* () { const policy = yield* BackgroundPolicy.BackgroundPolicy; diff --git a/apps/server/src/background/BackgroundPolicy.ts b/apps/server/src/background/BackgroundPolicy.ts index 331b4c4f3365b..4a5bf1c70048c 100644 --- a/apps/server/src/background/BackgroundPolicy.ts +++ b/apps/server/src/background/BackgroundPolicy.ts @@ -36,10 +36,14 @@ export class BackgroundPolicy extends Context.Service< readonly snapshot: Effect.Effect; readonly shouldRunScopeWork: (scope: BackgroundScope) => Effect.Effect; readonly shouldRunOpportunisticWork: Effect.Effect; + readonly hasAttendedClient: Effect.Effect; } >()("t3/background/BackgroundPolicy") {} const DEFAULT_LEASE_TTL_MS = 45_000; +// Long enough to read a diff without touching the mouse, short enough that a +// turn finishing after the person walks away still reaches their phone. +const ATTENDED_USER_IDLE_MS = 2 * 60_000; const MAX_LEASE_TTL_MS = 120_000; export const MAX_CLIENT_ACTIVITY_LEASES_PER_RPC_CLIENT = 16; @@ -113,6 +117,15 @@ function isForegroundLease(lease: ClientActivityLease, now: DateTime.Utc): boole return isLeaseActive(lease, now) && lease.visible && (lease.focused || lease.recentlyInteracted); } +function isAttendedLease(lease: ClientActivityLease, now: DateTime.Utc): boolean { + return ( + isLeaseActive(lease, now) && + lease.lastUserInputAt !== undefined && + DateTime.toEpochMillis(now) - DateTime.toEpochMillis(lease.lastUserInputAt) < + ATTENDED_USER_IDLE_MS + ); +} + function leaseHasScope(lease: ClientActivityLease, scope: BackgroundScope): boolean { const key = scopeKey(scope); return lease.scopes.some((leaseScope) => scopeKey(leaseScope) === key); @@ -226,6 +239,9 @@ export const make = Effect.fn("background.policy.make")(function* () { visible: input.visible, focused: input.focused, recentlyInteracted: input.recentlyInteracted, + ...(input.userIdleMs !== undefined + ? { lastUserInputAt: DateTime.subtract(now, { milliseconds: input.userIdleMs }) } + : {}), ...(input.appState !== undefined ? { appState: input.appState } : {}), ...(input.lowPowerMode !== undefined ? { lowPowerMode: input.lowPowerMode } : {}), scopes: input.scopes, @@ -266,6 +282,10 @@ export const make = Effect.fn("background.policy.make")(function* () { (current) => current.shouldRunOpportunisticWork, ); + const hasAttendedClient = Effect.map(snapshot, (current) => + current.leases.some((lease) => isAttendedLease(lease, current.updatedAt)), + ); + yield* Effect.forever( Effect.sleep("15 seconds").pipe( Effect.andThen( @@ -291,6 +311,7 @@ export const make = Effect.fn("background.policy.make")(function* () { snapshot, shouldRunScopeWork, shouldRunOpportunisticWork, + hasAttendedClient, }); }); diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.ts index 7f0e86bd6cf49..de82e47eadc5f 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.test.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts @@ -15,6 +15,7 @@ import { TurnItemId, } from "@t3tools/contracts"; import { RelayAgentActivityState } from "@t3tools/contracts/relay"; +import { decodeRelayJwt } from "@t3tools/shared/relayJwt"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -29,6 +30,7 @@ import * as TestClock from "effect/testing/TestClock"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import { PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, @@ -92,8 +94,14 @@ function shell(overrides: Partial = {}): Orchestrati }; } -const PublishPayload = Schema.Struct({ state: Schema.NullOr(RelayAgentActivityState) }); +const PublishPayload = Schema.Struct({ + state: Schema.NullOr(RelayAgentActivityState), + proof: Schema.String, +}); const decodePublishPayload = Schema.decodeUnknownSync(Schema.fromJsonString(PublishPayload)); +const decodeSignedState = Schema.decodeUnknownSync( + Schema.Struct({ state: Schema.NullOr(RelayAgentActivityState) }), +); const unused = () => Effect.die("Unexpected test dependency call"); describe("startup agent activity", () => { @@ -144,6 +152,7 @@ const makeTestRelay = Effect.fnUntraced(function* ( readonly linked?: boolean; readonly readShell?: (threadId: ThreadId) => Effect.Effect; readonly domainEvents?: Stream.Stream; + readonly attended?: Ref.Ref; } = {}, ) { const values = new Map( @@ -230,6 +239,8 @@ const makeTestRelay = Effect.fnUntraced(function* ( readonly url: string; readonly authorization: string | null; readonly state: RelayAgentActivityState | null; + readonly notify: boolean | undefined; + readonly signedNotify: boolean | undefined; }> = []; const fetch: typeof globalThis.fetch = Object.assign( ( @@ -247,6 +258,8 @@ const makeTestRelay = Effect.fnUntraced(function* ( url: String(input), authorization: new Headers(init?.headers).get("authorization"), state: payload.state, + notify: payload.state?.notify, + signedNotify: decodeSignedState(decodeRelayJwt(payload.proof)).state?.notify, }); return Promise.resolve( options.respond?.(publications.length) ?? Response.json({ ok: true, deliveries: [] }), @@ -283,6 +296,11 @@ const makeTestRelay = Effect.fnUntraced(function* ( deletedAt: null, }), }), + Effect.provide( + Layer.mock(BackgroundPolicy.BackgroundPolicy, { + hasAttendedClient: options.attended ? Ref.get(options.attended) : Effect.succeed(false), + }), + ), Effect.provideService(FetchHttpClient.Fetch, fetch), Effect.provide(NodeCrypto.layer), ); @@ -354,6 +372,61 @@ describe("AgentAwarenessRelay", () => { }), ); + it.effect("signs quietness and keeps it until the published phase changes", () => + Effect.gen(function* () { + const attended = yield* Ref.make(true); + const { relay, currentShell, publications } = yield* makeTestRelay({ attended }); + yield* relay.publishThread(THREAD_ID); + yield* Ref.set(attended, false); + yield* Ref.set(currentShell, shell({ title: "Renamed thread" })); + yield* relay.publishThread(THREAD_ID); + yield* Ref.set(currentShell, shell({ title: "Renamed thread", status: "failed" })); + yield* relay.publishThread(THREAD_ID); + assert.deepStrictEqual( + publications.map(({ notify, signedNotify }) => ({ notify, signedNotify })), + [ + { notify: false, signedNotify: false }, + { notify: false, signedNotify: false }, + { notify: undefined, signedNotify: undefined }, + ], + ); + }), + ); + + it.effect("treats a later run as a new result, even in the same phase", () => + Effect.gen(function* () { + const attended = yield* Ref.make(true); + const { relay, currentShell, publications } = yield* makeTestRelay({ attended }); + yield* Ref.set(currentShell, shell({ activeRunId: RunId.make("run-a") })); + yield* relay.publishThread(THREAD_ID); + yield* Ref.set(attended, false); + yield* Ref.set(currentShell, shell({ activeRunId: RunId.make("run-b") })); + yield* relay.publishThread(THREAD_ID); + assert.deepStrictEqual( + publications.map(({ notify }) => notify), + [false, undefined], + ); + }), + ); + + it.effect("keeps a seen result quiet when relinking republishes it", () => + Effect.gen(function* () { + const attended = yield* Ref.make(true); + const { relay, secrets, publications } = yield* makeTestRelay({ attended }); + yield* relay.publishThread(THREAD_ID); + yield* Ref.set(attended, false); + yield* secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + new TextEncoder().encode("credential-2"), + ); + yield* relay.publishThread(THREAD_ID); + assert.deepStrictEqual( + publications.map(({ notify }) => notify), + [false, false], + ); + }), + ); + it.effect("republishes unchanged state with fresh credentials after relinking", () => Effect.gen(function* () { const { relay, secrets, publications } = yield* makeTestRelay(); diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index cbe7b35ef409e..4442560f294ac 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -38,6 +38,7 @@ import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as HttpApiClient from "effect/http-api/HttpApiClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import { isAgentActivityPublishingEnabledValue, PUBLISH_AGENT_ACTIVITY_SECRET, @@ -155,12 +156,17 @@ export const makeAgentAwarenessPublishWorker = Effect.fnUntraced(function* ( return { enqueue, drain: worker.drain }; }); -function agentAwarenessPublishIdentity(state: RelayAgentActivityState | null): string { +// Projected state carries no run, so without it a later run with the same title and phase +// looks unchanged and never alerts. +function agentAwarenessPublishIdentity( + state: RelayAgentActivityState | null, + runId: string | null, +): string { if (state === null) { return "null"; } const { updatedAt: _updatedAt, ...meaningfulState } = state; - return JSON.stringify(meaningfulState); + return JSON.stringify({ ...meaningfulState, runId }); } function resolveAgentActivityPublishingStartupState(input: { @@ -364,6 +370,7 @@ export const make = Effect.gen(function* () { const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; const threads = yield* ThreadManagement.ThreadManagementService; const projects = yield* ProjectService.ProjectService; + const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy; const crypto = yield* Crypto.Crypto; const scope = yield* Effect.scope; const cloudLinkKeyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(secrets); @@ -410,6 +417,9 @@ export const make = Effect.gen(function* () { // Confirm through the same worker so tombstones cannot race live updates; recovery clears the deadline. const publishConfirmDeadlines = new Map(); + // Last result published quietly per thread. It survives connection resets, whose republish + // would otherwise ring for a result the person already saw. + const quietResultByThread = new Map(); // Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts. const heldCompletions = new Set(); let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect = () => Effect.void; @@ -466,18 +476,29 @@ export const make = Effect.gen(function* () { const publishState = Effect.fn("relay.agent_activity.publish")(function* (input: { readonly projectId: string | null; readonly state: RelayAgentActivityState | null; + readonly runId: string | null; readonly reason: string; }) { if (cachedRelayClient === undefined) { cachedRelayClient = yield* makeRelayClient(relayConfig); } const relayClient = cachedRelayClient; + // Quiet until the result changes: the relay alerts on any republish, so a rename after + // the person walks away would ring for a result they saw. A later run is a new result. + const result = input.state && `${input.state.phase}:${input.runId ?? ""}`; + const notify = + !(yield* backgroundPolicy.hasAttendedClient) && + quietResultByThread.get(threadId) !== result; + const state = input.state && !notify ? { ...input.state, notify: false } : input.state; + // Before sending: a publish the relay stored but whose response timed out stays quiet. + if (result !== null && !notify) quietResultByThread.set(threadId, result); + else quietResultByThread.delete(threadId); const proof = yield* makePublishProof({ privateKey: cloudLinkKeyPair.privateKey, relayIssuer: relayConfig.issuer, environmentId, threadId, - state: input.state, + state, jti: yield* crypto.randomUUIDv4(), }); @@ -487,6 +508,7 @@ export const make = Effect.gen(function* () { projectId: input.projectId, statePhase: input.state?.phase ?? null, hasState: input.state !== null, + notify, reason: input.reason, }); @@ -497,7 +519,7 @@ export const make = Effect.gen(function* () { threadId, }, payload: { - state: input.state, + state, proof, }, }) @@ -532,7 +554,10 @@ export const make = Effect.gen(function* () { thread, project, }); - const publishIdentity = agentAwarenessPublishIdentity(snapshot.state); + const runId = Option.isSome(thread) + ? (thread.value.activeRunId ?? thread.value.latestRunId) + : null; + const publishIdentity = agentAwarenessPublishIdentity(snapshot.state, runId); const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef); if ( (snapshot.state?.phase === "completed" || snapshot.state?.phase === "failed") && @@ -557,7 +582,7 @@ export const make = Effect.gen(function* () { // alerts on every move into completed. const requiresConfirmation = (snapshot.state === null && - publishedStateByThread.get(threadId) !== agentAwarenessPublishIdentity(null)) || + publishedStateByThread.get(threadId) !== agentAwarenessPublishIdentity(null, null)) || snapshot.state?.phase === "completed"; if (requiresConfirmation) { if (snapshot.state?.phase === "completed") heldCompletions.add(threadId); @@ -603,12 +628,13 @@ export const make = Effect.gen(function* () { yield* publishState({ projectId: unsettled.projectId, state: completion, + runId, reason: unsettled.reason, }).pipe(withRelayClientTracing); heldCompletions.delete(threadId); publishConfirmDeadlines.set(threadId, nowMs + 5_000); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => - new Map(publishedStates).set(threadId, agentAwarenessPublishIdentity(completion)), + new Map(publishedStates).set(threadId, agentAwarenessPublishIdentity(completion, runId)), ); yield* schedulePublishConfirm(threadId); return; @@ -634,6 +660,7 @@ export const make = Effect.gen(function* () { yield* publishState({ projectId: snapshot.projectId, state: snapshot.state, + runId, reason: snapshot.reason, }).pipe(withRelayClientTracing); publishConfirmDeadlines.delete(threadId); diff --git a/apps/web/src/components/ThreadNotificationCoordinator.tsx b/apps/web/src/components/ThreadNotificationCoordinator.tsx index fa4fdbb672faa..3fb134755c5e1 100644 --- a/apps/web/src/components/ThreadNotificationCoordinator.tsx +++ b/apps/web/src/components/ThreadNotificationCoordinator.tsx @@ -17,11 +17,13 @@ import { } from "lucide-react"; import { getClientSettings, subscribeClientSettings } from "../hooks/useSettings"; +import { registerCanNotifyOutsideWindow } from "../lib/backgroundActivityReporter"; import { isWindowsPlatform } from "../lib/utils"; import { environmentSummaries } from "../state/presentation"; import { environmentShell } from "../state/shell"; import { environmentThreadShells } from "../state/threads"; import { + canNotifyOutsideWindow, hasDesktopNotifications, hasNotificationSound, playNotificationSound, @@ -340,6 +342,9 @@ export function startThreadNotifications( syncEnvironments(); }; const unsubscribeSettings = subscribeClientSettings(onSettings); + const unregisterCanNotify = registerCanNotifyOutsideWindow(() => + canNotifyOutsideWindow(getClientSettings().notificationMode), + ); const unsubscribeEnvironmentIds = registry.subscribe( environmentSummaries.environmentIdsAtom, () => { @@ -351,6 +356,7 @@ export function startThreadNotifications( return () => { unsubscribeSettings(); + unregisterCanNotify(); unsubscribeEnvironmentIds(); for (const entry of watched.values()) entry.unsubscribe(); watched.clear(); diff --git a/apps/web/src/lib/backgroundActivityReporter.test.ts b/apps/web/src/lib/backgroundActivityReporter.test.ts index ae4b980afe0a0..9cb8acb35e7d1 100644 --- a/apps/web/src/lib/backgroundActivityReporter.test.ts +++ b/apps/web/src/lib/backgroundActivityReporter.test.ts @@ -8,12 +8,45 @@ import * as SubscriptionRef from "effect/SubscriptionRef"; import * as TestClock from "effect/testing/TestClock"; import { vi } from "vite-plus/test"; +const reports = vi.hoisted((): Array<{ readonly userIdleMs?: number }> => []); +vi.mock("@t3tools/client-runtime/rpc", async (importOriginal) => { + const Effect = await import("effect/Effect"); + return { + ...(await importOriginal()), + request: (_tag: string, input: { readonly userIdleMs?: number }) => + Effect.sync(() => void reports.push(input)), + }; +}); + import { layer, observeBackgroundActivitySubscription, + resolveUserIdleMs, retainedBackgroundScopes, } from "./backgroundActivityReporter.ts"; +describe("resolveUserIdleMs", () => { + const away = { visible: false, focused: false, canNotify: true, pageIdleMs: 5_000 }; + it.each([ + ["a client that cannot alert reports no presence", { ...away, canNotify: false }, undefined], + ["a client that can alert reports page input", away, 5_000], + [ + "a focused window reports page input", + { ...away, visible: true, focused: true, canNotify: false }, + 5_000, + ], + [ + "a page that has not seen input reports no presence", + { ...away, pageIdleMs: undefined }, + undefined, + ], + ["the desktop's system idle clock wins", { ...away, systemIdleMs: 1_000 }, 1_000], + ["a locked computer is away", { ...away, systemIdleMs: null }, undefined], + ])("%s", (_name, input, expected) => { + expect(resolveUserIdleMs({ systemIdleMs: undefined, ...input })).toBe(expected); + }); +}); + describe("wasRecentlyInteracted", () => { it.effect("retains an observed subscription until its returned finalizer runs", () => Effect.gen(function* () { @@ -74,6 +107,7 @@ describe("layer", () => { const listeners = { addEventListener: target.addEventListener.bind(target), removeEventListener: target.removeEventListener.bind(target), + dispatchEvent: target.dispatchEvent.bind(target), }; vi.stubGlobal("window", { ...listeners, @@ -111,4 +145,30 @@ describe("layer", () => { expect(calls).toBe(2); }).pipe(Effect.scoped), ); + + it.effect("counts a moving pointer as presence, but not a motionless pointermove", () => + Effect.gen(function* () { + reports.length = 0; + const entries = yield* SubscriptionRef.make( + new Map([[EnvironmentId.make("environment-reporter-test"), {}]]), + ); + const registry = EnvironmentRegistry.EnvironmentRegistry.of({ + entries, + run: (_environmentId: EnvironmentId, effect: Effect.Effect) => effect, + } as unknown as EnvironmentRegistry.EnvironmentRegistry["Service"]); + const pointerMove = (movementX: number) => + window.dispatchEvent(Object.assign(new Event("pointermove"), { movementX, movementY: 0 })); + + yield* Layer.build( + layer.pipe(Layer.provide(Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, registry))), + ); + pointerMove(0); + yield* TestClock.adjust("25 seconds"); + expect(reports.at(-1)?.userIdleMs).toBeUndefined(); + + pointerMove(4); + yield* TestClock.adjust("25 seconds"); + expect(reports.at(-1)?.userIdleMs).toBeDefined(); + }).pipe(Effect.scoped), + ); }); diff --git a/apps/web/src/lib/backgroundActivityReporter.ts b/apps/web/src/lib/backgroundActivityReporter.ts index 70b34f9140cdc..c083512115ae0 100644 --- a/apps/web/src/lib/backgroundActivityReporter.ts +++ b/apps/web/src/lib/backgroundActivityReporter.ts @@ -82,10 +82,46 @@ function wasRecentlyInteracted(lastInteractionAtMs: number, observedAtMs: number ); } +// Set by the notification coordinator while it runs. +let canNotify = () => false; + +export function registerCanNotifyOutsideWindow(check: () => boolean): () => void { + canNotify = check; + return () => { + if (canNotify === check) canNotify = () => false; + }; +} + +// Presence silences the phone, so it only counts from a client that can alert +// the person itself: a focused window, or one able to alert outside it. +export function resolveUserIdleMs(input: { + readonly visible: boolean; + readonly focused: boolean; + readonly canNotify: boolean; + /** `undefined` without a working desktop idle clock, `null` while the computer is locked or asleep. */ + readonly systemIdleMs: number | null | undefined; + /** `undefined` until this page has seen input. */ + readonly pageIdleMs: number | undefined; +}): number | undefined { + const reachable = (input.visible && input.focused) || input.canNotify; + if (!reachable || input.systemIdleMs === null) return undefined; + const idleMs = input.systemIdleMs ?? input.pageIdleMs; + return idleMs === undefined ? undefined : Math.max(0, idleMs); +} + +const readSystemIdleMs = Effect.tryPromise(async () => window.desktopBridge?.getUserIdleMs()).pipe( + Effect.orElseSucceed(() => undefined), +); + function createActivityReport( environmentId: EnvironmentId, lastInteractionAtMs: number, observedAtMs: number, + presence: { + readonly visible: boolean; + readonly focused: boolean; + readonly userIdleMs: number | undefined; + }, ): ClientActivityReportInput { const scopes = [...BASELINE_SCOPES]; for (const entry of retainedScopes.values()) { @@ -97,10 +133,11 @@ function createActivityReport( environmentId, clientId: getClientId(), clientKind: resolveClientKind(), - visible: document.visibilityState === "visible", - focused: document.hasFocus(), + visible: presence.visible, + focused: presence.focused, recentlyInteracted: wasRecentlyInteracted(lastInteractionAtMs, observedAtMs), - appState: document.visibilityState === "visible" ? "active" : "background", + ...(presence.userIdleMs !== undefined ? { userIdleMs: presence.userIdleMs } : {}), + appState: presence.visible ? "active" : "background", scopes, ttlMs: LEASE_TTL_MS, observedAt: DateTime.makeUnsafe(observedAtMs), @@ -181,18 +218,35 @@ export const layer = Layer.effectDiscard( const reportRequests = yield* Queue.sliding(1); const requestReport = () => Queue.offerUnsafe(reportRequests, undefined); let lastInteractionAtMs = clock.currentTimeMillisUnsafe(); + // Loading or restoring a tab is not input: presence waits for a real event. + let lastInputAtMs: number | null = null; const recordInteraction = () => { const observedAtMs = clock.currentTimeMillisUnsafe(); const wasRecent = wasRecentlyInteracted(lastInteractionAtMs, observedAtMs); lastInteractionAtMs = observedAtMs; + lastInputAtMs = observedAtMs; if (!wasRecent) { requestReport(); } }; + // Chromium fires motionless pointermoves when layout shifts under a resting + // cursor, as on load or tab restore; only real movement is input. + const recordPointerMove = (event: PointerEvent) => { + if (event.movementX !== 0 || event.movementY !== 0) recordInteraction(); + }; const passiveListenerOptions = { passive: true } as const; const report = Effect.gen(function* () { const observedAtMs = yield* Clock.currentTimeMillis; + const visible = document.visibilityState === "visible"; + const focused = document.hasFocus(); + const userIdleMs = resolveUserIdleMs({ + visible, + focused, + canNotify: canNotify(), + systemIdleMs: yield* readSystemIdleMs, + pageIdleMs: lastInputAtMs === null ? undefined : observedAtMs - lastInputAtMs, + }); const entries = yield* SubscriptionRef.get(registry.entries); yield* Effect.forEach( entries.keys(), @@ -202,7 +256,11 @@ export const layer = Layer.effectDiscard( environmentId, request( WS_METHODS.serverReportClientActivity, - createActivityReport(environmentId, lastInteractionAtMs, observedAtMs), + createActivityReport(environmentId, lastInteractionAtMs, observedAtMs, { + visible, + focused, + userIdleMs, + }), ), ) // Not `ignore`: tearing down an environment mid-request interrupts its report, @@ -212,6 +270,10 @@ export const layer = Layer.effectDiscard( ); }).pipe(Effect.withSpan("web.backgroundActivity.report")); + yield* Effect.acquireRelease( + Effect.sync(() => window.desktopBridge?.onUserAwayChange(requestReport)), + (unsubscribe) => Effect.sync(() => unsubscribe?.()), + ); yield* Effect.acquireRelease( Effect.sync(() => { retainedScopeListeners.add(requestReport); @@ -219,7 +281,8 @@ export const layer = Layer.effectDiscard( window.addEventListener("focus", requestReport); window.addEventListener("blur", requestReport); window.addEventListener("online", requestReport); - window.addEventListener("pointermove", recordInteraction); + window.addEventListener("pointermove", recordPointerMove); + window.addEventListener("pointerdown", recordInteraction, passiveListenerOptions); window.addEventListener("keydown", recordInteraction); window.addEventListener("wheel", recordInteraction, passiveListenerOptions); window.addEventListener("touchstart", recordInteraction, passiveListenerOptions); @@ -231,7 +294,8 @@ export const layer = Layer.effectDiscard( window.removeEventListener("focus", requestReport); window.removeEventListener("blur", requestReport); window.removeEventListener("online", requestReport); - window.removeEventListener("pointermove", recordInteraction); + window.removeEventListener("pointermove", recordPointerMove); + window.removeEventListener("pointerdown", recordInteraction); window.removeEventListener("keydown", recordInteraction); window.removeEventListener("wheel", recordInteraction); window.removeEventListener("touchstart", recordInteraction); diff --git a/apps/web/src/threadNotifications.test.ts b/apps/web/src/threadNotifications.test.ts index 412edfbb43309..83b8e7baa9881 100644 --- a/apps/web/src/threadNotifications.test.ts +++ b/apps/web/src/threadNotifications.test.ts @@ -77,3 +77,14 @@ describe("playNotificationSound", () => { expect(audio.started).toBe(1); }); }); + +describe("canNotifyOutsideWindow", () => { + it("counts notification sound only once audio is unlocked", async () => { + const notifications = await import("./threadNotifications"); + expect(notifications.canNotifyOutsideWindow("sound")).toBe(false); + + notifications.unlockNotificationAudio(); + expect(notifications.canNotifyOutsideWindow("sound")).toBe(true); + expect(notifications.canNotifyOutsideWindow("off")).toBe(false); + }); +}); diff --git a/apps/web/src/threadNotifications.ts b/apps/web/src/threadNotifications.ts index 22ec58e0d7af0..f1ee9f2d787f1 100644 --- a/apps/web/src/threadNotifications.ts +++ b/apps/web/src/threadNotifications.ts @@ -85,6 +85,16 @@ const buffers = new Map>(); /** When each sound's current playback ends, on the audio context's clock. */ const playingUntil = new Map(); +// Sound counts only once a user gesture has unlocked audio, as playback requires. +export function canNotifyOutsideWindow(mode: NotificationMode): boolean { + return ( + (hasNotificationSound(mode) && audioContext?.state === "running") || + (hasDesktopNotifications(mode) && + typeof Notification !== "undefined" && + Notification.permission === "granted") + ); +} + /** Called from a gesture so browsers allow later background playback. */ export function unlockNotificationAudio() { audioContext ??= new AudioContext(); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 4c9bbcb780bee..603616eb64e6a 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -56,6 +56,11 @@ The relay holds the signing authority for mint requests. DPoP protects an honest exchange from credential reuse; it does not make a compromised relay signing key harmless. Keep that trust assumption explicit when changing the protocol. +The environment decides whether an agent state may ring the phone, because it +knows whether someone is at a computer. The `notify` flag lives inside the signed +`RelayAgentActivityState`, so nobody can make the relay alert for a quiet version. +Any path that decides an alert later, such as a queued delivery job, must treat quietness as part of the state's version. + Managed tunnels expose only a validated loopback HTTP origin. Link proof checks reject forwarded authority headers, and the relay resolves endpoints from its own managed allocations rather than a caller-supplied URL. Health and mint diff --git a/docs/operations/android-notifications.md b/docs/operations/android-notifications.md index 04ba16d678c50..5d31ad1df93c9 100644 --- a/docs/operations/android-notifications.md +++ b/docs/operations/android-notifications.md @@ -70,6 +70,8 @@ Supported states are `running`, `approval`, `input`, `completed`, `failed`, and With the app in the foreground, Android suppresses an alert only for the thread currently on screen, matching iOS notification presentation; alerts for other threads still show. Activity cards still update in the foreground and retain finished results silently. Check that completion stays quiet while its thread is open, alerts while another screen is open, alerts after backgrounding, and that retrying a suppressed alert does not show it later. This uses the app lifecycle and route on the receiving phone, not thread visibility on other clients. +The environment publishes alerts as quiet while someone is at a computer connected to it: a web or desktop client with input in the last two minutes that can alert there (a focused window, notification sound, or permitted desktop notifications). Activity cards still update. When testing real agent turns through the relay, lock or leave that computer, or close T3 Code on it, or alerts will correctly stay silent. The local watcher below derives its own states and ignores this, so it alerts either way. + With ongoing activity enabled, verify two threads entering approval/input together produce one `2 agents need attention` alert, and two observed active threads completing/failing together produce one `2 agents finished` alert. The body lists their titles. The relay shares iOS transition selection and retains its delivered baseline when work finishes; publishing the same states again must not produce another alert. Grouped alerts open the aggregate’s priority thread; individual alerts retain their thread link. Verify an expanded card with five threads, attention/failure priority, project names and statuses. When all work finishes, the card should show **Agent work completed** or **Agent work failed**, lose its ongoing/promotion flag, and expire 15 minutes after the newest displayed result. Replays must not extend that deadline. Quiet running work uses the relay’s two-hour state lifetime; approval/input states use 24 hours. Reopen the same signed-in app and confirm existing alerts and dismissal survive, with a silent aggregate replay on cold start or a foreground after at least 60 seconds. Also check empty replays remove an orphaned card and completions older than two minutes never alert, even with ongoing activity disabled. diff --git a/docs/user/mobile-notifications.md b/docs/user/mobile-notifications.md index 9c5e30ef76074..c85027342524b 100644 --- a/docs/user/mobile-notifications.md +++ b/docs/user/mobile-notifications.md @@ -4,7 +4,9 @@ Sign in to T3 Connect, link your environments, and enable **Device Notifications Enable **Ongoing Agent Activity** on Android or **Live Activity Updates** on iOS to follow work without opening the app. Finished results remain visible for up to 15 minutes. You can dismiss an Android activity card without disabling alerts; turn off ongoing activity in Settings to stop future cards. -Ordinary alerts stay quiet while the mobile app is in the foreground. Ongoing activity continues to update. Viewing a thread on another device does not silence your phone's alerts. +Ordinary alerts stay quiet while the mobile app is in the foreground. Ongoing activity continues to update. + +Alerts also stay quiet while you're using T3 Code on a computer connected to the same environment. Your phone starts alerting again once that computer has gone two minutes without input, locks, or sleeps, or when you close T3 Code on it. If notification sounds and desktop notifications are both off on the computer, only a focused T3 Code window counts. In a browser, only activity in the T3 Code tab counts. Android notifications require Android 7.0 or newer and Google Play services. Android 16 and newer can promote ongoing activity to a Live Update, subject to system settings and device support. Other devices show a regular ongoing notification. Android 7's battery-saving modes can delay removal of expired cards. diff --git a/infra/relay/src/agentActivity/AgentActivityPublisher.ts b/infra/relay/src/agentActivity/AgentActivityPublisher.ts index 6ef258acdb6d3..a0b1248914bd7 100644 --- a/infra/relay/src/agentActivity/AgentActivityPublisher.ts +++ b/infra/relay/src/agentActivity/AgentActivityPublisher.ts @@ -13,6 +13,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import { allowsAlert } from "./agentActivityAlerts.ts"; import { isTerminalPhase } from "./agentActivityPayloads.ts"; export { isExpiredAgentActivityState } from "./agentActivityPayloads.ts"; @@ -205,6 +206,7 @@ const make = Effect.gen(function* () { "relay.environment_id": input.environmentId, "relay.thread_id": input.threadId, "relay.agent_activity.phase": input.state?.phase ?? "deleted", + "relay.agent_activity.notify": !input.state || allowsAlert(input.state), }); if (input.state) { // Retain terminal rows until cron pruning so completed threads remain visible alongside active ones. diff --git a/infra/relay/src/agentActivity/ApnsDeliveries.test.ts b/infra/relay/src/agentActivity/ApnsDeliveries.test.ts index b2445b0ab4c08..c5bf936be64ce 100644 --- a/infra/relay/src/agentActivity/ApnsDeliveries.test.ts +++ b/infra/relay/src/agentActivity/ApnsDeliveries.test.ts @@ -1166,6 +1166,164 @@ describe("fast completion delivery", () => { }); }); +describe("quiet state delivery", () => { + const attendedRow = aggregate.activities[0]!; + const awayRow = { + ...attendedRow, + environmentId: "away-env" as RelayAgentActivityState["environmentId"], + threadId: "away-thread" as RelayAgentActivityState["threadId"], + threadTitle: "Away thread", + }; + const running = { ...aggregate, activeCount: 2, activities: [attendedRow, awayRow] }; + const device = { ...target, last_aggregate_json: JSON.stringify(running) }; + const finish = ( + rows: ReadonlyArray, + quiet: ReadonlyArray, + ) => ({ + ...aggregate, + activeCount: 2 - rows.length, + activities: running.activities.map((row) => ({ + ...row, + ...(rows.includes(row) ? { phase: "completed" as const, status: "Done" } : {}), + ...(quiet.includes(row) ? { notify: false } : {}), + })), + }); + const statesFor = (done: ReturnType): RelayAgentActivityState[] => + done.activities.map((row) => ({ + ...state, + environmentId: row.environmentId, + threadId: row.threadId, + threadTitle: row.threadTitle, + phase: row.phase, + updatedAt: row.updatedAt, + ...(row.notify === false ? { notify: false } : {}), + })); + const jobFor = ( + payload: + | { kind: "live_activity_update"; aggregate: ReturnType } + | { kind: "push_notification"; state: RelayAgentActivityState }, + ) => + signApnsDeliveryJob({ + secret: config.apnsDeliveryJobSigningSecret, + payload: makeApnsDeliveryJobPayload({ + userId: target.user_id, + deviceId: target.device_id, + createdAt: state.updatedAt, + expiresAt: "1970-01-01T00:10:00.000Z", + jobId: "job-quiet", + ...(payload.kind === "live_activity_update" + ? { + kind: payload.kind, + token: "activity-token", + aggregate: payload.aggregate, + alert: { title: "2 agents finished", body: "Thread, Away thread" }, + } + : { + kind: payload.kind, + token: "apns-device-token", + aggregate: null, + notification: { + title: "Thread", + body: "Done: Project", + environmentId: payload.state.environmentId, + threadId: payload.state.threadId, + deepLink: "/", + phase: payload.state.phase, + updatedAt: payload.state.updatedAt, + }, + }), + }), + }); + const deliver = ( + job: SignedApnsDeliveryJob, + activityStates: ReadonlyArray, + currentTargets: ReadonlyArray = [device], + ) => { + const requests: HttpClientRequest.HttpClientRequest[] = []; + const attempts: Array = []; + return Effect.gen(function* () { + const deliveries = yield* ApnsDeliveries.ApnsDeliveries; + yield* deliveries.processSignedJob(job); + return { requests, attempts }; + }).pipe( + Effect.provide( + layerFor({ + attempts, + config: signingConfig, + currentTargets: [...currentTargets], + activityStates: [...activityStates], + execute: (request) => + Effect.sync(() => { + requests.push(request); + return HttpClientResponse.fromWeb(request, new Response("", { status: 200 })); + }), + }), + ), + ); + }; + + it.effect.each([ + { label: "only the quiet state finishes", finished: [attendedRow], alertTitle: undefined }, + { + label: "another state finishes too", + finished: [attendedRow, awayRow], + alertTitle: "Away thread", + }, + ])("updates the card and alerts only for loud states when $label", ({ finished, alertTitle }) => { + const queuedJobs: SignedApnsDeliveryJob[] = []; + return Effect.gen(function* () { + const deliveries = yield* ApnsDeliveries.ApnsDeliveries; + yield* deliveries.sendForTarget({ + target: device, + aggregate: finish(finished, [attendedRow]), + nowMs: 0, + }); + expect(queuedJobs).toHaveLength(1); + expect(queuedJobs[0]?.payload.alert?.title).toBe(alertTitle); + }).pipe(Effect.provide(layerFor({ attempts: [], queuedJobs }))); + }); + + it.effect("re-picks a queued alert at delivery without the quiet state", () => { + const done = finish([attendedRow, awayRow], [attendedRow]); + return Effect.gen(function* () { + const { requests } = yield* deliver( + jobFor({ kind: "live_activity_update", aggregate: done }), + statesFor(done), + ); + const body = requests[0]?.body; + expect(body?._tag).toBe("Uint8Array"); + if (body?._tag !== "Uint8Array") return; + expect(JSON.parse(new TextDecoder().decode(body.body)).aps.alert).toMatchObject({ + title: awayRow.threadTitle, + }); + }); + }); + + const completed = { ...state, phase: "completed" as const, headline: "Done" }; + it.effect.each([ + { + label: "card update", + job: jobFor({ kind: "live_activity_update", aggregate: finish([attendedRow, awayRow], []) }), + states: statesFor(finish([attendedRow, awayRow], [attendedRow])), + targets: [device], + }, + { + label: "push", + job: jobFor({ kind: "push_notification", state: completed }), + states: [{ ...completed, notify: false }], + targets: [{ ...target, push_token: "apns-device-token" }], + }, + ])("skips a queued $label for a version republished quietly since", ({ job, states, targets }) => + Effect.gen(function* () { + const { requests, attempts } = yield* deliver(job, states, targets); + expect(requests).toHaveLength(0); + expect(attempts).toMatchObject([ + { sourceJobId: "job-quiet", apnsReason: "Stale agent activity state skipped." }, + ]); + }), + ); +}); + describe("signed APNs registration metadata", () => { it.effect.each( (["live_activity_update", "push_notification"] as const).flatMap((kind) => diff --git a/infra/relay/src/agentActivity/ApnsDeliveries.ts b/infra/relay/src/agentActivity/ApnsDeliveries.ts index 6e0e4bdc6e236..4297f9c1a5ba6 100644 --- a/infra/relay/src/agentActivity/ApnsDeliveries.ts +++ b/infra/relay/src/agentActivity/ApnsDeliveries.ts @@ -46,6 +46,7 @@ import { alertForAttentionTransition, alertForNewlyTerminal, alertForTerminalAggregate, + allowsAlert, rowPhasesChanged, shouldAlertForActivity, } from "./agentActivityAlerts.ts"; @@ -548,11 +549,13 @@ const make = Effect.gen(function* () { threadId: input.threadId, }) .pipe( + // A version republished quietly since the job was queued must not ring. Effect.map( (current) => current !== null && current.phase === input.phase && - current.updatedAt === input.updatedAt, + current.updatedAt === input.updatedAt && + allowsAlert(current), ), Effect.catchCause((cause) => Effect.logWarning("agent-activity state recheck failed; allowing queued delivery", { @@ -576,12 +579,14 @@ const make = Effect.gen(function* () { current, ]), ); + // A quiet flip is a new version; its own job carries the right alert. return input.aggregate.activities.every((row) => { const current = currentByThread.get(`${row.environmentId}\u0000${row.threadId}`); return ( current !== undefined && current.phase === row.phase && - current.updatedAt === row.updatedAt + current.updatedAt === row.updatedAt && + allowsAlert(current) === allowsAlert(row) ); }); }), diff --git a/infra/relay/src/agentActivity/FcmDeliveries.test.ts b/infra/relay/src/agentActivity/FcmDeliveries.test.ts index 21df615900f30..c57758f2c1ae0 100644 --- a/infra/relay/src/agentActivity/FcmDeliveries.test.ts +++ b/infra/relay/src/agentActivity/FcmDeliveries.test.ts @@ -606,6 +606,59 @@ describe("delivery policy regressions", () => { }); }); +describe("quietly published states", () => { + const publishQuietly = (h: ReturnType, next: RelayAgentActivityState) => { + h.current.state = { ...next, notify: false }; + }; + + it.effect("update the ongoing card silently and never alert for that state later", () => { + const h = harness(); + return Effect.gen(function* () { + const d = yield* FcmDeliveries.FcmDeliveries; + yield* d.process(h.job); + const olderJob = { ...h.job, state: { ...state } }; + publishQuietly(h, { ...state, phase: "completed" }); + yield* d.process({ ...h.job, state: h.current.state }); + expect(h.sent.at(-1)).toMatchObject({ alert: false, data: { active: "false" } }); + yield* d.process({ ...h.job, state: h.current.state }); + yield* d.process(olderJob); + expect(h.sent.filter((x) => x.alert)).toHaveLength(0); + }).pipe(Effect.provide(h.layer)); + }); + + it.effect("still alert for another environment finishing at the same time", () => { + const h = harness(); + const other = { + ...state, + environmentId: EnvironmentId.make("other"), + threadId: ThreadId.make("other"), + threadTitle: "Other thread", + }; + h.current.target.last_aggregate_json = encodeJson(aggregateFor([state, other])); + publishQuietly(h, { ...state, phase: "completed" }); + h.current.otherStates = [{ ...other, phase: "completed" }]; + return Effect.gen(function* () { + const d = yield* FcmDeliveries.FcmDeliveries; + yield* d.process({ ...h.job, state: h.current.state }); + expect(h.sent[0]).toMatchObject({ alert: true, data: { alert_title: "Other thread" } }); + }).pipe(Effect.provide(h.layer)); + }); + + it.effect("do not alert with ongoing activity disabled", () => { + const h = harness(); + publishQuietly(h, { ...state, phase: "completed" }); + h.current.target.preferences_json = encodeJson({ + ...preferences, + liveActivitiesEnabled: false, + }); + return Effect.gen(function* () { + const d = yield* FcmDeliveries.FcmDeliveries; + yield* d.process({ ...h.job, state: h.current.state }); + expect(h.sent.filter((x) => x.alert)).toHaveLength(0); + }).pipe(Effect.provide(h.layer)); + }); +}); + describe("notification-only environments", () => { it.effect("alerts independently while another environment has a live card", () => { const h = harness(); diff --git a/infra/relay/src/agentActivity/agentActivityAggregate.ts b/infra/relay/src/agentActivity/agentActivityAggregate.ts index 99fdcf71ef90d..55e417014fc38 100644 --- a/infra/relay/src/agentActivity/agentActivityAggregate.ts +++ b/infra/relay/src/agentActivity/agentActivityAggregate.ts @@ -42,6 +42,9 @@ function aggregateRowForState(state: RelayAgentActivityState) { status: statusForPhase(state.phase), updatedAt: state.updatedAt, deepLink: state.deepLink, + // Omitted when true: shouldUpdateLiveActivity compares stored aggregates + // as JSON, and an explicit `notify: true` would read as a change. + ...(state.notify === false ? { notify: false } : {}), }; } diff --git a/infra/relay/src/agentActivity/agentActivityAlerts.ts b/infra/relay/src/agentActivity/agentActivityAlerts.ts index c6008950e95e8..b58fa96f56735 100644 --- a/infra/relay/src/agentActivity/agentActivityAlerts.ts +++ b/infra/relay/src/agentActivity/agentActivityAlerts.ts @@ -26,6 +26,10 @@ type TransitionInput = { readonly preferences: RelayAgentAwarenessPreferences | null; }; +export function allowsAlert(activity: { readonly notify?: boolean | undefined }): boolean { + return activity.notify !== false; +} + function rowKey(row: RelayAgentActivityAggregateRow): string { return JSON.stringify([row.environmentId, row.threadId]); } @@ -45,12 +49,13 @@ function isAttentionPhase(phase: string): boolean { return phase === "waiting_for_approval" || phase === "waiting_for_input"; } -function alertAllowedForPhase( +function alertAllowedFor( preferences: RelayAgentAwarenessPreferences | null, - phase: string, + activity: { readonly phase: string; readonly notify?: boolean | undefined }, ): boolean { + if (!allowsAlert(activity)) return false; if (preferences === null) return true; - switch (phase) { + switch (activity.phase) { case "waiting_for_approval": return preferences.notifyOnApproval; case "waiting_for_input": @@ -74,7 +79,7 @@ export function attentionTransitionRows(input: TransitionInput) { (row) => isAttentionPhase(row.phase) && !previouslyAttention.has(rowKey(row)) && - alertAllowedForPhase(input.preferences, row.phase), + alertAllowedFor(input.preferences, row), ); } @@ -107,7 +112,7 @@ export function terminalTransitionRows( input.includeUnobserved, ).filter((row) => { return ( - alertAllowedForPhase(input.preferences, row.phase) && + alertAllowedFor(input.preferences, row) && isFreshTerminalNotification(row.updatedAt, input.nowMs) ); }); @@ -143,18 +148,19 @@ export function alertForTerminalAggregate(input: { }): AgentActivityAlert | null { const row = input.aggregate?.activities[0]; if (!row || (row.phase !== "completed" && row.phase !== "failed")) return null; - return alertAllowedForPhase(input.preferences, row.phase) ? alertForActivityRows([row]) : null; + return alertAllowedFor(input.preferences, row) ? alertForActivityRows([row]) : null; } export function shouldAlertForActivity(input: { readonly phase: RelayAgentActivityAggregateRow["phase"]; readonly updatedAt: string; + readonly notify?: boolean | undefined; readonly preferences: RelayAgentAwarenessPreferences | null; readonly nowMs: number; }): boolean { return ( input.preferences?.notificationsEnabled === true && - alertAllowedForPhase(input.preferences, input.phase) && + alertAllowedFor(input.preferences, input) && ((input.phase !== "completed" && input.phase !== "failed") || isFreshTerminalNotification(input.updatedAt, input.nowMs)) ); diff --git a/packages/contracts/src/background.ts b/packages/contracts/src/background.ts index 8bed6d52bdd49..449f4a89c84a5 100644 --- a/packages/contracts/src/background.ts +++ b/packages/contracts/src/background.ts @@ -71,6 +71,9 @@ export const ClientActivityReportInput = Schema.Struct({ visible: Schema.Boolean, focused: Schema.Boolean, recentlyInteracted: Schema.Boolean, + // How long since the person last used this computer. Omitted when they are + // known to be away (screen locked, asleep) or the client cannot tell. + userIdleMs: Schema.optionalKey(Schema.Number.check(Schema.isGreaterThanOrEqualTo(0))), appState: Schema.optionalKey(Schema.Literals(["active", "inactive", "background", "unknown"])), lowPowerMode: Schema.optionalKey(BackgroundBooleanState), scopes: Schema.Array(BackgroundScope), @@ -87,6 +90,7 @@ export const ClientActivityLease = Schema.Struct({ visible: Schema.Boolean, focused: Schema.Boolean, recentlyInteracted: Schema.Boolean, + lastUserInputAt: Schema.optionalKey(Schema.DateTimeUtc), appState: Schema.optionalKey(Schema.Literals(["active", "inactive", "background", "unknown"])), lowPowerMode: Schema.optionalKey(BackgroundBooleanState), scopes: Schema.Array(BackgroundScope), diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index f460619f11355..57a159f1c54d7 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1018,6 +1018,12 @@ export interface DesktopBridge { getClientPlatform: () => string; setNotificationBadge: (badge: { count: number; image: string | null }) => Promise; onNotificationBadgeClear: (listener: () => void) => () => void; + /** + * Milliseconds since any input on this computer, null while it is locked or asleep, + * or undefined until the OS idle clock has moved once. + */ + getUserIdleMs: () => Promise; + onUserAwayChange: (listener: () => void) => () => void; onTrackpadScrollEnd: (listener: () => void) => () => void; /** * Renderer `window.focus()` cannot raise an Electron window, and on Wayland diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 4dbb6a199f472..0b3fd598e9644 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -127,6 +127,8 @@ export const RelayAgentActivityState = Schema.Struct({ modelTitle: TrimmedNonEmptyString, updatedAt: TrimmedNonEmptyString, deepLink: TrimmedNonEmptyString, + // False when published while someone sat at a computer: cards update, nothing alerts. + notify: Schema.optional(Schema.Boolean), }); export type RelayAgentActivityState = typeof RelayAgentActivityState.Type; @@ -140,6 +142,7 @@ export const RelayAgentActivityAggregateRow = Schema.Struct({ status: TrimmedNonEmptyString, updatedAt: TrimmedNonEmptyString, deepLink: TrimmedNonEmptyString, + notify: Schema.optional(Schema.Boolean), }); export type RelayAgentActivityAggregateRow = typeof RelayAgentActivityAggregateRow.Type;