diff --git a/apps/sim/lib/logs/execution/logging-session.test.ts b/apps/sim/lib/logs/execution/logging-session.test.ts index 4c4b625496d..69c1211fb30 100644 --- a/apps/sim/lib/logs/execution/logging-session.test.ts +++ b/apps/sim/lib/logs/execution/logging-session.test.ts @@ -6,6 +6,7 @@ import { } from '@sim/testing/mocks/billing-usage-reservation.mock' import { encryptionMock, encryptionMockFns } from '@sim/testing/mocks/encryption.mock' import { executionPayloadStoreMock } from '@sim/testing/mocks/execution-payload-store.mock' +import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { getMockPlatformEvent, telemetryMock, @@ -774,6 +775,35 @@ describe('LoggingSession completion retries', () => { expect(rawLogs[128].output.value).toBe('row-128:raw-secret') }) + it('labels a live block-log projection warning with the live phase', async () => { + const session = new LoggingSession('workflow-1', 'execution-live-phase', 'manual', 'req-1') + session.setResolvedSecretTraceRegistry( + createSecretRegistry([{ plaintext: 'raw-secret', replacement: '{{TOKEN}}' }]) + ) + + await session.projectBlockLogsForDisplay([ + { + blockId: 'function-1', + blockName: 'Function 1', + blockType: 'function', + startedAt: '2026-07-01T00:00:00.000Z', + endedAt: '2026-07-01T00:00:00.001Z', + durationMs: 1, + success: true, + executionOrder: 1, + output: { rows: Array.from({ length: 100_001 }, (_, index) => index) }, + displayResolvedSecretTraceProvenance: createDisplayProvenance([ + { plaintext: 'raw-secret', replacement: '{{TOKEN}}' }, + ]), + }, + ]) + + expect(getMockLogger('TraceSecretProjection').warn).toHaveBeenCalledWith( + 'Omitting trace content that could not be sanitized', + expect.objectContaining({ executionId: 'execution-live-phase', phase: 'live' }) + ) + }) + it('projects a numeric Function result produced by a resolved numeric secret', async () => { const session = new LoggingSession('workflow-1', 'execution-numeric-secret', 'manual', 'req-1') session.setResolvedSecretTraceRegistry( diff --git a/apps/sim/lib/logs/execution/logging-session.ts b/apps/sim/lib/logs/execution/logging-session.ts index a3e4f8d88c9..dc205891752 100644 --- a/apps/sim/lib/logs/execution/logging-session.ts +++ b/apps/sim/lib/logs/execution/logging-session.ts @@ -309,11 +309,13 @@ export class LoggingSession { private async projectRawTraceSpans( traceSpans: TraceSpan[], - registry = this.resolvedSecretTraceRegistry + registry = this.resolvedSecretTraceRegistry, + phase?: 'live' ): Promise { return projectTraceSpansForSecrets(traceSpans, { registry, store: this.getSecretProjectionStore(), + phase, }) } @@ -365,7 +367,8 @@ export class LoggingSession { output: envelope, }, ], - registry + registry, + 'live' ) const projected = this.readProjectedDisplayContent(projectedSpan?.output) return this.shouldClearLiveDisplay(registry) @@ -426,7 +429,8 @@ export class LoggingSession { ...(log.childTraceSpans ? { children: log.childTraceSpans } : {}), }, ], - registry + registry, + 'live' ) const display = this.readProjectedDisplayContent(projectedLog?.output) displayLogs.push({ @@ -1503,7 +1507,7 @@ export class LoggingSession { * structure with no content rather than unprojected values. */ async projectTraceSpansForLiveDisplay(traceSpans: TraceSpan[]): Promise { - return this.projectRawTraceSpans(traceSpans) + return this.projectRawTraceSpans(traceSpans, undefined, 'live') } async safeComplete(params: SessionCompleteParams = {}): Promise { diff --git a/apps/sim/lib/logs/execution/trace-secret-projection.test.ts b/apps/sim/lib/logs/execution/trace-secret-projection.test.ts index dc82652bd25..bd339ed6f7a 100644 --- a/apps/sim/lib/logs/execution/trace-secret-projection.test.ts +++ b/apps/sim/lib/logs/execution/trace-secret-projection.test.ts @@ -1176,24 +1176,86 @@ describe('projectTraceSpansForSecrets whole-trace content budget', () => { expect(projected[1].toolCalls).toHaveLength(50_000) }) - it('withholds the whole top-level span whose descendant crossed the budget', async () => { + it('keeps checked content under a single workflow root when a descendant crosses the budget', async () => { const projected = await projectTraceSpansForSecrets( [ - heavySpan('span-a', 'first'), createSpan({ - id: 'span-b', - output: { marker: 'parent top-secret' }, - children: [heavySpan('child-1', 'child one'), heavySpan('child-2', 'child two')], + id: 'workflow-execution', + output: { marker: 'root top-secret' }, + children: [ + heavySpan('block-a', 'first top-secret'), + createSpan({ + id: 'loop', + output: { marker: 'loop top-secret' }, + children: [ + heavySpan('iteration-1', 'second'), + heavySpan('iteration-2', 'third top-secret', { + children: [createSpan({ id: 'nested', output: { marker: 'nested top-secret' } })], + }), + createSpan({ id: 'iteration-3', output: { marker: 'fourth top-secret' } }), + ], + }), + createSpan({ id: 'block-b', output: { marker: 'fifth top-secret' } }), + ], }), + createSpan({ id: 'after-root', output: { marker: 'sixth top-secret' } }), ], { registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false } ) - expect(projected[0].output?.marker).toBe('first') - expect(projected[1]).not.toHaveProperty('output') - expect(projected[1].contentWithheld).toBe(true) - expect(projected[1].children?.map((child) => child.id)).toEqual(['child-1', 'child-2']) - expect(projected[1].children?.every((child) => !Object.hasOwn(child, 'output'))).toBe(true) + const [root, afterRoot] = projected + const [blockA, loop, blockB] = root.children ?? [] + const [iteration1, iteration2, iteration3] = loop.children ?? [] + + for (const [kept, marker] of [ + [root, 'root {{API_SECRET}}'], + [blockA, 'first {{API_SECRET}}'], + [loop, 'loop {{API_SECRET}}'], + [iteration1, 'second'], + ] as const) { + expect(kept.output?.marker).toBe(marker) + expect(kept).not.toHaveProperty('contentWithheld') + } + for (const withheld of [iteration2, iteration2.children?.[0], iteration3, blockB, afterRoot]) { + expect(withheld).toMatchObject({ contentWithheld: true }) + expect(withheld).not.toHaveProperty('output') + } expect(JSON.stringify(projected)).not.toContain('top-secret') + expect(warnMock).toHaveBeenCalledWith( + 'Trace content exceeded the projection budget; withholding the remaining content', + expect.objectContaining({ retainedSpanCount: 4, withheldSpanCount: 5 }) + ) + }) + + it('withholds every span when a descendant fails for any reason but the budget', async () => { + const projected = await projectTraceSpansForSecrets( + [ + createSpan({ + id: 'workflow-execution', + output: { marker: 'root' }, + children: [ + createSpan({ id: 'block-a', output: { marker: 'first' } }), + createSpan({ + id: 'block-b', + toolCalls: Array.from({ length: 100_001 }, (_, index) => ({ + name: `tool-${index}`, + duration: 0, + startTime: '2026-07-01T00:00:00.000Z', + endTime: '2026-07-01T00:00:00.000Z', + status: 'success', + })), + }), + ], + }), + ], + { registry: SECRET_REGISTRY, store: STORE, allowLargeValueWrites: false } + ) + + expect(projected[0]).not.toHaveProperty('output') + expect(projected[0].children?.[0]).not.toHaveProperty('output') + expect(warnMock).toHaveBeenCalledWith( + 'Trace secret projection failed; retaining structural spans only', + expect.objectContaining({ executionId: 'execution-1', phase: 'read' }) + ) }) }) diff --git a/apps/sim/lib/logs/execution/trace-secret-projection.ts b/apps/sim/lib/logs/execution/trace-secret-projection.ts index 0e0d7f1c733..5edc3638769 100644 --- a/apps/sim/lib/logs/execution/trace-secret-projection.ts +++ b/apps/sim/lib/logs/execution/trace-secret-projection.ts @@ -86,6 +86,8 @@ interface ProjectionContext { omittedContentFields: number logDetails: ProjectionLogDetails displayFieldsBySpanId?: ReadonlyMap + /** Set once the content walk runs out of budget; every span after that keeps only its structure. */ + contentBudgetFailure?: TraceContentBudgetError } interface OversizedHydrationGate { @@ -125,6 +127,8 @@ export interface ProjectTraceSpansForSecretsOptions { store: LargeValueStoreContext /** Read-only display projections omit ref-backed content instead of writing another ref. */ allowLargeValueWrites?: boolean + /** Labels warnings; defaults to `write`, or `read` when large-value writes are disabled. */ + phase?: ProjectionPhase /** * Synthetic spans that each carry one display-envelope value, keyed by span id. Warnings name * the envelope field instead of the synthetic span and count it apart from real spans. @@ -146,19 +150,22 @@ class TraceSecretProjectionError extends Error { class TraceContentBudgetError extends TraceSecretProjectionError {} /** - * Identifies the run behind a projection warning. `phase` says whether the run was being written - * to the log or a stored log was being read for display. + * Why a projection ran: `write` prepares the run's stored log, `read` displays a stored log, and + * `live` streams a running execution to a viewer. */ +type ProjectionPhase = 'write' | 'read' | 'live' + +/** Identifies the run behind a projection warning. */ interface ProjectionLogDetails { executionId?: string workflowId?: string workspaceId?: string - phase: 'read' | 'write' + phase: ProjectionPhase } function projectionLogDetails( store: LargeValueStoreContext, - phase: ProjectionLogDetails['phase'] + phase: ProjectionPhase ): ProjectionLogDetails { return { ...filterUndefined({ @@ -1046,16 +1053,52 @@ async function sanitizeTraceSpan( if (span.children !== undefined) { assertTraceStructureArrayFits(span.children, state) - const children: TraceSpan[] = [] - for (const child of span.children) { - children.push(await sanitizeTraceSpan(child, context, depth + 1, state, assertContentIsSafe)) - } - projected.children = children + projected.children = await sanitizeSiblingSpans( + span.children, + context, + depth + 1, + state, + assertContentIsSafe + ) } return projected } +/** + * A budget failure stops at the span that crossed it, so the spans checked before it keep their + * content at any depth, ancestors included: each span's own content passed the whole-trace check + * before its children were walked. The span that crossed and every span after it fall back to + * structure, which resumes the budget where the retained spans left it. + */ +async function sanitizeSiblingSpans( + spans: readonly TraceSpan[], + context: ProjectionContext, + depth: number, + state: BoundedTraceStructureState, + assertContentIsSafe: SpanContentAssertion +): Promise { + const projected: TraceSpan[] = [] + let index = 0 + for (; index < spans.length && !context.contentBudgetFailure; index += 1) { + const retainedStructureNodes = state.nodes + try { + projected.push( + await sanitizeTraceSpan(spans[index], context, depth, state, assertContentIsSafe) + ) + } catch (error) { + if (!(error instanceof TraceContentBudgetError)) throw error + context.contentBudgetFailure = error + state.nodes = retainedStructureNodes + break + } + } + const withheld = projectBoundedArray(spans.slice(index), (span) => + structuralOnlySpan(span, state, depth) + ) + return projected.concat(withheld) +} + function stripLegacyToolCallContent(call: ToolCall): ToolCall { return omit(call, ['input', 'output', 'error']) as ToolCall } @@ -1279,9 +1322,9 @@ function projectBoundedTraceSpans( state: BoundedTraceStructureState, depth: number ) => TraceSpan | undefined, - logDetails: ProjectionLogDetails, - state: BoundedTraceStructureState = { nodes: 0, truncated: false } + logDetails: ProjectionLogDetails ): TraceSpan[] { + const state: BoundedTraceStructureState = { nodes: 0, truncated: false } const projected = projectBoundedArray(traceSpans, (span) => project(span, state, 0)) if (state.truncated) { logger.warn( @@ -1292,14 +1335,12 @@ function projectBoundedTraceSpans( return projected } -/** Pass `state` to continue a structure budget that earlier spans of the same trace already spent. */ function structuralOnlyTraceSpans( traceSpans: TraceSpan[], - logDetails: ProjectionLogDetails, - state?: BoundedTraceStructureState + logDetails: ProjectionLogDetails ): TraceSpan[] { try { - return projectBoundedTraceSpans(traceSpans, structuralOnlySpan, logDetails, state) + return projectBoundedTraceSpans(traceSpans, structuralOnlySpan, logDetails) } catch (error) { logger.warn('Trace structure could not be safely traversed; omitting projected spans', { ...logDetails, @@ -1643,28 +1684,30 @@ async function assertPostTransformTraceSpansAreSafe( await verifyPostTransformLargeValues(refs, context, new Set()) } -/** - * Counts the real spans kept before `withheldFrom` and withheld from it on, naming the withheld - * display-envelope fields apart from them. - */ +/** Counts the real spans that kept or lost content, naming the withheld display-envelope fields apart. */ function describeWithheldContent( - traceSpans: readonly TraceSpan[], - withheldFrom: number, + projected: readonly TraceSpan[], displayFieldsBySpanId: ReadonlyMap | undefined ): { retainedSpanCount: number; withheldSpanCount: number; withheldDisplayFields?: string[] } { - let retainedDisplayFieldCount = 0 + let retainedSpanCount = 0 + let withheldSpanCount = 0 const withheldDisplayFields: string[] = [] - if (displayFieldsBySpanId?.size) { - for (let index = 0; index < traceSpans.length; index += 1) { - const displayField = displayFieldsBySpanId.get(traceSpans[index].id) - if (displayField === undefined) continue - if (index < withheldFrom) retainedDisplayFieldCount += 1 - else withheldDisplayFields.push(displayField) + const visit = (spans: readonly TraceSpan[]) => { + for (const span of spans) { + const displayField = displayFieldsBySpanId?.get(span.id) + if (displayField === undefined) { + if (span.contentWithheld) withheldSpanCount += 1 + else retainedSpanCount += 1 + } else if (span.contentWithheld) { + withheldDisplayFields.push(displayField) + } + if (span.children) visit(span.children) } } + visit(projected) return { - retainedSpanCount: withheldFrom - retainedDisplayFieldCount, - withheldSpanCount: traceSpans.length - withheldFrom - withheldDisplayFields.length, + retainedSpanCount, + withheldSpanCount, ...(withheldDisplayFields.length > 0 ? { withheldDisplayFields } : {}), } } @@ -1713,7 +1756,10 @@ export async function projectTraceSpansForSecrets( options: ProjectTraceSpansForSecretsOptions ): Promise { const allowLargeValueWrites = options.allowLargeValueWrites !== false - const logDetails = projectionLogDetails(options.store, allowLargeValueWrites ? 'write' : 'read') + const logDetails = projectionLogDetails( + options.store, + options.phase ?? (allowLargeValueWrites ? 'write' : 'read') + ) if (!options.registry?.isComplete()) { return structuralOnlyTraceSpans(traceSpans, logDetails) } @@ -1732,40 +1778,30 @@ export async function projectTraceSpansForSecrets( options.displayFieldsBySpanId ) - const projected: TraceSpan[] = [] const state: BoundedTraceStructureState = { nodes: 0, truncated: false } assertTraceStructureArrayFits(traceSpans, state) - const assertContentIsSafe = createSpanContentAssertion(traceSpans, context) - for (let index = 0; index < traceSpans.length; index += 1) { - const retainedStructureNodes = state.nodes - try { - projected.push( - await sanitizeTraceSpan(traceSpans[index], context, 0, state, assertContentIsSafe) - ) - } catch (error) { - if (!(error instanceof TraceContentBudgetError)) throw error - /** - * Every span already returned was sanitized and then passed the whole-trace check, - * descendants included, so only the span that crossed the budget and those after it - * lose their content. The fallback resumes the structure budget where the retained - * spans left it, so the trace keeps exactly the structure a whole-trace fallback would. - */ - logger.warn( - 'Trace content exceeded the projection budget; withholding the remaining content', - { - ...logDetails, - ...describeWithheldContent(traceSpans, index, options.displayFieldsBySpanId), - failure: describeProjectionFailure(error), - } - ) - return [ - ...projected, - ...structuralOnlyTraceSpans(traceSpans.slice(index), logDetails, { - nodes: retainedStructureNodes, - truncated: false, - }), - ] - } + const projected = await sanitizeSiblingSpans( + traceSpans, + context, + 0, + state, + createSpanContentAssertion(traceSpans, context) + ) + if (context.contentBudgetFailure) { + logger.warn( + 'Trace content exceeded the projection budget; withholding the remaining content', + { + ...logDetails, + ...describeWithheldContent(projected, options.displayFieldsBySpanId), + failure: describeProjectionFailure(context.contentBudgetFailure), + } + ) + } + if (state.truncated) { + logger.warn( + 'Trace structure exceeded projection limits; truncating projected structure', + logDetails + ) } return projected } catch (error) {