diff --git a/src/per_isolate/webstreams/common.ts b/src/per_isolate/webstreams/common.ts index 714692d96e8..d96475e9b53 100644 --- a/src/per_isolate/webstreams/common.ts +++ b/src/per_isolate/webstreams/common.ts @@ -105,7 +105,7 @@ function toStringTagProperty(tag: string): PropertyDescriptor { const kMaxUint64 = 0xffff_ffff_ffff_ffffn; -// Converts the non-standard expectedLength extension: the total bytes a byte +// Converts the non-standard expectedLength extension: the total bytes a // source or a transformer promises, or a FixedLengthStream's length. It // converts as the C++ implementation converts a uint64_t, so the two accept // the same values and agree on what they mean. undefined means no length. A @@ -328,9 +328,10 @@ function convertReadableStreamType(value: unknown): 'bytes' | undefined { // An underlying source as WebIDL converts it to an UnderlyingSource // dictionary, before the constructor checks anything: every member read // once, in lexicographic order, each callback checked, type converted as -// its enum, and autoAllocateChunkSize and the expectedLength extension -// converted, on a default source too, although only a byte source uses -// them. The callbacks still run with the source itself as their receiver. +// its enum, the expectedLength extension converted, and +// autoAllocateChunkSize converted on a default source too, although only a +// byte source uses it. The callbacks still run with the source itself as +// their receiver. // S narrows the callbacks to one kind of source, once type is known. export interface UnderlyingSourceDict< R, diff --git a/src/per_isolate/webstreams/readable.ts b/src/per_isolate/webstreams/readable.ts index c2ab9dfe3cb..46d4fb9ea27 100644 --- a/src/per_isolate/webstreams/readable.ts +++ b/src/per_isolate/webstreams/readable.ts @@ -394,12 +394,12 @@ let getReaderStream: (reader: object) => ReadableStream | undefined; // ReadableStream's static block (needs private-field access). let extractNativeSource: (this: ReadableStream) => object; -// The non-standard expectedLength pass-through for the DrainingReader -// (and the C++ bridge). Chained like the other controller helpers: -// default → the value installed by the TransformStream expectedLength -// extension (undefined otherwise); queued byte → cached construction -// value; native → cached construction value (joined in ReadableStream's -// static block). +// The non-standard expectedLength pass-through behind +// getReadableStreamExpectedLength. Chained like the other controller +// helpers, each reporting the value its underlying source declared, +// cached at construction: default and queued byte (a TransformStream's +// readable declares its transformer's), and native (joined in +// ReadableStream's static block). let getControllerExpectedLength: ( controller: AnyReadableController ) => bigint | undefined; @@ -411,10 +411,6 @@ let byteControllerSetConsumptionHook: ( controller: ReadableByteStreamController, hook: (() => void) | undefined ) => void; -let setDefaultControllerExpectedLength: ( - controller: ReadableStreamDefaultController, - length: bigint | undefined -) => void; let defaultControllerCanCloseOrEnqueue: ( controller: ReadableStreamDefaultController ) => boolean; @@ -1693,12 +1689,11 @@ class ReadableStreamDefaultController< > implements ReadableStreamDefaultControllerType { #stream: ReadableStream; #core: QueuedControllerCore; - // The non-standard expectedLength pass-through (undefined = unknown). - // Default controllers never read it from an underlying source — it is - // set ONLY through the internal setter, by the TransformStream - // constructor's workerd `expectedLength` extension, so the C++ bridge - // can derive Content-Length for bodies built from such transforms. - #expectedLength: bigint | undefined = undefined; + // The non-standard expectedLength the source declared (undefined = + // unknown): the TOTAL bytes the stream will deliver, from which the C++ + // bridge derives a body's Content-Length. Chunks need not be bytes, so + // the controller does not enforce it; body consumption does. + #expectedLength: bigint | undefined; static { assertIsReadableStreamDefaultController = function ( @@ -1726,18 +1721,11 @@ class ReadableStreamDefaultController< queuedCoreOf(controller)?.callPullIfNeeded(); }; - // Default controllers never read expectedLength from an underlying - // source (silently ignored if declared); it is populated only via the - // internal setter (the TransformStream `expectedLength` extension). // The byte and native arms of this chain report their own cached // values. getControllerExpectedLength = (controller) => (controller as ReadableStreamDefaultController).#expectedLength; - setDefaultControllerExpectedLength = (controller, length) => { - controller.#expectedLength = length; - }; - defaultControllerCanCloseOrEnqueue = (controller) => controller.#core.canCloseOrEnqueue(); @@ -1784,7 +1772,13 @@ class ReadableStreamDefaultController< assertPrivateSymbol(privateSymbol); this.#stream = stream; - const { cancel: cancelFn, pull: pullFn, start: startFn } = sourceDict; + const { + cancel: cancelFn, + expectedLength, + pull: pullFn, + start: startFn, + } = sourceDict; + this.#expectedLength = expectedLength; // Strategy already extracted by caller (WebIDL conversion order). const core = new QueuedControllerCore( stream, @@ -2133,10 +2127,10 @@ class ReadableByteStreamController implements ReadableByteStreamControllerType { throw new TypeError('autoAllocateChunkSize cannot be zero'); } this.#autoAllocateChunkSize = autoAllocateChunkSize; - // Non-standard extension, used by byte streams only: the TOTAL bytes - // this source promises to produce. This controller enforces the - // contract at enqueue/respond (overflow) and close() (underflow). - // Exposed to the C++ bridge via the DrainingReader. + // Non-standard extension: the TOTAL bytes this source promises to + // produce, from which the C++ bridge derives a body's Content-Length. + // This controller enforces the contract at enqueue/respond (overflow) + // and close() (underflow). this.#expectedLength = expectedLength; // Strategy already extracted by caller (WebIDL conversion order). @@ -2709,20 +2703,14 @@ class ReadableStreamDrainingReader { } } - // The non-standard expectedLength pass-through for the C++ bridge: the - // TOTAL bytes the underlying source declared it will produce - // (undefined = unknown → chunked encoding). A construction-time value, - // cached on the controller; backend-blind via the chained helper - // (byte/native report their cached value; a default controller the - // value the TransformStream expectedLength extension installed, if - // any). Returns undefined after release. + // The non-standard expectedLength: the bytes the stream still has to + // deliver, if known (undefined = unknown), as the C++ bridge sees it + // (getReadableStreamExpectedLength). Returns undefined after release. get expectedLength(): bigint | undefined { assertIsReadableStreamDrainingReader(this); const stream = getReaderStream(this); if (stream === undefined) return undefined; - const controller = getReadableStreamController(stream); - if (controller === undefined) return undefined; - return getControllerExpectedLength(controller); + return getReadableStreamExpectedLength(stream); } cancel(reason?: unknown): Promise { @@ -3588,6 +3576,10 @@ class ReadableStream { // teed again is closed at the tee (see readableStreamTee). #consumer?: StreamConsumerType | undefined; #disturbed: boolean = false; + // Whether this stream is a tee branch or detached copy (makeShell) of a + // stream that had been read from, or was itself such a copy, before the + // fork (see getReadableStreamExpectedLength). + #forkedAfterRead: boolean = false; #state: ReadableStreamState = 'readable'; #storedError?: unknown; // The Node.js interop closed-promise (see kIsClosedPromise), created on @@ -3657,9 +3649,23 @@ class ReadableStream { return promise; }; + // The total the stream still has to deliver, if known: the C++ bridge's + // length query (a body's Content-Length). A queued source's declaration + // is its stream's total, and the controller, shared by every tee branch, + // does not count what a stream has already passed. So a queued stream + // reports it only while nothing has been read from it, nor, for a tee + // branch or detached copy, before the fork. A native stream's forks are + // fresh streams over sources that count what is left, and a disturbed + // native stream cannot be pumped (extraction rejects it). getReadableStreamExpectedLength = (stream: ReadableStream) => { const controller = stream.#controller; if (controller === undefined) return undefined; + if ( + !isNativeController(controller) && + (stream.#disturbed || stream.#forkedAfterRead) + ) { + return undefined; + } return getControllerExpectedLength(controller); }; @@ -3901,8 +3907,10 @@ class ReadableStream { // A shell carries another stream's state forward: a tee branch, or the // stream detach() moves a body to. It copies the state, stored error and - // pending closure, and shares `controller`, if any; the caller gives it a - // consumer when it has data to read. + // pending closure, records whether anything was read before the fork, + // and shares `controller`, if any; the caller gives it a consumer when + // it has data to read. Callers make shells before handing `from` off, + // which disturbs it. const makeShell = ( from: ReadableStream, controller?: AnyReadableController @@ -3911,6 +3919,7 @@ class ReadableStream { shell.#state = from.#state; shell.#storedError = from.#storedError; shell.#pendingClosure = from.#pendingClosure; + shell.#forkedAfterRead = from.#disturbed || from.#forkedAfterRead; shell.#controller = controller; return shell; }; @@ -4571,9 +4580,10 @@ class ReadableStream { // Body consumption for the C++ bridge (arrayBuffer/bytes/text/json). Two // byte bounds apply: the caller's memory limit, capped at 128 MB, and the -// stream's declared expectedLength. A breach names its cause and cancels -// the stream with it, as the C++ AllReader does; a declaration the limit -// cannot hold is refused before a byte is read. +// stream's declared expectedLength, an exact total that a stream closing +// short of also breaches. A breach names its cause and cancels the stream +// with it, as the C++ AllReader does; a declaration the limit cannot hold +// is refused before a byte is read. const kMaximumAllowedLimit = 128n * 1024n * 1024n; function acquireReadableStreamDrainingReader( @@ -4753,7 +4763,17 @@ async function collectChunks( } collected.append(buffer, byteOffset, byteLength); } - if (result.done) return collected; + if (result.done) { + if (declaredBinds && collected.length < bound) { + return failCollect( + reader, + new RangeError( + 'stream delivered fewer bytes than its declared expectedLength' + ) + ); + } + return collected; + } } } @@ -4906,8 +4926,8 @@ module.exports = { ReadableStreamDrainingReader, // Internal operations consumed by the transform pairs (transform.ts, // identity.ts, compression.ts): the TransformStream cancel/flush - // coordination (finishPromise guard), the workerd expectedLength - // extension, and the interop error hook. Unreachable from user code. + // coordination (finishPromise guard) and the interop error hook. + // Unreachable from user code. internalsForTransform: ObjectFreeze({ getState: (stream: ReadableStream) => getReadableStreamGetState(stream), @@ -4931,14 +4951,6 @@ module.exports = { controller as ReadableByteStreamController, hook ), - setControllerExpectedLength: ( - controller: object, - length: bigint | undefined - ) => - setDefaultControllerExpectedLength( - controller as ReadableStreamDefaultController, - length - ), // A transform pair's notification, called synchronously with the // reason after the Node.js interop hook has errored this stream (the // only external error path that bypasses the source's cancel steps). diff --git a/src/per_isolate/webstreams/transform.ts b/src/per_isolate/webstreams/transform.ts index 32c073fed5d..b8f7a445687 100644 --- a/src/per_isolate/webstreams/transform.ts +++ b/src/per_isolate/webstreams/transform.ts @@ -566,6 +566,11 @@ class TransformStream { }, pull: sourcePull, cancel: (reason: unknown) => this.#finish('cancel', reason), + // Non-standard workerd extension: the TOTAL bytes the readable + // side will produce (undefined = unknown), from which the C++ + // bridge derives a Content-Length for bodies built from this + // transform. + expectedLength, }, { __proto__: null, @@ -574,16 +579,6 @@ class TransformStream { size: readableStrategyDict.size, } ); - // Non-standard workerd extension: the TOTAL bytes the readable side - // will produce (undefined = unknown). Advertised through the - // readable's controller so the C++ bridge derives a Content-Length - // for bodies built from this transform; not enforced here. - if (expectedLength !== undefined) { - readableInternals.setControllerExpectedLength( - this.#readableController, - expectedLength - ); - } // --- Start the transformer (the elided path has none to start) --- if (controller === undefined) { diff --git a/src/per_isolate/webstreams/types.d.ts b/src/per_isolate/webstreams/types.d.ts index 8b1f0533aa6..f0d1f5303fb 100644 --- a/src/per_isolate/webstreams/types.d.ts +++ b/src/per_isolate/webstreams/types.d.ts @@ -36,6 +36,12 @@ export interface UnderlyingDefaultSource { ) => void | Promise; cancel?: (reason?: unknown) => void | Promise; type?: undefined; + // Non-standard workerd extension: the TOTAL bytes this source promises + // to produce (undefined = unknown), converted as on a byte source. Used + // by the C++ bridge to set Content-Length instead of chunked encoding. + // Chunks need not be bytes, so the controller does not enforce it; body + // consumption rejects a delivery past it. + expectedLength?: bigint | number | undefined; } export interface UnderlyingByteSource { diff --git a/src/per_isolate/webstreams/writable.ts b/src/per_isolate/webstreams/writable.ts index 49ba0dfbacf..9500139ddb4 100644 --- a/src/per_isolate/webstreams/writable.ts +++ b/src/per_isolate/webstreams/writable.ts @@ -902,62 +902,6 @@ let controllerStartedOf: ControllerOp<[], boolean>; // --------------------------------------------------------------------------- // WritableStreamDefaultController -interface SinkAlgorithms { - write: ((chunk: W) => Promise) | undefined; - close: (() => Promise) | undefined; - abort: ((reason: unknown) => Promise) | undefined; -} - -// The controller's write, close and abort algorithms, calling the hooks -// through captured uncurryThis wrappers with the sink as `this` (spec -// SetUpWritableStreamDefaultControllerFromUnderlyingSink). Built outside -// the controller constructor so that `underlyingSink` is captured only by -// these closures, not by the constructor's shared context (which start()'s -// reactions keep alive until it settles): once ClearAlgorithms drops them, -// the sink is collectable. -function makeSinkAlgorithms( - underlyingSink: UnderlyingSink, - controller: WritableStreamDefaultController, - writeFn: UnderlyingSink['write'], - closeFn: UnderlyingSink['close'], - abortFn: UnderlyingSink['abort'] -): SinkAlgorithms { - let write: SinkAlgorithms['write']; - if (writeFn !== undefined) { - const callWrite = uncurryThis(writeFn); - write = (chunk: W) => { - try { - return PromiseResolve(callWrite(underlyingSink, chunk, controller)); - } catch (e) { - return PromiseReject(e); - } - }; - } - let close: SinkAlgorithms['close']; - if (closeFn !== undefined) { - const callClose = uncurryThis(closeFn); - close = () => { - try { - return PromiseResolve(callClose(underlyingSink)); - } catch (e) { - return PromiseReject(e); - } - }; - } - let abort: SinkAlgorithms['abort']; - if (abortFn !== undefined) { - const callAbort = uncurryThis(abortFn); - abort = (reason: unknown) => { - try { - return PromiseResolve(callAbort(underlyingSink, reason)); - } catch (e) { - return PromiseReject(e); - } - }; - } - return { write, close, abort }; -} - interface QueuedWrite { value: W | typeof kCloseMarker | typeof kFlushMarker; size: number; @@ -979,9 +923,18 @@ class WritableStreamDefaultController< #started: boolean = false; #strategyHWM: number; #sizeAlgorithm: ((chunk: W) => number) | undefined; - #writeAlgorithm: ((chunk: W) => Promise) | undefined; - #closeAlgorithm: (() => Promise) | undefined; - #abortAlgorithm: ((reason: unknown) => Promise) | undefined; + // The write, close and abort algorithms are these hooks, uncurried, called + // with the sink as `this` by #callWrite, #callClose and #callAbort. The + // sink lives in a field ClearAlgorithms clears, never in a closure: a + // closure's context can stay reachable after the closure is dropped (from + // JIT-compiled code), and the sink with it. + #underlyingSink: UnderlyingSink | undefined; + #writeHook: + | ((sink: UnderlyingSink, chunk: W, controller: this) => unknown) + | undefined; + #closeHook: ((sink: UnderlyingSink) => unknown) | undefined; + #abortHook: + ((sink: UnderlyingSink, reason: unknown) => unknown) | undefined; #abortController = new AbortController(); static { @@ -1034,11 +987,7 @@ class WritableStreamDefaultController< // returns, so a write made from inside sink.abort() still consults // size() (WritableStreamDefaultWriterWrite step 4) before it rejects. controllerAbortSteps = (controller, reason) => { - const abortAlgorithm = controller.#abortAlgorithm; - const result = - abortAlgorithm === undefined - ? PromiseResolve() - : abortAlgorithm(reason); + const result = controller.#callAbort(reason); controller.#clearAlgorithms(); return result; }; @@ -1142,16 +1091,10 @@ class WritableStreamDefaultController< start: startFn, write: writeFn, } = sinkDict; - const algorithms = makeSinkAlgorithms( - underlyingSink, - this, - writeFn, - closeFn, - abortFn - ); - this.#writeAlgorithm = algorithms.write; - this.#closeAlgorithm = algorithms.close; - this.#abortAlgorithm = algorithms.abort; + this.#underlyingSink = underlyingSink; + this.#writeHook = writeFn === undefined ? undefined : uncurryThis(writeFn); + this.#closeHook = closeFn === undefined ? undefined : uncurryThis(closeFn); + this.#abortHook = abortFn === undefined ? undefined : uncurryThis(abortFn); // Strategy already extracted by caller (WebIDL conversion order). this.#sizeAlgorithm = sizeAlgorithm; @@ -1201,12 +1144,52 @@ class WritableStreamDefaultController< } #clearAlgorithms(): void { - this.#writeAlgorithm = undefined; - this.#closeAlgorithm = undefined; - this.#abortAlgorithm = undefined; + this.#underlyingSink = undefined; + this.#writeHook = undefined; + this.#closeHook = undefined; + this.#abortHook = undefined; this.#sizeAlgorithm = undefined; } + // The sink algorithms: each calls its hook, if the sink has one and the + // algorithms are not cleared, as a promise, a sync throw becoming a + // rejection. + #callWrite(chunk: W): Promise { + const hook = this.#writeHook; + if (hook === undefined) return PromiseResolve(); + try { + return PromiseResolve( + hook(this.#underlyingSink as UnderlyingSink, chunk, this) + ) as Promise; + } catch (e) { + return PromiseReject(e); + } + } + + #callClose(): Promise { + const hook = this.#closeHook; + if (hook === undefined) return PromiseResolve(); + try { + return PromiseResolve( + hook(this.#underlyingSink as UnderlyingSink) + ) as Promise; + } catch (e) { + return PromiseReject(e); + } + } + + #callAbort(reason: unknown): Promise { + const hook = this.#abortHook; + if (hook === undefined) return PromiseResolve(); + try { + return PromiseResolve( + hook(this.#underlyingSink as UnderlyingSink, reason) + ) as Promise; + } catch (e) { + return PromiseReject(e); + } + } + #advanceQueueIfNeeded(): void { if (!this.#started) return; const stream = this.#stream; @@ -1247,9 +1230,7 @@ class WritableStreamDefaultController< // As in [[AbortSteps]], the algorithms are cleared after the close // algorithm returns: a write made from inside sink.close() consults // size() before it rejects. - const closeAlgorithm = this.#closeAlgorithm; - const promise = - closeAlgorithm === undefined ? PromiseResolve() : closeAlgorithm(); + const promise = this.#callClose(); this.#clearAlgorithms(); PromisePrototypeThen( promise, @@ -1265,9 +1246,7 @@ class WritableStreamDefaultController< #processWrite(chunk: W): void { const stream = this.#stream; writableStreamMarkFirstWriteRequestInFlight(stream); - const writeAlgorithm = this.#writeAlgorithm; - const promise = - writeAlgorithm === undefined ? PromiseResolve() : writeAlgorithm(chunk); + const promise = this.#callWrite(chunk); // A detach (detachWritableStream) closes the stream with this write in // flight, and has already rejected its request and dropped the queue: // the sink's settlement is then ignored. Nothing else closes a stream diff --git a/src/tests/streams/identity/AGENTS.md b/src/tests/streams/identity/AGENTS.md index 06c02bcb0fa..b02723e11c8 100644 --- a/src/tests/streams/identity/AGENTS.md +++ b/src/tests/streams/identity/AGENTS.md @@ -203,6 +203,10 @@ The readable side supports `getReader({ mode: 'byob' })` (unlike a standard - Canceling one branch leaves the writer flowing to the survivor. The cancel promise semantics and the fate of writes after **both** branches cancel diverge (ledger). +- A FixedLengthStream's branches teed before any read carry its length as + their Content-Length, as the readable does. Teed after a read, they carry + what each has left under C++ and go out chunked under TypeScript + (ledger #26). ## Compatibility flags @@ -268,6 +272,7 @@ pattern; a change to either side fails its cell. | 23 | Write settlement and BYOB reads across tee branches | a write settles once one branch has read it (the sibling's copy uncounted); a branch's reads are capped at the sizes the first-reading branch requested | a write settles once the slowest branch has read it, or once a branch is starved (pending read, nothing left to take) — never once the writable is erroring, when it rejects instead; each branch's reads are sized independently and span writes | `teeCreatesNoDemand`, `singleBranchReadDrivesWriter`, `writerDesiredSizeAcrossTee`, `abortBeforeStarvedReadRejectsInFlightWrite`, `tee-byob.js` | | 24 | `highWaterMark` conversion | to a uint64: non-finite (`'abc'`, NaN, ±Infinity) and negative values throw `TypeError`; fractions truncate (`1.5`→1); a bigint converts (`3n`→3); `desiredSize` narrows to a 32-bit int (`2**32 + 5` reads 5) | the WHATWG writable conversion, the same for ITS and FLS: NaN and negative values throw `RangeError`; Infinity and fractions are kept (FLS caps the converted number at its length); a bigint throws `TypeError` | `highWaterMarkConversionDivergence` | | 25 | String chunk containing NUL | written up to the first NUL only (`'a\0b'` delivers `61`); the string branch of `WritableStreamInternalController::write` | every UTF-8 byte (`61 00 62`) | `stringWithNulDivergence` | +| 26 | the Content-Length of a FixedLengthStream's tee branches made after a read (the readable, and branches teed before any read, carry the full length in both) | what each branch has left (5 of 10 after a 5-byte read) | none, so they go out chunked: the branches share the readable's controller, which knows the total but not what is left once something has been read (the readable suite's ledger #35) | `fixedLengthTeeAfterReadContentLength`, `fixedLengthBodyContentLength` (parity) | ## Assertion catalogue @@ -293,6 +298,7 @@ pattern; a change to either side fails its cell. | `tee-nested.js` | nested tee delivers to all leaves in order; single-leaf read does not hang | | `draining-reader.js` | TS only (C++ cell asserts the global's absence): `expectedLength` pass-through (bigint for FLS, undefined for ITS, undefined after release); a single read drains every synchronously buffered chunk plus the close sentinel in one batch (tee-sibling backlog), and over a rendezvous stream the first read sweeps every write made before it, one chunk per write (the close sentinel with them or in its own read); write/close settlement through the conduit; lock exclusivity and release | | `body-integration.js` | Response/Request with identity-stream bodies: `text()` drives the rendezvous; `resp.body` is the same stream object (unwrapped, unconsumed, unlocked); FLS happy path and underwrite through body consumption (types per ledger #11); multi-megabyte patterned bodies verified byte-for-byte through `arrayBuffer()` (ITS and FLS Response, Request) | +| `content-length.js` | the Content-Length a FixedLengthStream body carries through the SELF echo: the readable and branches teed before any read carry the full length (parity); branches teed after a read (ledger #26) | | `pipe-integration.js` | `pipeTo`/`pipeThrough` between identity streams (ledger #15): TS delivery (small and multi-megabyte patterned bodies), completion, and both error-propagation directions with original reasons; C++ not-implemented wall (rejection for pipeTo, synchronous throw for pipeThrough); circular `pipeThrough(its)` with the `TODO(streams-ts)` pin | | `payload-helpers.js` | shared machinery: continuous prime-modulus byte pattern for large-body generation and byte-exact verification | | `reentrancy.js` | user code re-entering mid-processing: `Object.prototype.then` interception during read resolution (consulted once per read) incl. a re-entrant `writer.close()` or `writer.abort()` from inside the getter (an abort rejects the write being answered, in TS); close/write/sibling-cancel from read continuations; second-concurrent-read divergence (ledger #16) | diff --git a/src/tests/streams/identity/content-length.js b/src/tests/streams/identity/content-length.js new file mode 100644 index 00000000000..dbeb8393e50 --- /dev/null +++ b/src/tests/streams/identity/content-length.js @@ -0,0 +1,76 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// The Content-Length carried by a fetch body built from a FixedLengthStream's +// readable side, and by its tee branches, observed through the SELF echo +// service. +// +// PARITY: the readable, and branches teed before any read, carry the full +// length. +// +// DIVERGENCE (ledger #26): branches teed after a read. C++ carries what each +// branch has left. TypeScript sends them chunked: its branches share the +// readable's controller, which knows the declared total but not what is left +// once something has been read (the readable suite's ledger #35). + +import { deepStrictEqual, strictEqual } from 'node:assert'; +import { usingTsImpl } from 'which-impl'; + +const enc = new TextEncoder(); + +// A FixedLengthStream of 10 bytes written in two chunks, with the writes and +// the close still pending. +function tenByteFixedLengthStream() { + const { readable, writable } = new FixedLengthStream(10); + const writer = writable.getWriter(); + const written = Promise.all([ + writer.write(enc.encode('01234')), + writer.write(enc.encode('56789')), + writer.close(), + ]); + return { readable, written }; +} + +// Posts the body through SELF, and returns the Content-Length it carried and +// the body echoed back. +async function post(env, body) { + const resp = await env.SELF.fetch('http://example.org', { + method: 'POST', + body, + }); + const text = await resp.text(); + return [resp.headers.get('observed-content-length'), text]; +} + +export const fixedLengthBodyContentLength = { + async test(ctrl, env) { + { + const { readable, written } = tenByteFixedLengthStream(); + deepStrictEqual(await post(env, readable), ['10', '0123456789']); + await written; + } + { + const { readable, written } = tenByteFixedLengthStream(); + for (const branch of readable.tee()) { + deepStrictEqual(await post(env, branch), ['10', '0123456789']); + } + await written; + } + }, +}; + +export const fixedLengthTeeAfterReadContentLength = { + async test(ctrl, env) { + const { readable, written } = tenByteFixedLengthStream(); + const reader = readable.getReader(); + const { value } = await reader.read(); + reader.releaseLock(); + strictEqual(new TextDecoder().decode(value), '01234'); + const length = usingTsImpl ? 'null' : '5'; + for (const branch of readable.tee()) { + deepStrictEqual(await post(env, branch), [length, '56789']); + } + await written; + }, +}; diff --git a/src/tests/streams/identity/identity-cpp-pedantic.wd-test b/src/tests/streams/identity/identity-cpp-pedantic.wd-test index a0984d640be..d2098c53778 100644 --- a/src/tests/streams/identity/identity-cpp-pedantic.wd-test +++ b/src/tests/streams/identity/identity-cpp-pedantic.wd-test @@ -28,6 +28,9 @@ const unitTests :Workerd.Config = ( "enhanced_error_serialization", "pedantic_wpt", ], + bindings = [ + ( name = "SELF", service = "streams-identity-cpp-pedantic" ), + ], ) ), ], diff --git a/src/tests/streams/identity/identity-cpp.wd-test b/src/tests/streams/identity/identity-cpp.wd-test index 4cb8d3a9ec9..dbb55c393f3 100644 --- a/src/tests/streams/identity/identity-cpp.wd-test +++ b/src/tests/streams/identity/identity-cpp.wd-test @@ -43,6 +43,9 @@ const unitTests :Workerd.Config = ( "set_tostring_tag", "enhanced_error_serialization", ], + bindings = [ + ( name = "SELF", service = "streams-identity-cpp" ), + ], ) ), ], diff --git a/src/tests/streams/identity/identity-modules.capnp b/src/tests/streams/identity/identity-modules.capnp index 818323e08c6..0fb60be6e3c 100644 --- a/src/tests/streams/identity/identity-modules.capnp +++ b/src/tests/streams/identity/identity-modules.capnp @@ -37,6 +37,7 @@ const modules :List(Workerd.Worker.Module) = [ (name = "draining-reader", esModule = embed "draining-reader.js"), (name = "payload-helpers", esModule = embed "payload-helpers.js"), (name = "body-integration", esModule = embed "body-integration.js"), + (name = "content-length", esModule = embed "content-length.js"), (name = "pipe-integration", esModule = embed "pipe-integration.js"), (name = "reentrancy", esModule = embed "reentrancy.js"), (name = "lock-release", esModule = embed "lock-release.js"), diff --git a/src/tests/streams/identity/identity-ts.wd-test b/src/tests/streams/identity/identity-ts.wd-test index 1c40c4e9b93..f436a5ceaf9 100644 --- a/src/tests/streams/identity/identity-ts.wd-test +++ b/src/tests/streams/identity/identity-ts.wd-test @@ -29,6 +29,9 @@ const unitTests :Workerd.Config = ( "expose_draining_reader", "experimental", ], + bindings = [ + ( name = "SELF", service = "streams-identity-ts" ), + ], ) ), ], diff --git a/src/tests/streams/identity/main.js b/src/tests/streams/identity/main.js index fdffc79d479..645caba10aa 100644 --- a/src/tests/streams/identity/main.js +++ b/src/tests/streams/identity/main.js @@ -226,3 +226,22 @@ export { export { structuredCloneIdentity } from 'transfer'; export { internalDictionariesIgnorePollution } from 'pollution'; + +export { + fixedLengthBodyContentLength, + fixedLengthTeeAfterReadContentLength, +} from 'content-length'; + +// Echo handler for content-length.js: reports the Content-Length the +// subrequest actually carried and echoes the body. +export default { + async fetch(request) { + return new Response(request.body, { + headers: { + 'observed-content-length': String( + request.headers.get('content-length') + ), + }, + }); + }, +}; diff --git a/src/tests/streams/readable-byte/AGENTS.md b/src/tests/streams/readable-byte/AGENTS.md index b0f509d7294..756734ed602 100644 --- a/src/tests/streams/readable-byte/AGENTS.md +++ b/src/tests/streams/readable-byte/AGENTS.md @@ -58,6 +58,9 @@ behavior-parity (messages aside). | 41 | a bare ArrayBuffer passed to enqueue() or respondWithNewView(), whose WebIDL parameter is an ArrayBufferView (any other non-view there, and every non-view at read(view) and readAtLeast(), is a TypeError in both, messages aside) | taken as a view over the whole buffer (the parameter is a BufferSource): enqueue() detaches it and delivers its bytes; respondWithNewView() accepts one of the request's buffer length, detaches it and delivers its bytes | TypeError 'chunk must be an ArrayBufferView' or 'view must be an ArrayBufferView' (spec); the buffer stays attached, nothing is enqueued, and the pending read is left for a later response | `nonViewArgumentsRejected` | | 42 | a byte source's declared `expectedLength` as an exact total of the bytes it delivers through enqueue(), respond() and respondWithNewView(), a response to a byobRequest held across tee() (#25) included (delivering exactly the total, then close(), is parity) | not enforced: a delivery past the total reaches the reader, and close() short of it closes the stream | by design: the delivery past the total throws RangeError 'byte source delivered more bytes than its declared expectedLength', and close() short of it RangeError 'byte source closed before producing its declared expectedLength'; either errors the stream with that error | `byteSourceExpectedLengthContract`, `byteSourceExpectedLengthHeldRequest` | | 43 | close() with a pending default read, with or without autoAllocateChunkSize (the close() half of #22's default reads; a BYOB read pending at close() is #19) | read resolves done with an empty view (a zero-length Uint8Array) | read resolves done with value undefined (spec: ReadableStreamClose runs the read request's close steps) | `closeWithPendingDefaultRead` | +| 44 | the Content-Length of a fork (a tee branch, a branch teed again, the copy `new Request()` makes) of a byte source declaring `expectedLength`, made before any read (made after a read, the fork and its own forks go out chunked in both, and a read on one branch leaves its unread sibling as it was) | none: tee branches never carry a length, so they go out chunked | the declared length, as for a default source (the readable suite's ledger #35) | `byteSourceForkBeforeReadKeepsLength`, `byteSourceForkAfterReadSendsChunked` | +| 45 | the length a byte source declaring `expectedLength` reports once read from, to the consumers that take a partly read stream (JS RPC, KV `put()`, R2 `put()`) | DEFECT: its declared total, so the receiver's read fails | none (the readable suite's ledger #36 has both sides in full) | `readableLength` in `src/workerd/api/tests/js-rpc-streams-crossflag-test.js` (RPC; no consumer in this suite takes a partly read stream) | +| 46 | a response whose byte source declares `expectedLength` and is compressed on the way out by `Content-Encoding: gzip` (the readable suite's ledger #37) | DEFECT: the uncompressed length is kept as the Content-Length, so the reader fails with 'Network connection lost.' | no length for an encoded body: the response goes out chunked and decodes in full | `byteSourceCompressedResponseDropsLength` | Parity worth noting (probed, pinned): byte hwm defaults to 0 with NO automatic pull; pull-throw and error-then-throw identity; enqueue @@ -148,7 +151,7 @@ named suite test pins directly, differing only in incidental asserts. | `draining-reader.js` | TS only (C++ cell asserts the global's absence): a queued byte backlog plus the close sentinel swept in one batched read with chunks INTACT (no coalescing); a read whose maxSize runs out at the last bytes before the close reaches the close all the same, reporting done and closing the stream in that call; with autoAllocateChunkSize the conduit's wait-read synthesizes the descriptor so pull carries a byobRequest (ledger #6); expectedLength undefined when undeclared, a declared one passed through as the exact bigint (above 2^53 and up to the uint64 maximum); error/cancel propagation | | `data-volumes.js` | byte-transfer volumes 64 B / 64 KiB / 1 MiB / 8 MiB via default and BYOB readers (incl. mismatched view/enqueue granularity), continuous prime-modulus pattern verified byte-exact; the source closes WITH its last enqueue (single-shape loops; parked-read close settlement is ledger #19's) | | `pollution.js` | prototype pollution neither implementation observes: a patched byte controller error() still errors the stream; readAtLeast() ignores a patched read(); tee copies and respond() remainders never consult the ArrayBuffer or %TypedArray% species; a native body ignores Object.prototype `type`/`autoAllocateChunkSize`; one Object.prototype.then getter fire per read, on an ordinary result object (default reads, autoAllocateChunkSize, BYOB and readAtLeast, at close, native bodies); ledger #34; a then getter that releases the reader while cancel() or close() settles its reads (default, autoAllocateChunkSize and BYOB reads, BYOB ones at a close() answered by respond(0) or not), or errors the source from a tee branch's read while close() settles that branch's reads: the next read still resolves done (parity); one that closes the source from a tee branch's read before the enqueue() that answered it reaches the other branch: that branch's autoAllocateChunkSize reads, two filled by the chunk and one answered done, resolve in the order they were made (parity) | -| `expected-length.js` | ledger #35, with the converted length observed as a fetch body's Content-Length via SELF; integral lengths up to the uint64 maximum accepted; ledger #42, the declared length as an exact total through enqueue(), respond() and respondWithNewView(), and through a byobRequest held across tee() | +| `expected-length.js` | ledger #35, with the converted length observed as a fetch body's Content-Length via SELF; integral lengths up to the uint64 maximum accepted; ledger #42, the declared length as an exact total through enqueue(), respond() and respondWithNewView(), and through a byobRequest held across tee(); ledger #44, the Content-Length of forks made before and after a read; ledger #46, a declared length on a gzip-encoded response, served by SELF at `/gzip` | | `reader-brand-checks.js` | a reader member called with a reader of another class as `this` fails the brand check with a TypeError 'Illegal invocation' (C++ adds a pointer to its documentation) and leaves that reader as it was (parity): TS rejects from the members that return a promise, as WebIDL has them, and C++ throws from every member, the shape WPT's idlharness pins for a plain `this`; under TS the draining reader too, as the members' owner and as `this` | Consumed sources (deleted): streams-js-test.js (value halves were diff --git a/src/tests/streams/readable-byte/expected-length.js b/src/tests/streams/readable-byte/expected-length.js index 018886749ae..cf39b0ce32e 100644 --- a/src/tests/streams/readable-byte/expected-length.js +++ b/src/tests/streams/readable-byte/expected-length.js @@ -22,8 +22,26 @@ // and so does a close() short of it; either errors the stream with that // error. C++ enforces neither: every delivery reaches the reader, and // close() closes the stream. +// +// DIVERGENCE (ledger #44): the forks of a byte source declaring +// expectedLength (a tee branch, a branch teed again, the copy +// new Request() makes) carry the declared length in TypeScript when made +// before any read; C++ gives tee branches no length, so they go out +// chunked. The declaration is the stream's total, so once anything has +// been read before a fork, the fork and its own forks go out chunked in +// both (PARITY). A partly read stream itself (ledger #45, a C++ defect) +// is pinned in js-rpc-streams-crossflag-test.js, as no consumer here +// accepts one. +// +// C++ DEFECT (ledger #46): a response whose byte source declares +// expectedLength and is compressed on the way out by Content-Encoding: +// gzip. TypeScript reports no length for an encoded body, so it goes out +// chunked and decodes in full. C++ keeps the uncompressed length as the +// Content-Length, as its tryGetLength() ignores the encoding, so the +// compressed body falls short of it and the reader fails with 'Network +// connection lost.' -import { ok, rejects, strictEqual, throws } from 'node:assert'; +import { deepStrictEqual, ok, rejects, strictEqual, throws } from 'node:assert'; import { usingTsImpl } from 'which-impl'; function byteSource(expectedLength, byteCount = 0) { @@ -267,3 +285,121 @@ export const byteSourceExpectedLengthHeldRequest = { } }, }; + +// A byte source that declares the 10 bytes it delivers, in two chunks. +function tenByteSource() { + const enc = new TextEncoder(); + return new ReadableStream({ + type: 'bytes', + expectedLength: 10, + start(c) { + c.enqueue(enc.encode('01234')); + c.enqueue(enc.encode('56789')); + c.close(); + }, + }); +} + +// Reads the first five bytes and releases the reader. A BYOB read, as a +// default read of a C++ byte stream takes every queued chunk (ledger #17). +async function readFive(stream) { + const reader = stream.getReader({ mode: 'byob' }); + const { value } = await reader.read(new Uint8Array(5)); + reader.releaseLock(); + return new TextDecoder().decode(value); +} + +// Posts the request through SELF, and returns the Content-Length it carried +// and the body echoed back. +async function post(env, request) { + const resp = await env.SELF.fetch(request); + const text = await resp.text(); + return [resp.headers.get('observed-content-length'), text]; +} + +function requestWith(body) { + return new Request('http://example.org', { method: 'POST', body }); +} + +// DIVERGENCE (ledger #44) +export const byteSourceForkBeforeReadKeepsLength = { + async test(ctrl, env) { + const expected = usingTsImpl ? '10' : 'null'; + const [a, b] = tenByteSource().tee(); + const [c, d] = a.tee(); + for (const request of [ + requestWith(b), + requestWith(c), + new Request(requestWith(d)), + ]) { + deepStrictEqual(await post(env, request), [expected, '0123456789']); + } + }, +}; + +const PAYLOAD = 'compress me please '.repeat(2000); + +// Served by main.js at /gzip: a byte source declaring its uncompressed +// length, sent with Content-Encoding: gzip, which compresses it on the way +// out. +export function compressedByteResponse() { + const bytes = new TextEncoder().encode(PAYLOAD); + const body = new ReadableStream({ + type: 'bytes', + expectedLength: bytes.length, + start(c) { + c.enqueue(bytes); + c.close(); + }, + }); + return new Response(body, { headers: { 'content-encoding': 'gzip' } }); +} + +// C++ DEFECT (ledger #46) +export const byteSourceCompressedResponseDropsLength = { + async test(ctrl, env) { + const resp = await env.SELF.fetch('http://example.org/gzip'); + strictEqual(resp.headers.get('content-encoding'), 'gzip'); + if (usingTsImpl) { + strictEqual(await resp.text(), PAYLOAD); + } else { + await rejects(resp.text(), { message: 'Network connection lost.' }); + } + }, +}; + +// PARITY, except for the unread sibling (ledger #44) +export const byteSourceForkAfterReadSendsChunked = { + async test(ctrl, env) { + // Read before the tee: both branches, a branch teed again, and the + // copy new Request() makes. + { + const rs = tenByteSource(); + strictEqual(await readFive(rs), '01234'); + const [a, b] = rs.tee(); + const [c, d] = a.tee(); + for (const request of [ + requestWith(b), + requestWith(c), + new Request(requestWith(d)), + ]) { + deepStrictEqual(await post(env, request), ['null', '56789']); + } + } + + // A branch read, then teed again: its branches go out chunked, while + // its unread sibling keeps the declared length in TypeScript. + { + const [a, b] = tenByteSource().tee(); + strictEqual(await readFive(a), '01234'); + const [c, d] = a.tee(); + for (const request of [requestWith(c), new Request(requestWith(d))]) { + deepStrictEqual(await post(env, request), ['null', '56789']); + } + deepStrictEqual(await post(env, requestWith(b)), [ + usingTsImpl ? '10' : 'null', + '0123456789', + ]); + } + }, +}; diff --git a/src/tests/streams/readable-byte/main.js b/src/tests/streams/readable-byte/main.js index 8db32e8e574..fd411af173b 100644 --- a/src/tests/streams/readable-byte/main.js +++ b/src/tests/streams/readable-byte/main.js @@ -6,6 +6,8 @@ // only. The default export serves the SELF binding used by the // integration tests (echo). +import { compressedByteResponse } from 'expected-length'; + // Set by a request to /fail-now: the /failing body errors once it is set. let failNow = false; @@ -68,6 +70,9 @@ export default { failNow = true; return new Response(null); } + if (url.pathname === '/gzip') { + return compressedByteResponse(); + } return new Response(request.body, { headers: { 'content-type': 'application/octet-stream', @@ -314,6 +319,9 @@ export { byteSourceExpectedLengthRange, byteSourceExpectedLengthContract, byteSourceExpectedLengthHeldRequest, + byteSourceForkBeforeReadKeepsLength, + byteSourceForkAfterReadSendsChunked, + byteSourceCompressedResponseDropsLength, } from 'expected-length'; export { readerMembersOnOtherReaderClasses } from 'reader-brand-checks'; diff --git a/src/tests/streams/readable/AGENTS.md b/src/tests/streams/readable/AGENTS.md index 8b436bad852..5a6eda0a3b9 100644 --- a/src/tests/streams/readable/AGENTS.md +++ b/src/tests/streams/readable/AGENTS.md @@ -35,7 +35,7 @@ behavioral gaps; the reentrancy family is mostly parity at finite hwm. | 18 | error() while a close() is still pending (chunks queued) | ignored: the requested close is final, desiredSize is already 0 and the chunks drain to a clean close | desiredSize is hwm minus the queued chunks (-1) until the error, then the stream errors (spec: close() only requested the close, the state is still "readable"): chunks discarded, reads/closed reject, desiredSize null | `errorAfterCloseWithQueuedChunk` | | 19 | controller held, stream dropped and collected | the controller does not keep its stream alive: its consumer leaves the queue, enqueue() drops the chunk silently, desiredSize stays at the high-water mark (no backpressure for a producer holding only the controller) | the controller references the stream (spec [[stream]]): enqueues count, desiredSize 4 → 0 | `controllerOnlyHeldStreamLiveness` | | 20 | pull() after both tee branches are collected (controller held) | keeps pulling for consumers that no longer exist; DEFECT: a source that enqueues on every pull runs until the stream closes | the source is released: pull() is never called again (the parity half — enqueue drops, desiredSize at the high-water mark, close() as ever — is `teeBranchesCollected`) | `teeBranchesCollectedPullStops` | -| 21 | body consumption of a TransformStream readable that delivers more than its declared `expectedLength` | returns everything (consumption ignores the declaration) | rejects with RangeError 'stream delivered more bytes than its declared expectedLength' and cancels the readable with it, which errors the writable (the declaration is the exact-total contract the byte and native sources already enforce) | `transformExpectedLengthOverflow` | +| 21 | body consumption of a default source or a TransformStream readable that delivers more than its declared `expectedLength`, or closes short of it | returns everything (consumption ignores the declaration) | rejects with RangeError 'stream delivered more bytes than its declared expectedLength' and cancels the stream with it: a source's cancel() gets the error, and a TransformStream's writable errors with it; closing short rejects with RangeError 'stream delivered fewer bytes than its declared expectedLength', with nothing left to cancel (the declaration is the exact-total contract the byte and native sources already enforce) | `transformExpectedLengthOverflow`, `sourceExpectedLengthOverflow`, `expectedLengthUnderflow` | | 22 | async-iterator next() interleavings where WebIDL clears the ongoing promise | every call strictly serialized: a next() from a continuation registered on an earlier next() still waits for the queued ones, and one queued behind return() reports done; after a next() rejects, later next() and return() calls reject with the same error | spec: that next() reads ahead of the queued one (n2 'c', n3 'b') and ahead of a queued return() (reads data; the return still cancels); after a next() rejects, the iterator is finished: later next() calls report done and return() resolves done, each running its steps at once (this last shape is among the WPT async-iterator.any C++ expectedFailures; the read-ahead shapes are derived from the WebIDL algorithm) | `nextFromEarlierContinuationReadsAhead`, `nextFromEarlierContinuationBeatsReturn`, `nextAfterRejectedNextIsDone`, `thenGetterPerIteratorReturn` | | 23 | when a promise returned by start() starts the stream | adopts it: the first pull runs before the first marker chained on it | spec (Node agrees): a new promise is resolved with it, so the pull runs after the second marker, whether it was fulfilled on return or later | `startPromiseSettledInNewPromise` | | 24 | then-getter fires on the result of an async-iterator `next()` made while another is still pending | once | twice: the `next()` settles by adopting the promise of the step it waited for (WebIDL; Node agrees) | `thenGetterPerIteratorNext` | @@ -48,7 +48,10 @@ behavioral gaps; the reentrancy family is mostly parity at finite hwm. | 31 | (no divergence) the strategy argument converts as a WebIDL dictionary: null means no strategy, a function's members are read, and every other primitive throws TypeError (messages differ) | same | same | `strategyArgumentShapes` (parity) | | 32 | which error wins when two are possible (a primitive source's TypeError beats a throwing strategy getter in both; every source member converts before the type and the highWaterMark are checked, so a non-callable pull, TypeError with different messages, or a throwing pull getter beats an invalid type, and a non-callable pull beats an invalid highWaterMark, in both; spec) | an invalid highWaterMark (TypeError, integer conversion) beats an invalid type and a non-callable size | an invalid type and a non-callable size (TypeError) beat an invalid highWaterMark (spec) | `conversionErrorPrecedence` | | 33 | (no divergence) the stream tee() leaves behind (teed directly, a branch teed again, a Response body teed or cloned) is disturbed at tee(): bodyUsed is true from then on, and the Response constructor rejects the stream (the spec disturbs it at the tee's first read, so a teed byte stream such as a Response body stays undisturbed until a branch reads) | same | same | `teedStreamIntoResponse`, `bodyUsedAfterBodyTee` (parity) | -| 34 | autoAllocateChunkSize and expectedLength on a default source, which uses neither (both convert them as on a byte source, with no zero check, spec: expectedLength as a uint64, 'abc' and -1 TypeError with different messages; a fraction of autoAllocateChunkSize truncates, and 0 is accepted) | autoAllocateChunkSize as an int: -1, 'abc', NaN and Infinity accepted; 2 ** 31 and up TypeError 'Value out of range' | autoAllocateChunkSize as the spec's [EnforceRange] unsigned long long: -1, 'abc', NaN and Infinity TypeError; up to 2 ** 53 - 1 accepted | `byteMembersOnDefaultSource` | +| 34 | autoAllocateChunkSize on a default source, which does not use it (both convert it and expectedLength as on a byte source, with no zero check, spec: expectedLength as a uint64, 'abc' and -1 TypeError with different messages; a fraction of autoAllocateChunkSize truncates, and 0 is accepted) | autoAllocateChunkSize as an int: -1, 'abc', NaN and Infinity accepted; 2 ** 31 and up TypeError 'Value out of range' | autoAllocateChunkSize as the spec's [EnforceRange] unsigned long long: -1, 'abc', NaN and Infinity TypeError; up to 2 ** 53 - 1 accepted | `byteMembersOnDefaultSource` | +| 35 | the Content-Length of a fork (a tee branch, a branch teed again, the copy `new Request()` makes) of a default source declaring `expectedLength`, made before any read (made after a read, the fork and its own forks go out chunked in both, and a read on one branch leaves its unread sibling as it was) | none: tee branches never carry a length, so they go out chunked | the declared length: the stream's total, all still to come | `forkBeforeReadKeepsDeclaredLength`, `forkAfterReadSendsChunked` | +| 36 | the length a default or byte source declaring `expectedLength` reports once read from, to the consumers that take a partly read stream (JS RPC, KV `put()`, R2 `put()`; `fetch()` and `Response` reject one in both) | DEFECT: its declared total, whatever has been read (`ReadableStreamJsController::tryGetLength` reports the source's `expectedLength` as is): the receiver is promised more bytes than are left, and its read fails with 'Network connection lost.' | none: RPC and KV send the rest chunked, and R2 `put()`, which needs a length, rejects with TypeError 'Provided readable stream must have a known length (request/response body or readable half of FixedLengthStream)' before sending | `readableLength` in `src/workerd/api/tests/js-rpc-streams-crossflag-test.js` (RPC; no consumer in this suite takes a partly read stream) | +| 37 | a response whose body (a default source or a TransformStream readable) declares `expectedLength` and is compressed on the way out by `Content-Encoding: gzip` | DEFECT: the uncompressed length is kept as the Content-Length (`ReadableStreamJsController::tryGetLength` ignores the encoding, which a FixedLengthStream's does not), so the compressed body falls short of it: the reader fails with 'Network connection lost.' | no length for an encoded body, as for a FixedLengthStream in both: the response goes out chunked and decodes in full | `compressedResponseDropsDeclaredLength` | Parity worth noting (probed, pinned): pull serialization (never re-entered); pull/async-start rejection identity; error-undefined @@ -73,7 +76,7 @@ pull() it runs reading right behind it; async-iterator protocol interleavings (return/next no-await; the WebIDL ongoing-promise shapes are #22); chunks held BY REFERENCE (mutation visible, identity) + detach-while-queued observed; and the -ENTIRE integration surface except #15 and #21: body chunk normalization +ENTIRE integration surface except #15, #21, #35 and #37: body chunk normalization (messages included: 'This ReadableStream did not return bytes.'), a failing consumer cancelling the source with its error (the cancel's own rejection then wins) and leaving the stream locked, chunks collectible @@ -82,7 +85,12 @@ odd chunk sizes, disturbed-into-Response and locked-into-Response TypeErrors, clone-both-read, cancel-body-then-consume 'Body has already been used', readAll small/big/failed paths, SELF fetch round-trips (fetch body + -Request body, the workerd#5113 shapes). +Request body, the workerd#5113 shapes, and a Request copied by +`new Request(request)`) with a default source's declared +`expectedLength` as the subrequest's Content-Length, and a default +source or TransformStream readable delivering less than it declared +failing the transfer with 'Network connection lost.', as a request body +and as a response body. The ts cell additionally sets the internal-testing `expose_draining_reader` flag, installing the @@ -118,14 +126,15 @@ C++ implementation; `draining-reader.js` asserts both sides. | `async-iteration.js` | 7 migrated + values() options as a WebIDL dictionary (parity) + no-await interleavings + proto shape and class string (#13) + foreign `this` (#25) + first next() pulls synchronously, next() from that pull() (parity) + ongoing-promise interleavings (#22) | | `reentrancy.js` | enqueue/close/cancel-in-size (parity) + read-in-size (#14; guard the size() or C++ captures every later chunk) + close-in-size with chunks still to read (#30) and on a tee with a drained branch (the chunk dropped for every branch, the other branch's cancel settling; parity) | | `buffer-lifecycle.js` | chunk by reference, detach observed | -| `integration-body.js` | readAll family, normalization (incl. detached and out-of-bounds views, DataViews included, SharedArrayBuffer-backed views, resizable-extent pinning), clone, cancel-then-consume, SELF round-trips | -| `integration-body-failures.js` | consumer-side failures cancel the source with the error (every consumer; the cancel's rejection replaces it; nothing to cancel once the closing batch is in), leave the stream locked and stop pulls; TransformStream `expectedLength` overflow (#21) and exact delivery | +| `integration-body.js` | readAll family, normalization (incl. detached and out-of-bounds views, DataViews included, SharedArrayBuffer-backed views, resizable-extent pinning), clone, cancel-then-consume, SELF round-trips (the declared `expectedLength` as the Content-Length) | +| `integration-body-failures.js` | consumer-side failures cancel the source with the error (every consumer; the cancel's rejection replaces it; nothing to cancel once the closing batch is in), leave the stream locked and stop pulls; `expectedLength` overflow from a TransformStream readable and a default source, and underflow from both (#21), and a TransformStream's exact delivery | | `integration-body-memory.js` | body consumption copies bytes out as they arrive: chunk buffers are collectible mid-consumption (WeakRef + gc()), ~2.5 MiB of pseudo-random chunk sizes assembles exactly as bytes and as text, one chunk wider than a 1 MiB collection block assembles exactly, a declared-length byte body is exact | | `integration-locked-disturbed.js` | disturbed and locked rejected, body identity + lock coupling (#15), the stream tee() leaves behind into Response and bodyUsed after a body's tee() or clone() (#33) | +| `expected-length.js` | the Content-Length of forks of a default source declaring `expectedLength` (#35): a tee branch, a branch teed again and a `new Request()` copy, made before any read (the declared length in TS) and after one (chunked in both, the unread sibling unaffected); a declared length on a gzip-encoded response from a default source and a TransformStream readable, served by SELF at `/gzip` (#37); a body delivering less than it declared failing the transfer, as a request body and as a response body served by SELF at `/short` (parity) | | `gc.js` | pending read + async iteration survive gc(); a controller held with its stream collected (ledger #19); both tee branches collected while the controller is held: enqueue() drops without retaining (WeakRef-checked, backlog included), desiredSize at the high-water mark, close() then enqueue() as ever, and pull() stops (ledger #20); one branch cancelled and the other collected, observed in the gc()'s own job (before finalization callbacks): the controller's queries drop the consumer themselves; a closed, cancelled or errored stream releases its underlying source, and a source with only start() is released while its stream is still open (WeakRef-checked; parity) | | `then-interceptors.js` | one then-getter fire per read, on an ordinary result object, whether the read is answered from the queue, waits for enqueue() or close(), or finds the stream closed (a tee branch's too), and when it fires for a waiting read (ledger #16); none for a pipe's reads; per async-iterator next() (ledger #24) and return() (ledger #26); a then getter that cancels tee branches while a read result settles (during enqueue() and close(), one branch or two of three): the other branches' pending reads still resolve and their backlog still counts toward desiredSize; a then getter that releases the reader while cancel() or close() settles its reads: the next read still resolves done (parity); a self-removing patched Object.prototype.then intercepts a body consumption's result (arrayBuffer, bytes, blob, json, formData; text resolves with a string), never its internal steps, for native and queued bodies (parity) | | `legacy-constructors.js` | the unflagged cell (see flags table) | -| `draining-reader.js` | TS only (C++ cell asserts the global's absence): a queued backlog plus the close sentinel swept in ONE batched read; a read whose size limit runs out at the last chunk (its maxSize, or chunk sizes adding up to Infinity) reaches the close all the same, reporting done and closing the stream in that call; value chunks pass through UNTOUCHED (object identity); pull-driven yields per read with EOF as a separate empty batch; expectedLength undefined; error/cancel propagation; lock exclusivity and release | +| `draining-reader.js` | TS only (C++ cell asserts the global's absence): a queued backlog plus the close sentinel swept in ONE batched read; a read whose size limit runs out at the last chunk (its maxSize, or chunk sizes adding up to Infinity) reaches the close all the same, reporting done and closing the stream in that call; value chunks pass through UNTOUCHED (object identity); pull-driven yields per read with EOF as a separate empty batch; expectedLength the source's declaration (undefined without one, after release, and once the stream, or before the fork the stream it was forked from, has been read from); error/cancel propagation; lock exclusivity and release | | `data-volumes.js` | value-stream volume axes: 4096-chunk counts, a 1 MiB single string chunk, 8 MiB aggregate (128 × 64 KiB), and 1 MiB through tee on both branches — every chunk index-encoded | | `pollution.js` | prototype pollution neither implementation observes: a patched array iterator or replaced Number global leaves bodies intact; a patched controller error() still errors the stream; omitted dictionaries read nothing from Object.prototype, nor does from() for the source and strategy it builds; @@asyncIterator shape (TS: the values() function object, per WebIDL; C++: a separate function) and for-await ignoring a patched values() | | `unhandled-rejection.js` | a reader's closed promise is marked handled where it is made (on a readable stream, on an errored stream; replaced at release from a closed stream): rejected with nothing looking at it, it reports no unhandledrejection (parity) | diff --git a/src/tests/streams/readable/construction.js b/src/tests/streams/readable/construction.js index e13be37f073..f01f68767e6 100644 --- a/src/tests/streams/readable/construction.js +++ b/src/tests/streams/readable/construction.js @@ -224,16 +224,17 @@ export const conversionErrorPrecedence = { }, }; -// DIVERGENCE: autoAllocateChunkSize and expectedLength on a default -// source, which uses neither. Both convert them on every source, as on a -// byte source but with no zero check (spec): expectedLength as a uint64, -// so 'abc' and -1 throw TypeError (messages differ; the readable-byte -// suite's ledger #35), and autoAllocateChunkSize as a number whose -// fraction truncates, so 0, 1.5 and 2 ** 31 - 1 are accepted. C++ -// converts autoAllocateChunkSize as an int: it accepts -1, 'abc', NaN and -// Infinity, and throws from 2 ** 31. TypeScript converts it as the spec's -// [EnforceRange] unsigned long long: it throws for those, and accepts up -// to 2 ** 53 - 1. +// DIVERGENCE: autoAllocateChunkSize on a default source, which does not +// use it (its expectedLength becomes a body's Content-Length, as a byte +// source's does; see integration-body.js). Both convert the two on every +// source, as on a byte source but with no zero check (spec): +// expectedLength as a uint64, so 'abc' and -1 throw TypeError (messages +// differ; the readable-byte suite's ledger #35), and +// autoAllocateChunkSize as a number whose fraction truncates, so 0, 1.5 +// and 2 ** 31 - 1 are accepted. C++ converts autoAllocateChunkSize as an +// int: it accepts -1, 'abc', NaN and Infinity, and throws from 2 ** 31. +// TypeScript converts it as the spec's [EnforceRange] unsigned long long: +// it throws for those, and accepts up to 2 ** 53 - 1. export const byteMembersOnDefaultSource = { test() { const outOfRange = { diff --git a/src/tests/streams/readable/draining-reader.js b/src/tests/streams/readable/draining-reader.js index 4ab84b1cfdf..f8f2dadfb0f 100644 --- a/src/tests/streams/readable/draining-reader.js +++ b/src/tests/streams/readable/draining-reader.js @@ -17,9 +17,11 @@ // as the C++ bridge's, can meet), reaches the close all the same: it // reports done and closes the stream in that call, as a default read // closes it as it takes the last chunk; value chunks pass through the -// conduit UNTOUCHED (object identity included); a JS stream never -// declares an expectedLength; cancel and error propagate through the -// conduit like a default reader's. +// conduit UNTOUCHED (object identity included); a JS stream's +// expectedLength is its source's declaration, undefined if it declares +// none, or once the stream (or, before the fork, the stream it was forked +// from) has been read from; cancel and error propagate through the conduit like a default +// reader's. /* global ReadableStreamDrainingReader */ @@ -219,3 +221,52 @@ export const drainingReaderLockExclusivity = { ok(rs.getReader()); }, }; + +export const drainingReaderReportsDeclaredLength = { + test() { + if (!usingTsImpl) { + strictEqual(typeof ReadableStreamDrainingReader, 'undefined'); + return; + } + // The source's declaration, as a bigint, until the reader is released. + const rs = new ReadableStream({ expectedLength: 5 }); + const reader = new ReadableStreamDrainingReader(rs); + strictEqual(reader.expectedLength, 5n); + reader.releaseLock(); + strictEqual(reader.expectedLength, undefined); + }, +}; + +// The length the C++ bridge sees (#35 has the Content-Length): none once +// the stream has been read from, nor on a tee branch forked after that +// read, while a branch forked before it keeps the declaration. +export const drainingReaderReportsNoLengthAfterRead = { + async test() { + if (!usingTsImpl) { + strictEqual(typeof ReadableStreamDrainingReader, 'undefined'); + return; + } + function lengthOf(stream) { + const reader = new ReadableStreamDrainingReader(stream); + const length = reader.expectedLength; + reader.releaseLock(); + return length; + } + const rs = new ReadableStream({ + expectedLength: 2, + start(c) { + c.enqueue(new Uint8Array(1)); + c.enqueue(new Uint8Array(1)); + c.close(); + }, + }); + const [early, read] = rs.tee(); + const reader = read.getReader(); + await reader.read(); + reader.releaseLock(); + strictEqual(lengthOf(read), undefined); + const [late] = read.tee(); + strictEqual(lengthOf(early), 2n); + strictEqual(lengthOf(late), undefined); + }, +}; diff --git a/src/tests/streams/readable/expected-length.js b/src/tests/streams/readable/expected-length.js new file mode 100644 index 00000000000..cb28e223860 --- /dev/null +++ b/src/tests/streams/readable/expected-length.js @@ -0,0 +1,192 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// The Content-Length carried by a fork of a default source that declares +// expectedLength: a tee branch, a branch teed again, and the copy +// new Request() makes of a request's body. Observed through the SELF echo +// service. The declaration is the stream's total, so a fork reports it +// only while nothing has been read before the fork. +// +// DIVERGENCE (ledger #35): forks made before any read carry the declared +// length in TypeScript; C++ gives tee branches no length, so they go out +// chunked. Once anything has been read before a fork, the fork and its +// own forks go out chunked in both (PARITY). A partly read stream itself +// (ledger #36, a C++ defect) is pinned in js-rpc-streams-crossflag-test.js, +// as no consumer here accepts one. +// +// C++ DEFECT (ledger #37): a response whose body declares expectedLength +// (a default source or a TransformStream readable) and is compressed on +// the way out by Content-Encoding: gzip. TypeScript reports no length for +// an encoded body, so it goes out chunked and decodes in full. C++ keeps +// the uncompressed length as the Content-Length, as its tryGetLength() +// ignores the encoding, so the compressed body falls short of it and the +// reader fails with 'Network connection lost.' +// +// PARITY: a body (a default source or a TransformStream readable) that +// delivers less than its declared expectedLength fails the transfer with +// 'Network connection lost.', as a request body and as a response body: +// the declaration is the Content-Length, which the transport enforces. + +import { deepStrictEqual, rejects, strictEqual } from 'node:assert'; +import { usingTsImpl } from 'which-impl'; + +const enc = new TextEncoder(); + +// A source that declares the 10 bytes it delivers, in two chunks. +function tenByteSource() { + return new ReadableStream({ + expectedLength: 10, + start(c) { + c.enqueue(enc.encode('01234')); + c.enqueue(enc.encode('56789')); + c.close(); + }, + }); +} + +async function readChunk(stream) { + const reader = stream.getReader(); + const { value } = await reader.read(); + reader.releaseLock(); + return new TextDecoder().decode(value); +} + +// Posts the request through SELF, and returns the Content-Length it carried +// and the body echoed back. +async function post(env, request) { + const resp = await env.SELF.fetch(request); + const text = await resp.text(); + return [resp.headers.get('observed-content-length'), text]; +} + +function requestWith(body) { + return new Request('http://example.org', { method: 'POST', body }); +} + +export const forkBeforeReadKeepsDeclaredLength = { + async test(ctrl, env) { + const expected = usingTsImpl ? '10' : 'null'; + const [a, b] = tenByteSource().tee(); + const [c, d] = a.tee(); + for (const request of [ + requestWith(b), + requestWith(c), + new Request(requestWith(d)), + ]) { + deepStrictEqual(await post(env, request), [expected, '0123456789']); + } + }, +}; + +const PAYLOAD = 'compress me please '.repeat(2000); + +const BODY_KINDS = ['default', 'transform']; + +// A body that declares expectedLength and delivers text: a default +// source, or a TransformStream's readable through its transformer's +// expectedLength. +function declaredBody(kind, expectedLength, text) { + const bytes = enc.encode(text); + if (kind === 'transform') { + const { readable, writable } = new TransformStream({ expectedLength }); + const writer = writable.getWriter(); + writer.write(bytes); + writer.close(); + return readable; + } + return new ReadableStream({ + expectedLength, + start(c) { + c.enqueue(bytes); + c.close(); + }, + }); +} + +// Declares 10 bytes and delivers 5. +function shortBody(kind) { + return declaredBody(kind, 10, '01234'); +} + +// Served by main.js at /gzip?body=: a body declaring PAYLOAD's +// uncompressed length, sent with Content-Encoding: gzip, which compresses +// it on the way out. +export function compressedResponse(url) { + const body = declaredBody( + url.searchParams.get('body'), + enc.encode(PAYLOAD).length, + PAYLOAD + ); + return new Response(body, { headers: { 'content-encoding': 'gzip' } }); +} + +// Served by main.js at /short?body=. +export function shortResponse(url) { + return new Response(shortBody(url.searchParams.get('body'))); +} + +// PARITY: the declared length is the Content-Length, so a body that falls +// short of it fails the transfer rather than going out complete. +export const shortBodyFailsTransfer = { + async test(ctrl, env) { + const lost = { message: 'Network connection lost.' }; + for (const kind of BODY_KINDS) { + await rejects(post(env, requestWith(shortBody(kind))), lost, kind); + const resp = await env.SELF.fetch( + `http://example.org/short?body=${kind}` + ); + await rejects(resp.text(), lost, kind); + } + }, +}; + +// C++ DEFECT (ledger #37) +export const compressedResponseDropsDeclaredLength = { + async test(ctrl, env) { + for (const kind of BODY_KINDS) { + const resp = await env.SELF.fetch(`http://example.org/gzip?body=${kind}`); + strictEqual(resp.headers.get('content-encoding'), 'gzip', kind); + if (usingTsImpl) { + strictEqual(await resp.text(), PAYLOAD, kind); + } else { + await rejects(resp.text(), { message: 'Network connection lost.' }); + } + } + }, +}; + +export const forkAfterReadSendsChunked = { + async test(ctrl, env) { + // Read before the tee: both branches, a branch teed again, and the + // copy new Request() makes. + { + const rs = tenByteSource(); + strictEqual(await readChunk(rs), '01234'); + const [a, b] = rs.tee(); + const [c, d] = a.tee(); + for (const request of [ + requestWith(b), + requestWith(c), + new Request(requestWith(d)), + ]) { + deepStrictEqual(await post(env, request), ['null', '56789']); + } + } + + // A branch read, then teed again: its branches go out chunked, while + // its unread sibling keeps the declared length (ledger #35). + { + const [a, b] = tenByteSource().tee(); + strictEqual(await readChunk(a), '01234'); + const [c, d] = a.tee(); + for (const request of [requestWith(c), new Request(requestWith(d))]) { + deepStrictEqual(await post(env, request), ['null', '56789']); + } + deepStrictEqual(await post(env, requestWith(b)), [ + usingTsImpl ? '10' : 'null', + '0123456789', + ]); + } + }, +}; diff --git a/src/tests/streams/readable/integration-body-failures.js b/src/tests/streams/readable/integration-body-failures.js index 82f1b4f0d37..f353db65037 100644 --- a/src/tests/streams/readable/integration-body-failures.js +++ b/src/tests/streams/readable/integration-body-failures.js @@ -5,9 +5,10 @@ // What body consumption (text/arrayBuffer/bytes/json) does when it fails // on the consumer's side rather than the source's: the stream is cancelled // with the failure, so the source's cancel() runs and pulls stop, and the -// stream stays locked (PARITY). DIVERGENCE (ledger #21): a TransformStream -// readable declaring `expectedLength` and delivering more is such a -// failure in TypeScript; C++ consumption ignores the declaration. +// stream stays locked (PARITY). DIVERGENCE (ledger #21): a default source +// or a TransformStream readable declaring `expectedLength` and delivering +// more, or closing short of it, is such a failure in TypeScript; C++ +// consumption ignores the declaration. import { strictEqual, deepStrictEqual, ok, rejects, throws } from 'node:assert'; import { usingTsImpl } from 'which-impl'; @@ -183,3 +184,82 @@ export const transformExpectedLengthMet = { strictEqual(await consumed, 'hello'); }, }; + +// Ledger #21 for a default source declaring expectedLength: TypeScript +// rejects the consumer with the same RangeError and cancels the source +// with it; C++ returns everything. The source's second pull closes it +// only after a timer, so the stream is still open when TypeScript finds +// the overflow. +export const sourceExpectedLengthOverflow = { + async test() { + let pulls = 0; + let cancelReason; + const rs = new ReadableStream({ + expectedLength: 3, + async pull(c) { + if (++pulls === 1) { + c.enqueue(enc.encode('hello')); + return; + } + await scheduler.wait(1); + if (cancelReason === undefined) c.close(); + }, + cancel(reason) { + cancelReason = reason; + }, + }); + const consumed = new Response(rs).text(); + if (usingTsImpl) { + const expected = { + name: 'RangeError', + message: 'stream delivered more bytes than its declared expectedLength', + }; + await rejects(consumed, expected); + ok(cancelReason instanceof RangeError); + strictEqual(cancelReason.message, expected.message); + } else { + strictEqual(await consumed, 'hello'); + strictEqual(cancelReason, undefined); + } + }, +}; + +// Ledger #21 for a body that closes short of its declared expectedLength, +// from a default source and from a TransformStream readable: TypeScript +// rejects the consumer with a RangeError naming the declaration; C++ +// returns what arrived. The stream has already closed, so there is nothing +// to cancel: the source's cancel() does not run, and the writable side +// closes cleanly. +export const expectedLengthUnderflow = { + async test() { + let cancelled = false; + const rs = new ReadableStream({ + expectedLength: 5, + start(c) { + c.enqueue(enc.encode('hi')); + c.close(); + }, + cancel() { + cancelled = true; + }, + }); + const ts = new TransformStream({ expectedLength: 5 }); + const writer = ts.writable.getWriter(); + writer.write(enc.encode('hi')); + const closed = writer.close(); + for (const stream of [rs, ts.readable]) { + const consumed = new Response(stream).text(); + if (usingTsImpl) { + await rejects(consumed, { + name: 'RangeError', + message: + 'stream delivered fewer bytes than its declared expectedLength', + }); + } else { + strictEqual(await consumed, 'hi'); + } + } + await closed; + strictEqual(cancelled, false); + }, +}; diff --git a/src/tests/streams/readable/integration-body.js b/src/tests/streams/readable/integration-body.js index d4bc7198c7c..34265bced64 100644 --- a/src/tests/streams/readable/integration-body.js +++ b/src/tests/streams/readable/integration-body.js @@ -327,46 +327,52 @@ export const cancelBodyThenConsume = { }, }; +// A source that declares the 10 bytes it delivers. +function tenByteSource() { + return new ReadableStream({ + expectedLength: 10, + start(c) { + c.enqueue(enc.encode('hellohello')); + c.close(); + }, + }); +} + // Full fetch round-trip: a JS ReadableStream as a fetch() body, echoed // back by the SELF service and read via async iteration (the -// streams-test.js rs shape, workerd#5113 family). +// streams-test.js rs shape, workerd#5113 family). The source's declared +// expectedLength is the subrequest's Content-Length. export const fetchBodyRoundtrip = { async test(ctrl, env) { const resp = await env.SELF.fetch('http://example.org', { method: 'POST', - body: new ReadableStream({ - expectedLength: 10, - start(c) { - c.enqueue(enc.encode('hellohello')); - c.close(); - }, - }), + body: tenByteSource(), }); const chunks = []; for await (const chunk of resp.body) { chunks.push(...chunk); } strictEqual(new TextDecoder().decode(new Uint8Array(chunks)), 'hellohello'); + strictEqual(resp.headers.get('observed-content-length'), '10'); }, }; // The same round-trip with the stream attached via new Request() (the -// streams-test.js rsRequest shape, workerd#5113 regression). +// streams-test.js rsRequest shape, workerd#5113 regression), and with +// that Request copied by new Request(request), which moves the body to +// a new stream: the declared length is the Content-Length of both. export const fetchRequestBodyRoundtrip = { async test(ctrl, env) { - const resp = await env.SELF.fetch( + const makeRequest = () => new Request('http://example.org', { method: 'POST', - body: new ReadableStream({ - expectedLength: 10, - start(c) { - c.enqueue(enc.encode('hellohello')); - c.close(); - }, - }), - }) - ); - strictEqual(await resp.text(), 'hellohello'); + body: tenByteSource(), + }); + for (const request of [makeRequest(), new Request(makeRequest())]) { + const resp = await env.SELF.fetch(request); + strictEqual(await resp.text(), 'hellohello'); + strictEqual(resp.headers.get('observed-content-length'), '10'); + } }, }; diff --git a/src/tests/streams/readable/main.js b/src/tests/streams/readable/main.js index 192101f66e6..eb6c46b3592 100644 --- a/src/tests/streams/readable/main.js +++ b/src/tests/streams/readable/main.js @@ -7,12 +7,28 @@ // // The default export serves the SELF binding used by the integration // tests: it echoes the request body back, letting tests round-trip a -// ReadableStream through the fetch machinery in both directions. +// ReadableStream through the fetch machinery in both directions, and +// reports the Content-Length the request carried. /gzip and /short serve +// expected-length.js's compressed and short responses instead. + +import { compressedResponse, shortResponse } from 'expected-length'; export default { async fetch(request) { + const url = new URL(request.url); + if (url.pathname === '/gzip') { + return compressedResponse(url); + } + if (url.pathname === '/short') { + return shortResponse(url); + } return new Response(request.body, { - headers: { 'content-type': 'application/octet-stream' }, + headers: { + 'content-type': 'application/octet-stream', + 'observed-content-length': String( + request.headers.get('content-length') + ), + }, }); }, }; @@ -200,6 +216,8 @@ export { nonBytesChunkInClosingBatch, transformExpectedLengthOverflow, transformExpectedLengthMet, + sourceExpectedLengthOverflow, + expectedLengthUnderflow, } from 'integration-body-failures'; export { @@ -209,6 +227,13 @@ export { declaredLengthBodyIsExact, } from 'integration-body-memory'; +export { + forkBeforeReadKeepsDeclaredLength, + forkAfterReadSendsChunked, + compressedResponseDropsDeclaredLength, + shortBodyFailsTransfer, +} from 'expected-length'; + export { disturbedStreamIntoResponse, lockedStreamIntoResponse, @@ -252,6 +277,8 @@ export { drainingReaderErrorPropagation, drainingReaderCancelReachesSource, drainingReaderLockExclusivity, + drainingReaderReportsDeclaredLength, + drainingReaderReportsNoLengthAfterRead, } from 'draining-reader'; export { diff --git a/src/tests/streams/readable/readable-modules.capnp b/src/tests/streams/readable/readable-modules.capnp index c83f6dc0944..9f9e6966b7b 100644 --- a/src/tests/streams/readable/readable-modules.capnp +++ b/src/tests/streams/readable/readable-modules.capnp @@ -26,6 +26,7 @@ const modules :List(Workerd.Worker.Module) = [ (name = "integration-body-failures", esModule = embed "integration-body-failures.js"), (name = "integration-body-memory", esModule = embed "integration-body-memory.js"), (name = "integration-locked-disturbed", esModule = embed "integration-locked-disturbed.js"), + (name = "expected-length", esModule = embed "expected-length.js"), (name = "gc", esModule = embed "gc.js"), (name = "then-interceptors", esModule = embed "then-interceptors.js"), (name = "draining-reader", esModule = embed "draining-reader.js"), diff --git a/src/tests/streams/transform/AGENTS.md b/src/tests/streams/transform/AGENTS.md index caa81713966..730f14f4561 100644 --- a/src/tests/streams/transform/AGENTS.md +++ b/src/tests/streams/transform/AGENTS.md @@ -44,6 +44,7 @@ basics) are parity under direct observation and are pinned as such. | 25 | null transformer (every other primitive throws TypeError, and a function is accepted with its members read, in both; spec) | accepted as no transformer (a WebIDL dictionary) | TypeError 'transformer must be an object' (spec: the argument is typed `object`) | `garbageTransformerValidation` | | 26 | readable strategy without a highWaterMark (null or `{}`); either strategy otherwise converts as a WebIDL dictionary in both: a function's members are read, and every primitive except null throws TypeError (messages differ) | the ReadableStream default (1): the TransformStream default (0) applies only when the argument is omitted | 0 (spec ExtractHighWaterMark) | `strategyArgumentShapes` | | 27 | which error wins when two are possible (a primitive transformer's TypeError beats a throwing strategy getter, and a throwing writable strategy getter beats readableType 'bytes', in both; every transformer member converts before readableType is checked, so a non-callable start or flush, TypeError with different messages, or a throwing start getter beats readableType 'bytes', and a non-callable start beats an invalid readable highWaterMark, in both; spec) | each highWaterMark is converted to an integer with the arguments: an invalid writable highWaterMark (TypeError) beats readableType 'bytes' and a throwing readable strategy getter | spec: both strategies are converted, then readableType is checked, then each highWaterMark's range, so readableType 'bytes' (RangeError) and a throwing readable strategy getter beat an invalid writable highWaterMark | `conversionErrorPrecedence` | +| 28 | the Content-Length of a fork (a tee branch, a branch teed again, the copy `new Request()` makes) of the readable side of a transformer declaring `expectedLength`, made before any read (made after a read, the forks go out chunked in both) | none: tee branches never carry a length, so they go out chunked | the declared length, as for a default source (the readable suite's ledger #35) | `transformForkContentLength` | Parity worth noting (probed, pinned): cancel hook runs (not flush) with the reason for both readable.cancel() and writable.abort(), exactly once @@ -102,5 +103,5 @@ C++ implementation; `draining-reader.js` asserts both sides. | `draining-reader.js` | TS only (C++ cell asserts the global's absence): writes flow through the transformer into conduit reads; a readable-side backlog plus close sentinel swept in one batch; flush() output rides the final batch; expectedLength undefined; transformer errors propagate | | `data-volumes.js` | volumes through JS transformers with concurrent producer/consumer: 1 MiB passthrough, 8 MiB XOR (proves every byte passed through the transformer), 4096-chunk value mapping | | `pollution.js` | Object.prototype members stand in for neither an omitted transformer or strategy nor the internal source/sink dictionaries (neither implementation reads them) | -| `expected-length.js` | ledger #21, with the converted length observed as a fetch body's Content-Length (the `roundtrip.js` echo); integral lengths up to the uint64 maximum accepted | +| `expected-length.js` | ledger #21, with the converted length observed as a fetch body's Content-Length (the `roundtrip.js` echo); integral lengths up to the uint64 maximum accepted; ledger #28, the Content-Length of forks of the readable side made before and after a read | | `which-impl.js` / `helpers.js` | implementation detection; consume helpers | diff --git a/src/tests/streams/transform/expected-length.js b/src/tests/streams/transform/expected-length.js index 95c12c25990..80d3049c78e 100644 --- a/src/tests/streams/transform/expected-length.js +++ b/src/tests/streams/transform/expected-length.js @@ -15,8 +15,15 @@ // '5' is 5, null is 0. Every failure is a TypeError. C++ lets the number // 2 ** 64 through its range check to a cast whose result is undefined, so // that one value is asserted for TypeScript only. +// +// DIVERGENCE (ledger #28): the forks of the readable side (a tee branch, a +// branch teed again, the copy new Request() makes) carry the declared +// length in TypeScript when made before any read; C++ gives tee branches +// no length, so they go out chunked. The declaration is the readable's +// total, so once anything has been read before a fork, the forks go out +// chunked in both (PARITY). -import { ok, strictEqual, throws } from 'node:assert'; +import { deepStrictEqual, ok, strictEqual, throws } from 'node:assert'; import { usingTsImpl } from 'which-impl'; function transformWithLength(expectedLength) { @@ -108,3 +115,68 @@ export const transformExpectedLengthRange = { if (usingTsImpl) throws(() => transformWithLength(2 ** 64), TypeError); }, }; + +// The readable side of a transform that declares 10 bytes and is written +// them in two chunks, with the writes and the close still pending. +function tenByteReadable() { + const enc = new TextEncoder(); + const { readable, writable } = transformWithLength(10); + const writer = writable.getWriter(); + const written = Promise.all([ + writer.write(enc.encode('01234')), + writer.write(enc.encode('56789')), + writer.close(), + ]); + return { readable, written }; +} + +async function readChunk(stream) { + const reader = stream.getReader(); + const { value } = await reader.read(); + reader.releaseLock(); + return new TextDecoder().decode(value); +} + +// Posts each fork in turn through SELF, and returns the Content-Length each +// carried and the body echoed back. +async function postForks(env, readable) { + const [a, b] = readable.tee(); + const [c, d] = a.tee(); + const requestWith = (body) => + new Request('http://example.org', { method: 'POST', body }); + const observed = []; + for (const request of [ + requestWith(b), + requestWith(c), + new Request(requestWith(d)), + ]) { + const resp = await env.SELF.fetch(request); + const text = await resp.text(); + observed.push([resp.headers.get('observed-content-length'), text]); + } + return observed; +} + +// DIVERGENCE (ledger #28) before a read; PARITY after one. +export const transformForkContentLength = { + async test(ctrl, env) { + { + const { readable, written } = tenByteReadable(); + const length = usingTsImpl ? '10' : 'null'; + deepStrictEqual( + await postForks(env, readable), + Array(3).fill([length, '0123456789']) + ); + await written; + } + { + const { readable, written } = tenByteReadable(); + strictEqual(await readChunk(readable), '01234'); + deepStrictEqual( + await postForks(env, readable), + Array(3).fill(['null', '56789']) + ); + await written; + } + }, +}; diff --git a/src/tests/streams/transform/main.js b/src/tests/streams/transform/main.js index 31a4c216841..c942e35f441 100644 --- a/src/tests/streams/transform/main.js +++ b/src/tests/streams/transform/main.js @@ -124,6 +124,7 @@ export { transformExpectedLengthRejected, transformExpectedLengthCoercion, transformExpectedLengthRange, + transformForkContentLength, } from 'expected-length'; // Echo handler for the expectedLength fetch-body tests (roundtrip.js, diff --git a/src/tests/streams/writable/AGENTS.md b/src/tests/streams/writable/AGENTS.md index f3aebbc816e..21d510630c5 100644 --- a/src/tests/streams/writable/AGENTS.md +++ b/src/tests/streams/writable/AGENTS.md @@ -106,7 +106,7 @@ promises (they resolve with undefined). | `backpressure.js` | desiredSize accounting/recovery; ready replaced at capacity; WPT floating-point scenarios and invalid sizes, incl. after close() (#9); erroring desiredSize (#11) | | `reentrancy.js` | size()-reentrant write ordering; releaseLock inside size (#12; flag-gated, cf. legacy-writer); controller.error inside write hook; doomed-write size skip; size receiver/arity; size() consulted for a write from inside sink.close() (#15) and sink.abort() | | `then-interceptors.js` | then-getter never fires on writer promise settlement | -| `gc.js` | pending write survives gc() with all user refs dropped (--expose-gc); a stream that has closed, been aborted or errored (by its controller, or by a rejected write) releases its underlying sink, at once when its controller errors it while start() is still pending, while an open one keeps it (WeakRef-checked; parity) | +| `gc.js` | pending write survives gc() with all user refs dropped (--expose-gc); a stream that has closed, been aborted or errored (by its controller, or by a rejected write) releases its underlying sink, at once when its controller errors it while start() is still pending, while an open one keeps it (WeakRef-checked; parity), with the sink algorithms already optimized (both main cells set early Maglev tier-up flags) | | `legacy-ctor-gate.js` | fully-unflagged: ctor Error + absent controller global | | `legacy-writer.js` | pre-flag writer semantics (see Compatibility flags) | | `data-volumes.js` | write-side volumes: 4096 × 16 B writes, single 1 MiB write, 1 MiB / 8 MiB chunked with writer.ready honored; the sink verifies the continuous prime-modulus pattern as chunks arrive | diff --git a/src/tests/streams/writable/writable-cpp.wd-test b/src/tests/streams/writable/writable-cpp.wd-test index b07265fdd5f..2b9a6879d3b 100644 --- a/src/tests/streams/writable/writable-cpp.wd-test +++ b/src/tests/streams/writable/writable-cpp.wd-test @@ -39,6 +39,12 @@ const unitTests :Workerd.Config = ( ) ), ], - # For gc.js. - v8Flags = ["--expose-gc"], + # For gc.js. Maglev tiers up after a few calls, so the sink algorithms + # run optimized well before gc.js: optimized code must not keep an ended + # stream's sink reachable (sinkReleasedOnceEnded). + v8Flags = [ + "--expose-gc", + "--invocation-count-for-maglev=5", + "--minimum-invocations-after-ic-update=2", + ], ); diff --git a/src/tests/streams/writable/writable-ts.wd-test b/src/tests/streams/writable/writable-ts.wd-test index ed78a4673ad..98211a53c0f 100644 --- a/src/tests/streams/writable/writable-ts.wd-test +++ b/src/tests/streams/writable/writable-ts.wd-test @@ -21,6 +21,12 @@ const unitTests :Workerd.Config = ( ) ), ], - # For gc.js. - v8Flags = ["--expose-gc"], + # For gc.js. Maglev tiers up after a few calls, so the sink algorithms + # run optimized well before gc.js: optimized code must not keep an ended + # stream's sink reachable (sinkReleasedOnceEnded). + v8Flags = [ + "--expose-gc", + "--invocation-count-for-maglev=5", + "--minimum-invocations-after-ic-update=2", + ], ); diff --git a/src/workerd/api/js-readable-stream-test.c++ b/src/workerd/api/js-readable-stream-test.c++ index 70052cc042d..08325bb7f11 100644 --- a/src/workerd/api/js-readable-stream-test.c++ +++ b/src/workerd/api/js-readable-stream-test.c++ @@ -3104,9 +3104,9 @@ KJ_TEST("JsReadableStream tryGetLength answers none for encoded queries on non-n fixture.runInIoContext([&](const TestFixture::Environment& env) { auto& js = env.js; - // A queued byte stream declaring the (identity) expectedLength extension: the identity - // query reports it, but an encoded query must NOT (an identity byte count is never a - // valid encoded length). + // Queued streams declaring the (identity) expectedLength extension, from a byte source + // or a default one: the identity query reports it, but an encoded query must NOT (an + // identity byte count is never a valid encoded length). auto underlying = js.obj(); underlying.set(js, "type"_kj, js.str("bytes"_kj)); underlying.set(js, "expectedLength"_kj, jsg::JsValue(js.num(11))); @@ -3114,6 +3114,12 @@ KJ_TEST("JsReadableStream tryGetLength answers none for encoded queries on non-n KJ_EXPECT(KJ_ASSERT_NONNULL(queued.tryGetLength(js)) == 11); KJ_EXPECT(queued.tryGetLength(js, StreamEncoding::GZIP) == kj::none); + auto defaultUnderlying = js.obj(); + defaultUnderlying.set(js, "expectedLength"_kj, jsg::JsValue(js.num(7))); + auto queuedDefault = makeTsStream(js, jsg::JsValue(defaultUnderlying)); + KJ_EXPECT(KJ_ASSERT_NONNULL(queuedDefault.tryGetLength(js)) == 7); + KJ_EXPECT(queuedDefault.tryGetLength(js, StreamEncoding::GZIP) == kj::none); + // Buffer-backed streams sit on the in-memory source, which is identity-only. JsReadableStream buffered(js, kj::str(kData)); KJ_EXPECT(KJ_ASSERT_NONNULL(buffered.tryGetLength(js)) == kData.size()); diff --git a/src/workerd/api/tests/js-rpc-streams-crossflag-test.js b/src/workerd/api/tests/js-rpc-streams-crossflag-test.js index 35cc51f2463..eaf02acf4dd 100644 --- a/src/workerd/api/tests/js-rpc-streams-crossflag-test.js +++ b/src/workerd/api/tests/js-rpc-streams-crossflag-test.js @@ -248,7 +248,8 @@ export const readableLength = { assert.deepStrictEqual(await inspectLength(returned, env.HTTP), expected); } - // Native HTTP bodies and the two queued controller types supply lengths differently. + // Native HTTP bodies, a TransformStream's readable, and default and byte sources each + // declare their length. const text = 'known length'; const bytes = enc.encode(text); const expected = { length: String(bytes.byteLength), text }; @@ -266,6 +267,16 @@ export const readableLength = { .body, ], ['value', valueStream.readable], + [ + 'source', + new ReadableStream({ + expectedLength: bytes.byteLength, + start(c) { + c.enqueue(bytes.slice()); + c.close(); + }, + }), + ], [ 'bytes', new ReadableStream({ @@ -309,6 +320,43 @@ export const readableLength = { length: null, text: 'unsized', }); + + // A stream read from before the transfer has only part of its declared total left. A + // TypeScript origin sends it with no length. A C++ origin still declares the total, a defect + // (the streams readable suite's ledger #36), so the body the peer forwards falls short and + // reading it fails. + for (const type of [undefined, 'bytes']) { + const partlyRead = new ReadableStream({ + type, + expectedLength: bytes.byteLength, + start(c) { + c.enqueue(bytes.slice(0, 5)); + c.enqueue(bytes.slice(5)); + c.close(); + }, + }); + // The byte stream is read through a BYOB reader, as a default read of a C++ byte stream + // takes every queued chunk. + const byob = type === 'bytes'; + const reader = partlyRead.getReader(byob ? { mode: 'byob' } : {}); + const { value } = await reader.read(byob ? new Uint8Array(5) : undefined); + assert.strictEqual(new TextDecoder().decode(value), 'known'); + reader.releaseLock(); + const inspected = env.PEER.inspectLength(partlyRead); + if (usingTsImpl) { + assert.deepStrictEqual( + await inspected, + { length: null, text: ' length' }, + String(type) + ); + } else { + await assert.rejects( + inspected, + { message: 'Network connection lost.' }, + String(type) + ); + } + } }, };