From e60e1029aadd061ca207d4db379e1345276aa037 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 24 Sep 2026 23:01:46 -0700 Subject: [PATCH 1/2] test: use inference-testing fakes and add e2e retry, base64, live suites Units shrink to index placement, short reply, and base64 decode, driven by @intx/inference-testing instead of hand-rolled fetch/scheduler fakes. tests/ adds a 429-then-200 retry on the harness clock and a 401 that does not retry, plus a base64-vs-float check over a real HTTP stub. live/ adds a /v1/embeddings round trip, run by bun run test:live, that skips unless EMBEDDING_E2E_BASE_URL is set. Also closes CL-9132, CL-9114, CL-9116, CL-9137. --- .github/workflows/ci.yml | 4 +- IMPLEMENTATION.md | 15 +- package.json | 2 +- src/embed.test.ts | 399 +++++++++------------------------------ tests/base64.test.ts | 47 +++++ tests/live.test.ts | 47 +++++ tests/retry.test.ts | 55 ++++++ tsconfig.json | 2 +- 8 files changed, 250 insertions(+), 321 deletions(-) create mode 100644 tests/base64.test.ts create mode 100644 tests/live.test.ts create mode 100644 tests/retry.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88e8bc9..3a0c654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,8 @@ jobs: bun-version: latest - run: bun install --frozen-lockfile - run: bunx tsc --noEmit - - run: bun test ./src - - run: bunx prettier --check src + - run: bun run test + - run: bunx prettier --check src tests - run: bun run build - name: Import packed dist with Node run: | diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 4556a07..5131319 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -101,8 +101,13 @@ package depends only on the published surface). ## Tests -`bun run test` is `bun test ./src`. Coverage is the client contract: empty -input, URL and body knobs, auth header presence, batch split and order, -`batchSize` rejection, malformed / short / duplicate-index replies, -index-not-position placement, 429 retry, 401 abort, `probeEmbedDims`, -overridable `Retry-After`. +`bun run test` runs `bun test ./src ./tests`. + +- `src/embed.test.ts` — units on `@intx/inference-testing` fakes: index + placement across batches, short reply, base64 decode. +- `tests/retry.test.ts` — 429 retried once after `Retry-After` on the + harness clock; 401 rejects without retry. +- `tests/base64.test.ts` — `base64` and `float` agree over a real HTTP stub. +- `tests/live.test.ts` — round trip against a real `/v1/embeddings` + (`EMBEDDING_E2E_BASE_URL`, default local Ollama; `EMBEDDING_E2E_MODEL`, + default `nomic-embed-text`). Skips when unreachable. diff --git a/package.json b/package.json index bf6d5c8..c210bf3 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build": "tsc -p tsconfig.build.json", "prepack": "bun run build", "typecheck": "tsc --noEmit", - "test": "bun test ./src", + "test": "bun test ./src ./tests", "format": "prettier -w ." }, "dependencies": { diff --git a/src/embed.test.ts b/src/embed.test.ts index a3bd7d0..ced580b 100644 --- a/src/embed.test.ts +++ b/src/embed.test.ts @@ -1,13 +1,7 @@ -import { type } from "arktype"; -import { describe, expect, it } from "bun:test"; -import type { Dependencies } from "@intx/inference"; +import { afterEach, expect, test } from "bun:test"; +import { setupHarness, type Harness } from "@intx/inference-testing"; -import { - embedTexts, - probeEmbedDims, - EmbedConfigSchema, - type EmbedConfig, -} from "./embed"; +import { embedTexts, type EmbedConfig } from "./embed"; import { EmbeddingRequestError } from "./request"; const CONFIG: EmbedConfig = { @@ -15,324 +9,105 @@ const CONFIG: EmbedConfig = { model: "text-embedding-3-small", }; -type Call = { url: string; init: RequestInit }; - -/** - * `Dependencies` carrying only what `runJSONRequest` reads, with a virtual - * clock so retry backoff costs no wall time and the tests stay deterministic. - */ -function deps(responses: Array Promise)>): { - deps: Dependencies; - calls: Call[]; -} { - const calls: Call[] = []; - let i = 0; - let virtualNow = 0; - - const fetchImpl = (async (url: string, init: RequestInit) => { - calls.push({ url, init }); - const next = responses[Math.min(i++, responses.length - 1)]; - if (next === undefined) throw new Error("no response queued"); - return typeof next === "function" ? await next() : next; - }) as unknown as Dependencies["fetch"]; - - const scheduler = { - now: () => virtualNow, - setTimeout: (callback: () => void, delayMs: number) => { - virtualNow += delayMs; - queueMicrotask(callback); - return () => {}; - }, - }; +let harness: Harness | undefined; +afterEach(() => { + harness?.dispose(); + harness = undefined; +}); - return { - deps: { fetch: fetchImpl, scheduler } as unknown as Dependencies, - calls, - }; +/** Queues one JSON reply per call, served to requests in order. */ +function replies(...bodies: unknown[]): Harness { + const h = setupHarness(); + for (const body of bodies) { + const stream = h.scenario.createStream(); + h.scenario.whenRequestMatches(() => true, stream); + stream.enqueueAll([new TextEncoder().encode(JSON.stringify(body))], { + startAt: 1, + }); + } + return h; } -/** A well-formed reply: one entry per input, each echoing its `index`. */ -function embedding(dims: number, count = 1): Response { - return Response.json({ - data: Array.from({ length: count }, (_unused, index) => ({ - index, - // First component encodes the index so ordering is checkable. - embedding: [index, ...new Array(dims - 1).fill(0.1)], - })), - }); +async function run(h: Harness, pending: Promise): Promise { + await h.run(); + return await pending; } -describe("embedTexts", () => { - it("returns no vectors and issues no request for empty input", async () => { - const { deps: d, calls } = deps([]); - expect(await embedTexts([], CONFIG, { deps: d })).toEqual([]); - expect(calls).toHaveLength(0); - }); - - it("posts to /embeddings under the configured base url", async () => { - const { deps: d, calls } = deps([embedding(3)]); - await embedTexts(["hello"], CONFIG, { deps: d }); - expect(calls[0]?.url).toBe("https://embed.example/v1/embeddings"); - }); - - it("omits dimensions and encoding_format unless configured", async () => { - // Models without Matryoshka reject `dimensions` outright, so an unset knob - // must not appear in the body at all. - const { deps: d, calls } = deps([embedding(3)]); - await embedTexts(["hello"], CONFIG, { deps: d }); - const body = JSON.parse(String(calls[0]?.init.body)) as Record< - string, - unknown - >; - expect(body).not.toHaveProperty("dimensions"); - expect(body).not.toHaveProperty("encoding_format"); - }); - - it("sends dimensions and encoding_format when configured", async () => { - const { deps: d, calls } = deps([embedding(256)]); - await embedTexts( - ["hello"], - { ...CONFIG, dimensions: 256, encodingFormat: "base64" }, - { deps: d }, - ); - const body = JSON.parse(String(calls[0]?.init.body)) as Record< - string, - unknown - >; - expect(body.dimensions).toBe(256); - expect(body.encoding_format).toBe("base64"); - }); - - it("sends an authorization header only when an apiKey is set", async () => { - const { deps: d, calls } = deps([embedding(3), embedding(3)]); - await embedTexts(["a"], CONFIG, { deps: d }); - expect( - (calls[0]?.init.headers as Record).authorization, - ).toBeUndefined(); - - await embedTexts(["a"], { ...CONFIG, apiKey: "sk-test" }, { deps: d }); - expect( - (calls[1]?.init.headers as Record).authorization, - ).toBe("Bearer sk-test"); - }); - - it("splits input into batches and preserves order across them", async () => { - const { deps: d, calls } = deps([embedding(2, 2), embedding(2, 1)]); - const vectors = await embedTexts( +test("places vectors by echoed index and keeps order across batches", async () => { + // Batch 2 restarts at index 0, and batch 1 arrives reversed: a parser that + // placed by reply position would mispair text and vector. + harness = replies( + { + data: [ + { index: 1, embedding: [1] }, + { index: 0, embedding: [0] }, + ], + }, + { data: [{ index: 0, embedding: [2] }] }, + ); + const vectors = await run( + harness, + embedTexts( ["a", "b", "c"], { ...CONFIG, batchSize: 2 }, - { deps: d }, - ); - expect(calls).toHaveLength(2); - expect(JSON.parse(String(calls[0]?.init.body)).input).toEqual(["a", "b"]); - expect(JSON.parse(String(calls[1]?.init.body)).input).toEqual(["c"]); - // Content, not just count: batch 2 restarts at index 0, so a parser that - // concatenated by reply position would still produce three vectors. - expect(vectors.map((v) => v[0])).toEqual([0, 1, 0]); - }); - - it("rejects a batchSize that is not an integer >= 1 instead of looping forever", async () => { - // `batches` advances by `i += size`, so a size of 0 never advances and - // grows the batch list until OOM. Fractional sizes would also stall or - // slice wrongly. The schema requires integer >= 1. - for (const batchSize of [0, -1, 0.5, Number.NaN, 1.5]) { - const { deps: d, calls } = deps([embedding(3)]); - await expect( - embedTexts(["a"], { ...CONFIG, batchSize }, { deps: d }), - ).rejects.toThrow(/batchSize/); - expect(calls).toHaveLength(0); - } - }); - - it("rejects a malformed response rather than returning junk vectors", async () => { - const { deps: d } = deps([ - Response.json({ data: [{ index: 0, embedding: true }] }), - ]); - const rejection = expect(embedTexts(["a"], CONFIG, { deps: d })).rejects; - await rejection.toBeInstanceOf(EmbeddingRequestError); - await rejection.toThrow(/malformed embeddings response/); - }); -}); - -describe("EmbedConfigSchema", () => { - const base = { baseURL: "https://embed.example/v1", model: "m" }; - - it("accepts an integer batchSize >= 1 and an omitted batchSize", () => { - expect(EmbedConfigSchema({ ...base, batchSize: 1 })).toEqual({ - ...base, - batchSize: 1, - }); - expect(EmbedConfigSchema(base)).toEqual(base); - }); - - it("rejects a fractional, non-integer, or sub-1 batchSize", () => { - for (const batchSize of [0, -1, 0.5, Number.NaN, 1.5]) { - expect(EmbedConfigSchema({ ...base, batchSize })).toBeInstanceOf( - type.errors, - ); - } - }); -}); - -describe("retry behaviour inherited from the inference policy", () => { - it("retries a 429 and succeeds on the follow-up", async () => { - // The hand-rolled client this replaced had no retry at all: a single 429 - // failed an entire capture. - const { deps: d, calls } = deps([ - new Response("rate limited", { - status: 429, - headers: { "retry-after": "0" }, - }), - embedding(3), - ]); - const vectors = await embedTexts(["a"], CONFIG, { deps: d }); - expect(calls).toHaveLength(2); - expect(vectors[0]).toHaveLength(3); - }); - - it("aborts immediately on 401 instead of hammering a bad key", async () => { - const { deps: d, calls } = deps([ - new Response("unauthorized", { status: 401 }), - ]); - await expect(embedTexts(["a"], CONFIG, { deps: d })).rejects.toBeInstanceOf( - EmbeddingRequestError, - ); - expect(calls).toHaveLength(1); - }); -}); - -describe("probeEmbedDims", () => { - it("reports the dimensionality the provider actually returned", async () => { - // Never inferred from the model name: it varies by provider and is changed - // by `dimensions`. - const { deps: d } = deps([embedding(384)]); - expect(await probeEmbedDims(CONFIG, { deps: d })).toBe(384); - }); + { deps: harness.deps }, + ), + ); + expect(vectors).toEqual([[0], [1], [2]]); }); -describe("retry-after handling", () => { - it("lets a caller override how Retry-After is read", async () => { - // Parity with ProviderAdapter.extractRetryAfterMs: a provider that signals - // pacing its own way overrides without the transport knowing. - const seen: Array = []; - const { deps: d } = deps([ - new Response("slow down", { - status: 429, - headers: { "x-ratelimit-reset": "7" }, - }), - Response.json({ data: [{ index: 0, embedding: [0.1, 0.2] }] }), - ]); - - await embedTexts(["hello"], CONFIG, { - deps: d, - extractRetryAfterMs: (headers) => { - seen.push(headers.get("x-ratelimit-reset")); - return 0; - }, - }); - expect(seen).toEqual(["7"]); - }); +test("rejects a short reply rather than shifting every later batch", async () => { + harness = replies({ data: [{ index: 0, embedding: [0.1] }] }); + const pending = embedTexts(["a", "b"], CONFIG, { deps: harness.deps }); + await harness.run(); + await expect(pending).rejects.toThrow(/expected 2 embeddings, got 1/); }); -describe("reply integrity", () => { - it("places vectors by the echoed index, not reply position", async () => { - // OpenAI does not promise reply order. Callers pair vectors with their own - // texts positionally, so a reordered reply must not silently mispair them. - const { deps: d } = deps([ - Response.json({ - data: [ - { index: 1, embedding: [9, 9] }, - { index: 0, embedding: [1, 1] }, - ], - }), - ]); - const vectors = await embedTexts(["a", "b"], CONFIG, { deps: d }); - expect(vectors).toEqual([ - [1, 1], - [9, 9], - ]); - }); - - it("rejects a short reply rather than shifting every later batch", async () => { - const { deps: d } = deps([ - Response.json({ data: [{ index: 0, embedding: [0.1] }] }), - ]); - await expect(embedTexts(["a", "b"], CONFIG, { deps: d })).rejects.toThrow( - /expected 2 embeddings, got 1/, - ); - }); - - it("rejects a duplicated index", async () => { - const { deps: d } = deps([ - Response.json({ - data: [ - { index: 0, embedding: [0.1] }, - { index: 0, embedding: [0.2] }, - ], - }), - ]); - await expect(embedTexts(["a", "b"], CONFIG, { deps: d })).rejects.toThrow( - /bad embedding index 0/, - ); - }); - - it("decodes the base64 encoding it advertises", async () => { - // The option is documented and reaches the wire, so the reply form it - // produces has to round-trip. - const packed = Buffer.from(new Float32Array([1.5, -2.5]).buffer).toString( - "base64", - ); - const { deps: d } = deps([ - Response.json({ data: [{ index: 0, embedding: packed }] }), - ]); - const vectors = await embedTexts( +test("decodes the base64 encoding it advertises", async () => { + const packed = Buffer.from(new Float32Array([1.5, -2.5]).buffer).toString( + "base64", + ); + harness = replies({ data: [{ index: 0, embedding: packed }] }); + const vectors = await run( + harness, + embedTexts( ["a"], { ...CONFIG, encodingFormat: "base64" }, - { deps: d }, - ); - expect(vectors).toEqual([[1.5, -2.5]]); - }); + { deps: harness.deps }, + ), + ); + expect(vectors).toEqual([[1.5, -2.5]]); }); -describe("transport failure modes", () => { - it("classifies a 200 that is not JSON instead of leaking a SyntaxError", async () => { - // A captive portal or proxy interstitial in front of a local model server. - const { deps: d } = deps([ - new Response("gateway", { - status: 200, - headers: { "content-type": "text/html" }, - }), - ]); - await expect(embedTexts(["a"], CONFIG, { deps: d })).rejects.toBeInstanceOf( - EmbeddingRequestError, - ); - }); - - it("still enforces timeoutMs when the caller also passes a signal", async () => { - // The two must compose: honouring only the caller's signal would let a - // request hang forever despite a configured per-attempt ceiling. - const controller = new AbortController(); - const calls: Call[] = []; - const d = { - fetch: (async (url: string, init: RequestInit) => { - calls.push({ url, init }); - // Never settles on its own — only the signal can end this. - return await new Promise((_resolve, reject) => { - init.signal?.addEventListener("abort", () => { - reject(new DOMException("aborted", "AbortError")); - }); - }); - }) as unknown as Dependencies["fetch"], - scheduler: { now: () => 0, setTimeout: () => () => {} }, - } as unknown as Dependencies; +test("rejects duplicate indices and undecodable base64", async () => { + for (const data of [ + [ + { index: 0, embedding: [0.1] }, + { index: 0, embedding: [0.2] }, + ], + [ + { index: 0, embedding: "!!!notbase64" }, + { index: 1, embedding: "AAA=" }, + ], + ]) { + harness?.dispose(); + harness = replies({ data }); + const pending = embedTexts(["a", "b"], CONFIG, { deps: harness.deps }); + await harness.run(); + await expect(pending).rejects.toBeInstanceOf(EmbeddingRequestError); + } +}); - await expect( - embedTexts( - ["a"], - { ...CONFIG, timeoutMs: 5 }, - { deps: d, signal: controller.signal }, - ), - ).rejects.toBeInstanceOf(EmbeddingRequestError); - expect(calls).toHaveLength(1); +test("classifies a 200 that is not JSON instead of leaking a SyntaxError", async () => { + harness = setupHarness(); + const stream = harness.scenario.createStream(); + harness.scenario.whenRequestMatches(() => true, stream); + stream.enqueueAll([new TextEncoder().encode("gateway")], { + startAt: 1, + }); + const pending = embedTexts(["a"], CONFIG, { deps: harness.deps }); + await harness.run(); + await expect(pending).rejects.toMatchObject({ + reason: { category: "protocol_mismatch" }, }); }); diff --git a/tests/base64.test.ts b/tests/base64.test.ts new file mode 100644 index 0000000..fb49232 --- /dev/null +++ b/tests/base64.test.ts @@ -0,0 +1,47 @@ +import { afterAll, expect, test } from "bun:test"; +import { type } from "arktype"; + +import { embedTexts } from "../src/index"; + +// Values that are not exact in float32, so the comparison exercises the +// tolerance rather than an exact round trip. +const VECTORS = [ + [0.1, -0.2, 0.3], + [1.1, 2.2, -3.3], +]; + +// A `/v1/embeddings` stub that honours `encoding_format` the way OpenAI does. +const server = Bun.serve({ + port: 0, + async fetch(req) { + const body = type({ "encoding_format?": "string" }).assert( + await req.json(), + ); + return Response.json({ + data: VECTORS.map((vector, index) => ({ + index, + embedding: + body.encoding_format === "base64" + ? Buffer.from(new Float32Array(vector).buffer).toString("base64") + : vector, + })), + }); + }, +}); +afterAll(() => server.stop()); + +test("base64 replies decode to the float vectors within float32 tolerance", async () => { + const config = { baseURL: `${server.url.href}v1`, model: "m" }; + const floats = await embedTexts(["a", "b"], config); + const decoded = await embedTexts(["a", "b"], { + ...config, + encodingFormat: "base64", + }); + + expect(floats).toEqual(VECTORS); + decoded.forEach((vector, i) => { + vector.forEach((value, j) => { + expect(value).toBeCloseTo(VECTORS[i]?.[j] ?? Number.NaN, 6); + }); + }); +}); diff --git a/tests/live.test.ts b/tests/live.test.ts new file mode 100644 index 0000000..c3c7f32 --- /dev/null +++ b/tests/live.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; + +import { embedTexts, probeEmbedDims } from "../src/index"; + +// Any OpenAI-compatible `/v1/embeddings` server; defaults to local Ollama. +const config = { + baseURL: process.env.EMBEDDING_E2E_BASE_URL ?? "http://localhost:11434/v1", + model: process.env.EMBEDDING_E2E_MODEL ?? "nomic-embed-text", +}; + +// Skips unless the server answers and lists the model. +const reachable = await fetch(`${config.baseURL}/models`, { + signal: AbortSignal.timeout(1_000), +}).then( + async (res) => res.ok && (await res.text()).includes(`"${config.model}`), + () => false, +); + +function cosine(a: number[], b: number[]): number { + let dot = 0; + let na = 0; + let nb = 0; + a.forEach((x, i) => { + const y = b[i] ?? 0; + dot += x * y; + na += x * x; + nb += y * y; + }); + return dot / Math.sqrt(na * nb); +} + +describe.skipIf(!reachable)(`live /v1/embeddings at ${config.baseURL}`, () => { + test("embeds three texts in input order at the probed width", async () => { + const texts = ["the cat sat", "quarterly revenue grew", "rust borrowck"]; + const dims = await probeEmbedDims(config); + const vectors = await embedTexts(texts, config); + + expect(vectors).toHaveLength(texts.length); + for (const vector of vectors) expect(vector).toHaveLength(dims); + + // Order: each batched vector matches that text embedded alone. + for (const [i, text] of texts.entries()) { + const [alone] = await embedTexts([text], config); + expect(cosine(vectors[i] ?? [], alone ?? [])).toBeGreaterThan(0.99); + } + }, 120_000); // a cold local model can take a while to page in +}); diff --git a/tests/retry.test.ts b/tests/retry.test.ts new file mode 100644 index 0000000..37da608 --- /dev/null +++ b/tests/retry.test.ts @@ -0,0 +1,55 @@ +import { afterEach, expect, test } from "bun:test"; +import { setupHarness, type Harness } from "@intx/inference-testing"; + +import { EmbeddingRequestError, embedTexts } from "../src/index"; + +const CONFIG = { baseURL: "https://embed.example/v1", model: "m" }; + +let harness: Harness | undefined; +afterEach(() => { + harness?.dispose(); + harness = undefined; +}); + +// Each reply matches exactly one request; an unplanned extra request fails +// `harness.run()` with UnmatchedFetchError. +function reply( + h: Harness, + body: string, + status: number, + headers: Record = {}, +): void { + const stream = h.scenario.createStream(); + h.scenario.whenRequestMatches(() => true, stream, { status, headers }); + stream.enqueueAll([new TextEncoder().encode(body)], { startAt: 1 }); +} + +test("retries a 429 once, after the advertised Retry-After", async () => { + harness = setupHarness({ enableInferenceTimers: true }); + reply(harness, "rate limited", 429, { "retry-after": "30" }); + reply( + harness, + JSON.stringify({ data: [{ index: 0, embedding: [0.5, 0.25] }] }), + 200, + ); + + const pending = embedTexts(["a"], CONFIG, { deps: harness.deps }); + await harness.run(); + + expect(await pending).toEqual([[0.5, 0.25]]); + expect(harness.clock.now()).toBeGreaterThanOrEqual(30_000); +}); + +test("rejects a 401 without retrying", async () => { + harness = setupHarness({ enableInferenceTimers: true }); + reply(harness, "unauthorized", 401); + + const pending = embedTexts(["a"], CONFIG, { deps: harness.deps }); + await harness.run(); + + const rejection = expect(pending).rejects; + await rejection.toBeInstanceOf(EmbeddingRequestError); + await rejection.toMatchObject({ + reason: { category: "credential_failure", statusCode: 401 }, + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 7ba7d4d..5fce075 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,5 +25,5 @@ "lib": ["ESNext"] }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "tests/**/*.ts"] } From 1787a0aeaf4284927d5e4cfedf0e14b4fb5c2c3c Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 25 Sep 2026 06:54:07 -0700 Subject: [PATCH 2/2] test: cover wire, auth, batchSize, Retry-After dates, override, timeout --- IMPLEMENTATION.md | 7 ++- tests/live.test.ts => live/embeddings.test.ts | 24 ++++---- package.json | 1 + src/embed.test.ts | 57 +++++++++++++++++++ tests/retry.test.ts | 56 ++++++++++++++++++ tsconfig.json | 2 +- 6 files changed, 132 insertions(+), 15 deletions(-) rename tests/live.test.ts => live/embeddings.test.ts (68%) diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 5131319..fc3f4a2 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -108,6 +108,7 @@ package depends only on the published surface). - `tests/retry.test.ts` — 429 retried once after `Retry-After` on the harness clock; 401 rejects without retry. - `tests/base64.test.ts` — `base64` and `float` agree over a real HTTP stub. -- `tests/live.test.ts` — round trip against a real `/v1/embeddings` - (`EMBEDDING_E2E_BASE_URL`, default local Ollama; `EMBEDDING_E2E_MODEL`, - default `nomic-embed-text`). Skips when unreachable. +- `live/embeddings.test.ts` (`bun run test:live`, not in `test` or CI) — + round trip against a real `/v1/embeddings` (`EMBEDDING_E2E_BASE_URL`, + required; `EMBEDDING_E2E_MODEL`, default `nomic-embed-text`). + Skips when unset or unreachable. diff --git a/tests/live.test.ts b/live/embeddings.test.ts similarity index 68% rename from tests/live.test.ts rename to live/embeddings.test.ts index c3c7f32..d392e3f 100644 --- a/tests/live.test.ts +++ b/live/embeddings.test.ts @@ -2,19 +2,21 @@ import { describe, expect, test } from "bun:test"; import { embedTexts, probeEmbedDims } from "../src/index"; -// Any OpenAI-compatible `/v1/embeddings` server; defaults to local Ollama. -const config = { - baseURL: process.env.EMBEDDING_E2E_BASE_URL ?? "http://localhost:11434/v1", - model: process.env.EMBEDDING_E2E_MODEL ?? "nomic-embed-text", -}; +// Any OpenAI-compatible `/v1/embeddings` server. Skips unless +// EMBEDDING_E2E_BASE_URL is set, so it never loads a model on this machine. +const baseURL = process.env.EMBEDDING_E2E_BASE_URL; +const model = process.env.EMBEDDING_E2E_MODEL ?? "nomic-embed-text"; // Skips unless the server answers and lists the model. -const reachable = await fetch(`${config.baseURL}/models`, { - signal: AbortSignal.timeout(1_000), -}).then( - async (res) => res.ok && (await res.text()).includes(`"${config.model}`), - () => false, -); +const reachable = + baseURL !== undefined && + (await fetch(`${baseURL}/models`, { + signal: AbortSignal.timeout(1_000), + }).then( + async (res) => res.ok && (await res.text()).includes(`"${model}`), + () => false, + )); +const config = { baseURL: baseURL ?? "", model }; function cosine(a: number[], b: number[]): number { let dot = 0; diff --git a/package.json b/package.json index c210bf3..0e8537e 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "prepack": "bun run build", "typecheck": "tsc --noEmit", "test": "bun test ./src ./tests", + "test:live": "bun test ./live", "format": "prettier -w ." }, "dependencies": { diff --git a/src/embed.test.ts b/src/embed.test.ts index ced580b..9d9aea3 100644 --- a/src/embed.test.ts +++ b/src/embed.test.ts @@ -87,6 +87,10 @@ test("rejects duplicate indices and undecodable base64", async () => { ], [ { index: 0, embedding: "!!!notbase64" }, + { index: 1, embedding: "AAAAAA==" }, + ], + [ + { index: 0, embedding: "AAAAAA==" }, { index: 1, embedding: "AAA=" }, ], ]) { @@ -111,3 +115,56 @@ test("classifies a 200 that is not JSON instead of leaking a SyntaxError", async reason: { category: "protocol_mismatch" }, }); }); + +test("sends only configured fields to {baseURL}/embeddings", async () => { + harness = setupHarness(); + const stream = harness.scenario.createStream(); + harness.scenario.whenRequestBodyMatches( + (body, req) => + req.url === "https://embed.example/v1/embeddings" && + !req.headers.has("authorization") && + body === JSON.stringify({ model: CONFIG.model, input: ["a"] }), + stream, + ); + stream.enqueueAll( + [new TextEncoder().encode('{"data":[{"index":0,"embedding":[1]}]}')], + { startAt: 1 }, + ); + const pending = embedTexts(["a"], CONFIG, { deps: harness.deps }); + expect(await run(harness, pending)).toEqual([[1]]); +}); + +test("sends a bearer token only when apiKey is set", async () => { + harness = setupHarness(); + const stream = harness.scenario.createStream(); + harness.scenario.whenRequestMatches( + (req) => req.headers.get("authorization") === "Bearer sk-test", + stream, + ); + stream.enqueueAll( + [new TextEncoder().encode('{"data":[{"index":0,"embedding":[1]}]}')], + { startAt: 1 }, + ); + const pending = embedTexts( + ["a"], + { ...CONFIG, apiKey: "sk-test" }, + { deps: harness.deps }, + ); + expect(await run(harness, pending)).toEqual([[1]]); +}); + +test("returns [] for empty input without a request", async () => { + harness = setupHarness(); + const pending = embedTexts([], CONFIG, { deps: harness.deps }); + expect(await run(harness, pending)).toEqual([]); +}); + +test("rejects a non-positive batchSize before any request", async () => { + harness = setupHarness(); + for (const batchSize of [0, -1, 1.5]) { + await expect( + embedTexts(["a"], { ...CONFIG, batchSize }, { deps: harness.deps }), + ).rejects.toThrow(/batchSize/); + } + await harness.run(); +}); diff --git a/tests/retry.test.ts b/tests/retry.test.ts index 37da608..d378e64 100644 --- a/tests/retry.test.ts +++ b/tests/retry.test.ts @@ -53,3 +53,59 @@ test("rejects a 401 without retrying", async () => { reason: { category: "credential_failure", statusCode: 401 }, }); }); + +test("waits until an HTTP-date Retry-After", async () => { + harness = setupHarness({ enableInferenceTimers: true }); + const at = new Date(Date.now() + 20_000).toUTCString(); + reply(harness, "rate limited", 429, { "retry-after": at }); + reply(harness, JSON.stringify({ data: [{ index: 0, embedding: [1] }] }), 200); + + const pending = embedTexts(["a"], CONFIG, { deps: harness.deps }); + await harness.run(); + + expect(await pending).toEqual([[1]]); + expect(harness.clock.now()).toBeGreaterThanOrEqual(19_000); +}); + +test("treats a past HTTP-date Retry-After as no wait", async () => { + harness = setupHarness({ enableInferenceTimers: true }); + const at = new Date(Date.now() - 60_000).toUTCString(); + reply(harness, "rate limited", 429, { "retry-after": at }); + reply(harness, JSON.stringify({ data: [{ index: 0, embedding: [1] }] }), 200); + + const pending = embedTexts(["a"], CONFIG, { deps: harness.deps }); + await harness.run(); + + expect(await pending).toEqual([[1]]); + expect(harness.clock.now()).toBeLessThan(1_000); +}); + +test("honors a caller-supplied extractRetryAfterMs", async () => { + harness = setupHarness({ enableInferenceTimers: true }); + reply(harness, "rate limited", 429); + reply(harness, JSON.stringify({ data: [{ index: 0, embedding: [1] }] }), 200); + + const pending = embedTexts(["a"], CONFIG, { + deps: harness.deps, + extractRetryAfterMs: () => 45_000, + }); + await harness.run(); + + expect(await pending).toEqual([[1]]); + expect(harness.clock.now()).toBeGreaterThanOrEqual(45_000); +}); + +test("applies timeoutMs even when the caller passes a signal", async () => { + harness = setupHarness(); + const stream = harness.scenario.createStream(); + harness.scenario.whenRequestMatches(() => true, stream); + + const pending = embedTexts( + ["a"], + { ...CONFIG, timeoutMs: 50 }, + { deps: harness.deps, signal: new AbortController().signal }, + ); + await harness.run(); + + await expect(pending).rejects.toBeInstanceOf(EmbeddingRequestError); +}); diff --git a/tsconfig.json b/tsconfig.json index 5fce075..713b1f6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,5 +25,5 @@ "lib": ["ESNext"] }, - "include": ["src/**/*.ts", "tests/**/*.ts"] + "include": ["src/**/*.ts", "tests/**/*.ts", "live/**/*.ts"] }