diff --git a/package.json b/package.json index 218ba4c..9714da8 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "prepack": "bun run build", "typecheck": "tsc --noEmit", "test": "bun test ./src", + "test:e2e": "bun test ./tests", "format": "prettier -w ." }, "dependencies": { diff --git a/src/rerank.test.ts b/src/rerank.test.ts index 09ec363..4bf616e 100644 --- a/src/rerank.test.ts +++ b/src/rerank.test.ts @@ -1,12 +1,15 @@ -import { describe, expect, it } from "bun:test"; -import type { Dependencies } from "@intx/inference"; +import { afterEach, describe, expect, test } from "bun:test"; +import { setupHarness, type Harness } from "@intx/inference-testing"; -import { rerankDocuments, type RerankConfig } from "./rerank"; +import { + rerankDocuments, + type RerankConfig, + type RerankOptions, +} from "./rerank"; import { createRerankAdapterRegistry, rerankAdapterRegistry, type RerankAdapter, - type RerankAPIStyle, } from "./adapters"; import { extractRetryAfterMs, RerankRequestError } from "./request"; @@ -16,253 +19,266 @@ const DOCS = [ { id: "c", text: "charlie" }, ]; -type Call = { url: string; init: RequestInit }; - -function deps(responses: Response[]): { deps: Dependencies; calls: Call[] } { - const calls: Call[] = []; - let i = 0; - let virtualNow = 0; - - const fetchImpl = (async (url: string, init: RequestInit) => { - calls.push({ url, init }); - const next = responses[Math.min(i++, responses.length - 1)]; - if (next === undefined) throw new Error("no response queued"); - return next; - }) as unknown as Dependencies["fetch"]; - - const scheduler = { - now: () => virtualNow, - setTimeout: (callback: () => void, delayMs: number) => { - virtualNow += delayMs; - queueMicrotask(callback); - return () => {}; - }, - }; +const BASE_URL = "https://rerank.example"; - return { - deps: { fetch: fetchImpl, scheduler } as unknown as Dependencies, - calls, - }; +let harness: Harness; +let undisposed: Harness | undefined; + +afterEach(() => { + undisposed?.dispose(); + undisposed = undefined; +}); + +function setup(): Harness { + harness = setupHarness({ enableInferenceTimers: true }); + undisposed = harness; + return harness; +} + +function reply( + status: number, + body: string, + headers: Record = {}, +): void { + const stream = harness.scenario.createStream(); + stream.enqueue(new TextEncoder().encode(body)); + stream.closeAt(0); + harness.scenario.whenRequestMatches(() => true, stream, { status, headers }); } -function config(apiStyle: RerankAPIStyle): RerankConfig { - // Cohere and Voyage reject a request without a model; TEI ignores it. - return { baseURL: "https://rerank.example", apiStyle, model: "rerank-v2" }; +async function rerank( + config: RerankConfig, + docs = DOCS, + registry = rerankAdapterRegistry, +): Promise { + const options: RerankOptions = { deps: harness.deps, registry }; + const pending = rerankDocuments("q", docs, config, options).catch( + (error: unknown) => error, + ); + await harness.run(); + return pending; } -function body(call: Call | undefined): Record { - return JSON.parse(String(call?.init.body)) as Record; +function requestBodies(): Promise { + return Promise.all( + harness.scenario.matchedRequests().map((req) => req.json()), + ); } describe("wire formats", () => { - it("TEI posts /rerank with texts and reads a bare array", async () => { - const { deps: d, calls } = deps([ - Response.json([ + test.each([ + { + apiStyle: "tei", + path: "/rerank", + sent: { query: "q", texts: ["alpha", "bravo", "charlie"] }, + received: [ + { index: 1, score: 0.2 }, { index: 2, score: 0.9 }, - { index: 0, score: 0.1 }, - ]), - ]); - const out = await rerankDocuments("q", DOCS, config("tei"), { deps: d }); + ], + }, + { + apiStyle: "cohere", + path: "/v2/rerank", + sent: { + model: "rerank-v2", + query: "q", + documents: ["alpha", "bravo", "charlie"], + }, + received: { + results: [ + { index: 1, relevance_score: 0.2 }, + { index: 2, relevance_score: 0.9 }, + ], + }, + }, + { + apiStyle: "voyage", + path: "/v1/rerank", + sent: { + model: "rerank-v2", + query: "q", + documents: ["alpha", "bravo", "charlie"], + }, + received: { + data: [ + { index: 1, relevance_score: 0.2 }, + { index: 2, relevance_score: 0.9 }, + ], + }, + }, + ])("$apiStyle", async ({ apiStyle, path, sent, received }) => { + setup(); + reply(200, JSON.stringify(received)); + + const out = await rerank({ + baseURL: BASE_URL, + apiStyle, + model: "rerank-v2", + }); - expect(calls[0]?.url).toBe("https://rerank.example/rerank"); - expect(body(calls[0]).texts).toEqual(["alpha", "bravo", "charlie"]); + // Mapped through the reply's index, not its position, and sorted. expect(out).toEqual([ { id: "c", score: 0.9 }, - { id: "a", score: 0.1 }, + { id: "b", score: 0.2 }, ]); - }); - - it("Cohere posts /v2/rerank with documents and reads results[]", async () => { - const { deps: d, calls } = deps([ - Response.json({ results: [{ index: 1, relevance_score: 0.7 }] }), + expect(harness.scenario.matchedRequests().map((req) => req.url)).toEqual([ + `${BASE_URL}${path}`, ]); - const out = await rerankDocuments("q", DOCS, config("cohere"), { deps: d }); - - expect(calls[0]?.url).toBe("https://rerank.example/v2/rerank"); - expect(body(calls[0]).documents).toEqual(["alpha", "bravo", "charlie"]); - expect(out).toEqual([{ id: "b", score: 0.7 }]); + expect(await requestBodies()).toEqual([sent]); }); +}); - it("Voyage posts /v1/rerank and reads data[]", async () => { - const { deps: d, calls } = deps([ - Response.json({ data: [{ index: 0, relevance_score: 0.5 }] }), - ]); - const out = await rerankDocuments("q", DOCS, config("voyage"), { deps: d }); +describe("retry", () => { + test("backs off a 429 by Retry-After on the harness clock", async () => { + setup(); + reply(429, "slow down", { "retry-after": "2" }); + reply(200, JSON.stringify([{ index: 0, score: 0.4 }])); - expect(calls[0]?.url).toBe("https://rerank.example/v1/rerank"); - expect(out).toEqual([{ id: "a", score: 0.5 }]); - }); + const out = await rerank({ baseURL: BASE_URL, apiStyle: "tei" }); - it("requires a model for the styles that mandate one", async () => { - // Cohere and Voyage 400 without it. Fail by name here rather than in prod. - const { deps: d, calls } = deps([]); - for (const style of ["cohere", "voyage"] as const) { - await expect( - rerankDocuments( - "q", - DOCS, - { baseURL: "https://rerank.example", apiStyle: style }, - { deps: d }, - ), - ).rejects.toThrow("model must be a string (was undefined)"); - } - expect(calls).toHaveLength(0); + expect(out).toEqual([{ id: "a", score: 0.4 }]); + expect(harness.scenario.matchedRequests()).toHaveLength(2); + expect(harness.clock.now()).toBe(2_000); }); - it("does not require a model for TEI, which serves a fixed one", async () => { - const { deps: d } = deps([Response.json([{ index: 0, score: 0.3 }])]); - const out = await rerankDocuments( - "q", - DOCS, - { baseURL: "https://rerank.example", apiStyle: "tei" }, - { deps: d }, + test("aborts on a 401 without retrying", async () => { + setup(); + reply(401, "bad key"); + + const error = await rerank({ baseURL: BASE_URL, apiStyle: "tei" }); + + expect(error).toBeInstanceOf(RerankRequestError); + expect((error as RerankRequestError).reason.category).toBe( + "credential_failure", ); - expect(out).toEqual([{ id: "a", score: 0.3 }]); + expect((error as RerankRequestError).url).toBe(`${BASE_URL}/rerank`); + expect(harness.scenario.matchedRequests()).toHaveLength(1); }); - it("rejects an unknown apiStyle by name", async () => { - const { deps: d } = deps([]); - await expect( - rerankDocuments( - "q", - DOCS, - { baseURL: "https://rerank.example", apiStyle: "nope" }, - { deps: d }, - ), - ).rejects.toThrow(/Unknown rerank API style "nope"/); - }); + test("reads Retry-After through the adapter override", async () => { + setup(); + const tei = rerankAdapterRegistry.resolve("tei"); + const registry = createRerankAdapterRegistry({ + tei: { + buildRequest: tei.buildRequest, + parseResponse: tei.parseResponse, + extractRetryAfterMs: (headers) => + Number(headers.get("x-ratelimit-reset")) * 1_000, + }, + }); + reply(429, "slow down", { "x-ratelimit-reset": "7" }); + reply(200, JSON.stringify([{ index: 0, score: 0.4 }])); - it("rejects a schema-invalid config before posting", async () => { - // timeoutMs: 0 is a RerankConfigSchema error. Validate at the call - // boundary so a bad config never reaches /rerank. - const { deps: d, calls } = deps([]); - await expect( - rerankDocuments( - "q", - DOCS, - { ...config("tei"), timeoutMs: 0 }, - { deps: d }, - ), - ).rejects.toThrow("timeoutMs must be positive (was 0)"); - expect(calls).toHaveLength(0); + await rerank({ baseURL: BASE_URL, apiStyle: "tei" }, DOCS, registry); + + expect(harness.clock.now()).toBe(7_000); }); +}); + +describe("errors", () => { + test("a reply in another provider's shape is a protocol mismatch", async () => { + setup(); + reply(200, JSON.stringify({ results: [{ index: 0, relevance_score: 1 }] })); + + const error = await rerank({ baseURL: BASE_URL, apiStyle: "tei" }); - it("rejects a reply in another provider's shape", async () => { - // Cohere's body through the TEI adapter: the formats do not converge, and - // a mismatch must fail rather than silently rerank nothing. - const { deps: d } = deps([ - Response.json({ results: [{ index: 0, relevance_score: 0.5 }] }), - ]); - const error = await rerankDocuments("q", DOCS, config("tei"), { - deps: d, - }).catch((e: unknown) => e); expect(error).toBeInstanceOf(RerankRequestError); expect((error as RerankRequestError).reason.category).toBe( "protocol_mismatch", ); - expect((error as RerankRequestError).message).toMatch( - /malformed tei rerank response/, - ); }); -}); -describe("result mapping", () => { - it("maps by response index, not reply position", async () => { - // TEI's reply is explicitly unordered. Trusting position would silently - // attach the top score to the wrong document. - const { deps: d } = deps([ - Response.json([ - { index: 1, score: 0.2 }, - { index: 2, score: 0.95 }, - ]), - ]); - const out = await rerankDocuments("q", DOCS, config("tei"), { deps: d }); - expect(out.map((r) => r.id)).toEqual(["c", "b"]); - }); + test("an out-of-range index is a protocol mismatch", async () => { + setup(); + reply(200, JSON.stringify([{ index: 99, score: 0.5 }])); - it("raises when a provider returns an out-of-range index", async () => { - const { deps: d } = deps([Response.json([{ index: 99, score: 0.5 }])]); - await expect( - rerankDocuments("q", DOCS, config("tei"), { deps: d }), - ).rejects.toThrow( - new RerankRequestError( - { - category: "protocol_mismatch", - message: "rerank response index 99 out of bounds for 3 documents", - }, - "https://rerank.example/rerank", - ), - ); - }); + const error = await rerank({ baseURL: BASE_URL, apiStyle: "tei" }); - it("returns nothing and issues no request for empty input", async () => { - const { deps: d, calls } = deps([]); - expect(await rerankDocuments("q", [], config("tei"), { deps: d })).toEqual( - [], + expect(error).toBeInstanceOf(RerankRequestError); + expect((error as RerankRequestError).reason.message).toBe( + "rerank response index 99 out of bounds for 3 documents", ); - expect(calls).toHaveLength(0); }); -}); -describe("retry behaviour inherited from the inference policy", () => { - it("retries a 429 and succeeds on the follow-up", async () => { - const { deps: d, calls } = deps([ - new Response("slow down", { - status: 429, - headers: { "retry-after": "0" }, - }), - Response.json([{ index: 0, score: 0.4 }]), - ]); - const out = await rerankDocuments("q", DOCS, config("tei"), { deps: d }); - expect(calls).toHaveLength(2); - expect(out).toEqual([{ id: "a", score: 0.4 }]); + test.each(["cohere", "voyage"])( + "%s without a model fails before posting", + async (apiStyle) => { + setup(); + const error = await rerank({ baseURL: BASE_URL, apiStyle }); + + expect((error as Error).message).toBe( + "model must be a string (was undefined)", + ); + expect(harness.scenario.matchedRequests()).toHaveLength(0); + }, + ); + + test("an invalid config fails before posting", async () => { + setup(); + const error = await rerank({ + baseURL: BASE_URL, + apiStyle: "tei", + timeoutMs: 0, + }); + + expect((error as Error).message).toBe("timeoutMs must be positive (was 0)"); + expect(harness.scenario.matchedRequests()).toHaveLength(0); }); - it("aborts immediately on 401 instead of hammering a bad key", async () => { - const { deps: d, calls } = deps([new Response("nope", { status: 401 })]); - await expect( - rerankDocuments("q", DOCS, config("tei"), { deps: d }), - ).rejects.toBeInstanceOf(RerankRequestError); - expect(calls).toHaveLength(1); + test("an unknown apiStyle, including an inherited name, fails by name", async () => { + setup(); + for (const apiStyle of ["nope", "toString"]) { + const error = await rerank({ baseURL: BASE_URL, apiStyle }); + expect((error as Error).message).toMatch( + `Unknown rerank API style "${apiStyle}"`, + ); + } }); +}); - it("lets an adapter override how Retry-After is read", async () => { - // Mirrors ProviderAdapter.extractRetryAfterMs: a provider that signals - // pacing its own way overrides without the transport knowing. - const seen: Array = []; - const registry = createRerankAdapterRegistry({ - tei: { - ...rerankAdapterRegistry.resolve("tei"), - extractRetryAfterMs: (headers) => { - seen.push(headers.get("x-ratelimit-reset")); - return 0; - }, - }, - }); - const { deps: d } = deps([ - new Response("slow down", { - status: 429, - headers: { "x-ratelimit-reset": "7" }, - }), - Response.json([{ index: 0, score: 0.4 }]), - ]); - await rerankDocuments("q", DOCS, config("tei"), { deps: d, registry }); - expect(seen).toEqual(["7"]); +test("empty input returns nothing without a request", async () => { + setup(); + expect(await rerank({ baseURL: BASE_URL, apiStyle: "tei" }, [])).toEqual([]); + expect(harness.scenario.matchedRequests()).toHaveLength(0); +}); + +test("a registry is not mutable through the map it was built from", () => { + const source: Record = { + tei: rerankAdapterRegistry.resolve("tei"), + }; + const registry = createRerankAdapterRegistry(source); + source.voyage = rerankAdapterRegistry.resolve("voyage"); + expect(registry.has("voyage")).toBe(false); +}); + +test("options default to global fetch", async () => { + const server = Bun.serve({ + port: 0, + fetch: () => Response.json([{ index: 0, score: 0.5 }]), }); + try { + const out = await rerankDocuments("q", DOCS, { + baseURL: server.url.origin, + apiStyle: "tei", + }); + expect(out).toEqual([{ id: "a", score: 0.5 }]); + } finally { + await server.stop(); + } }); describe("retry-after parsing", () => { - it("reads the seconds form", () => { + test("reads the seconds form", () => { expect(extractRetryAfterMs(new Headers({ "retry-after": "2" }))).toBe( 2_000, ); }); - it("returns undefined when the header is absent", () => { + test("returns undefined when the header is absent", () => { expect(extractRetryAfterMs(new Headers())).toBeUndefined(); }); - it("clamps a date already past to zero", () => { + test("clamps a date already past to zero", () => { expect( extractRetryAfterMs( new Headers({ "retry-after": "Wed, 21 Oct 2015 07:28:00 GMT" }), @@ -270,42 +286,13 @@ describe("retry-after parsing", () => { ).toBe(0); }); - it("clamps a negative seconds value to zero", () => { + test("clamps a negative seconds value to zero", () => { expect(extractRetryAfterMs(new Headers({ "retry-after": "-5" }))).toBe(0); }); - it("treats a blank header as absent", () => { + test("treats a blank header as absent", () => { expect( extractRetryAfterMs(new Headers({ "retry-after": " " })), ).toBeUndefined(); }); }); - -describe("adapter registry", () => { - it("resolves each built-in style", () => { - for (const style of ["tei", "cohere", "voyage"]) { - expect(rerankAdapterRegistry.resolve(style).buildRequest).toBeTypeOf( - "function", - ); - } - }); - - it("does not let an inherited member masquerade as an adapter", () => { - // A plain object lookup returns Object.prototype.toString here, which then - // fails later with an opaque TypeError. apiStyle often comes from config, - // so the failure must be immediate and by name. - expect(rerankAdapterRegistry.has("toString")).toBe(false); - expect(() => rerankAdapterRegistry.resolve("toString")).toThrow( - /Unknown rerank API style "toString"/, - ); - }); - - it("is not mutable through the map it was built from", () => { - const source: Record = { - tei: rerankAdapterRegistry.resolve("tei"), - }; - const registry = createRerankAdapterRegistry(source); - source.voyage = rerankAdapterRegistry.resolve("voyage"); - expect(registry.has("voyage")).toBe(false); - }); -}); diff --git a/tests/tei.e2e.test.ts b/tests/tei.e2e.test.ts new file mode 100644 index 0000000..6014898 --- /dev/null +++ b/tests/tei.e2e.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, test } from "bun:test"; + +import { rerankDocuments } from "../src/index"; + +// A local TEI reranker, e.g. +// `docker run -p 8080:80 ghcr.io/huggingface/text-embeddings-inference:cpu-latest --model-id BAAI/bge-reranker-base` +const baseURL = process.env.TEI_RERANK_URL ?? "http://localhost:8080"; + +const reachable = await fetch(`${baseURL}/health`, { + signal: AbortSignal.timeout(1_000), +}) + .then((res) => res.ok) + .catch(() => false); + +describe.skipIf(!reachable)(`TEI at ${baseURL}`, () => { + test("ranks the relevant document first and maps every id back", async () => { + const docs = [ + { id: "weather", text: "Rain is expected across the valley tomorrow." }, + { id: "paris", text: "Paris is the capital and largest city of France." }, + { id: "recipe", text: "Whisk the eggs before folding in the flour." }, + ]; + + const out = await rerankDocuments("What is the capital of France?", docs, { + baseURL, + apiStyle: "tei", + }); + + expect(out[0]?.id).toBe("paris"); + expect(out.map((r) => r.id).sort()).toEqual(["paris", "recipe", "weather"]); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 7ba7d4d..5fce075 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,5 +25,5 @@ "lib": ["ESNext"] }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "tests/**/*.ts"] }