diff --git a/.github/workflows/model-review.yml b/.github/workflows/model-review.yml new file mode 100644 index 00000000..dc3b9e45 --- /dev/null +++ b/.github/workflows/model-review.yml @@ -0,0 +1,53 @@ +name: Model review +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * 1" + +permissions: + contents: read + +jobs: + review: + name: Review models + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: ".nvmrc" + + - name: Review models + id: review + run: | + status=0 + node scripts/model-review.mjs > "$RUNNER_TEMP/report.md" || status=$? + # A catalog that could not be read leaves no report, and fails the job here. + test -s "$RUNNER_TEMP/report.md" + cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY" + echo "status=$status" >> "$GITHUB_OUTPUT" + + - name: Update the review issue + run: | + gh label create model-review --description "Weekly transcription model review" --force + number=$(gh issue list --label model-review --state open --limit 1 --json number --jq '.[0].number // empty') + if [ -n "$number" ]; then + gh issue edit "$number" --body-file "$RUNNER_TEMP/report.md" + elif grep -qE '^## (Curated models gone or ending|Newer models in a curated family)$' "$RUNNER_TEMP/report.md"; then + # Uncurated candidates alone do not reopen a review the maintainer closed. + gh issue create --title "Review transcription models" --label model-review --body-file "$RUNNER_TEMP/report.md" + fi + env: + GH_REPO: ${{ github.repository }} + + - name: Fail when a curated model is gone or past its end date + if: steps.review.outputs.status != '0' + run: exit 1 diff --git a/AGENTS.md b/AGENTS.md index aa22c9d5..36d83bf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,7 @@ root symlinks for local Obsidian loading. - `npm run test:tooling`: run the real-process Svelte checker failure tests and the build's syntax floor test, including rejection of tagged templates that disable Obsidian's eval cache. CI runs these separately on every PR. - `npm run build`: type-check and produce the production plugin bundle. - `npm run plant -- [ ...] [-- ]`: plant a defect and pass only when the exactly named test fails on it and the command passes again once restored. Each search must match exactly once, and the plant must type-check. The files are restored on exit or interrupt. The command defaults to the unit suite filtered to the test name. +- `npm run review:models`: compare the curated transcription models in `src/services/transcription/providers/` with OpenRouter's public catalog, report models that are gone, ending, newer in a curated family, or not curated, and exit 1 when a curated OpenRouter model is gone or past its end date. The `Model review` workflow runs it weekly and keeps one `model-review` issue current. It is not a required check. - `npm run docs:build`: build the Starlight documentation and check its internal links. `@typescript/native` aliases TypeScript 7. `typecheck` and `lint:tsconfig` call diff --git a/package.json b/package.json index 390c718c..448887fd 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "size:data": "node scripts/persistence-size.mjs", "count:writes": "node scripts/write-counts.mjs", "plant": "node scripts/plant.mjs", + "review:models": "node scripts/model-review.mjs", "typecheck": "node node_modules/@typescript/native/bin/tsc --project tsconfig.json --noEmit && node node_modules/@typescript/native/bin/tsc --project tsconfig.test.json --noEmit", "lint": "oxlint --deny-warnings src tests/e2e tests/tooling scripts .claude/skills/verify && svelte-policy", "lint:tsconfig": "node node_modules/@typescript/native/bin/tsc --project tsconfig.json --noEmit --noCheck && node node_modules/@typescript/native/bin/tsc --project tsconfig.test.json --noEmit --noCheck", diff --git a/scripts/model-review.mjs b/scripts/model-review.mjs new file mode 100644 index 00000000..930fc6e3 --- /dev/null +++ b/scripts/model-review.mjs @@ -0,0 +1,190 @@ +// Reports whether the transcription models PodNotes curates are still current. The curated +// IDs are read from the provider files, and the models OpenRouter's public catalog lists +// are compared with them. Exits 1 when a curated OpenRouter model is gone or past its end date. +// +// npm run review:models [-- --catalog ] +import { readdirSync, readFileSync } from "node:fs"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const PROVIDERS = fileURLToPath( + new URL("../src/services/transcription/providers/", import.meta.url), +); +const CATALOG_URL = "https://openrouter.ai/api/v1/models?output_modalities=transcription"; + +/** + * @typedef {{ provider: string, id: string }} Curated + * @typedef {{ id: string, name?: string, created?: number, expiration_date?: string | null }} CatalogModel + */ + +/** + * Every model ID in the provider files, with the provider whose file lists it. + * @param {string} [directory] + * @returns {Curated[]} + */ +export function curatedModels(directory = PROVIDERS) { + return readdirSync(directory) + .filter((file) => file.endsWith(".ts") && !file.endsWith(".test.ts")) + .sort() + .flatMap((file) => { + const source = readFileSync(new URL(file, pathToFileURL(`${directory}/`)), "utf8"); + return [...source.matchAll(/^\s+id: "([^"]+)",$/gm)].map(([, id]) => ({ + provider: file.slice(0, -".ts".length), + id, + })); + }); +} + +/** + * A model's family is its vendor and the words of its name that are not a version, so + * `elevenlabs/scribe-v2` and the direct `scribe_v3` belong to one family. + * @param {string} provider the vendor of an ID that names none + * @param {string} id + */ +function parse(provider, id) { + const slash = id.indexOf("/"); + const vendor = slash < 0 ? provider : id.slice(0, slash); + const words = id + .slice(slash + 1) + .toLowerCase() + .split(/[-_]/); + const isVersion = (/** @type {string} */ word) => /^v?\d+(\.\d+)*$/.test(word); + return { + family: [vendor, ...words.filter((word) => !isVersion(word))].join("-"), + version: words + .filter(isVersion) + .flatMap((word) => word.replace(/^v/, "").split(".").map(Number)), + }; +} + +/** + * @param {number[]} a + * @param {number[]} b + */ +function compare(a, b) { + for (let i = 0; i < Math.max(a.length, b.length); i++) { + const difference = (a[i] ?? 0) - (b[i] ?? 0); + if (difference !== 0) return difference; + } + return 0; +} + +/** + * @param {{ curated: Curated[], catalog: CatalogModel[], today: string }} input + * @returns {{ + * gone: string[], + * ending: { id: string, ends: string, past: boolean }[], + * newer: { model: CatalogModel, than: string }[], + * candidates: CatalogModel[], + * }} + */ +export function review({ curated, catalog, today }) { + const listed = new Map(catalog.map((model) => [model.id, model])); + const gone = []; + const ending = []; + for (const { provider, id } of curated) { + if (provider !== "openrouter") continue; + const model = listed.get(id); + if (!model) gone.push(id); + else if (model.expiration_date) { + const ends = model.expiration_date.slice(0, 10); + ending.push({ id, ends, past: ends <= today }); + } + } + + /** @type {Map} */ + const families = new Map(); + for (const { provider, id } of curated) { + const { family, version } = parse(provider, id); + families.set(family, [...(families.get(family) ?? []), { provider, id, version }]); + } + + /** + * The newest curated model a catalog model is measured against: the OpenRouter one when + * the family has any, since a direct provider's version says nothing about it. + * @param {{ provider: string, id: string, version: number[] }[]} own + */ + const latest = (own) => { + const viaOpenRouter = own.filter(({ provider }) => provider === "openrouter"); + return (viaOpenRouter.length > 0 ? viaOpenRouter : own).reduce((a, b) => + compare(b.version, a.version) > 0 ? b : a, + ); + }; + + const newer = []; + const candidates = []; + const fresh = catalog + .filter((model) => !model.expiration_date) + .sort((a, b) => (b.created ?? 0) - (a.created ?? 0)); + for (const model of fresh) { + const { family, version } = parse("", model.id); + const own = families.get(family); + if (!own) candidates.push(model); + else if (compare(version, latest(own).version) > 0) { + newer.push({ model, than: latest(own).id }); + } else if (!own.some((curatedModel) => compare(version, curatedModel.version) === 0)) { + candidates.push(model); + } + } + return { gone, ending, newer, candidates }; +} + +/** + * @param {ReturnType} result + * @param {{ catalogSize: number, today: string }} context + */ +export function renderReport({ gone, ending, newer, candidates }, { catalogSize, today }) { + /** @type {string[]} */ + const lines = [ + "# Transcription model review", + "", + `Checked ${today} against the OpenRouter catalog (${catalogSize} models).`, + ]; + /** + * @param {string} heading + * @param {string[]} items + */ + const section = (heading, items) => { + if (items.length > 0) lines.push("", `## ${heading}`, "", ...items.map((i) => `- ${i}`)); + }; + section("Curated models gone or ending", [ + ...gone.map((id) => `\`${id}\` is not in the catalog.`), + ...ending.map(({ id, ends, past }) => `\`${id}\` ${past ? "ended" : "ends"} ${ends}.`), + ]); + section( + "Newer models in a curated family", + newer.map(({ model, than }) => `\`${model.id}\` is newer than \`${than}\`.`), + ); + section( + "Candidates, not curated", + candidates.map(({ id, name }) => `\`${id}\`${name ? ` (${name})` : ""}`), + ); + if (gone.length + ending.length + newer.length + candidates.length === 0) { + lines.push("", "Nothing to review."); + } + return `${lines.join("\n")}\n`; +} + +/** @param {string[]} args */ +async function main(args) { + const file = args[args.indexOf("--catalog") + 1]; + const body = + args.includes("--catalog") && file + ? JSON.parse(readFileSync(file, "utf8")) + : await fetch(CATALOG_URL, { signal: AbortSignal.timeout(30_000) }).then((response) => { + if (!response.ok) throw new Error(`The catalog answered ${response.status}.`); + return response.json(); + }); + /** @type {CatalogModel[]} */ + const catalog = body.data; + const today = new Date().toISOString().slice(0, 10); + const result = review({ curated: curatedModels(), catalog, today }); + process.stdout.write(renderReport(result, { catalogSize: catalog.length, today })); + if (result.gone.length > 0 || result.ending.some(({ past }) => past)) process.exitCode = 1; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).catch((error) => { + console.error(`Model review failed: ${error instanceof Error ? error.message : error}`); + process.exitCode = 1; + }); +} diff --git a/scripts/model-review.test.ts b/scripts/model-review.test.ts new file mode 100644 index 00000000..c1a38d4b --- /dev/null +++ b/scripts/model-review.test.ts @@ -0,0 +1,12 @@ +import { expect, it } from "vitest"; +import { PROVIDERS } from "src/services/transcription/registry"; +import { curatedModels } from "./model-review.mjs"; + +it("reads the model IDs the providers offer", () => { + const offered = Object.entries(PROVIDERS).flatMap(([provider, { models }]) => + models.map(({ id }) => ({ provider, id })), + ); + const byProvider = (a: { provider: string }, b: { provider: string }) => + a.provider.localeCompare(b.provider); + expect(curatedModels().sort(byProvider)).toEqual(offered.sort(byProvider)); +}); diff --git a/tests/fixtures/transcription/openrouter-catalog.json b/tests/fixtures/transcription/openrouter-catalog.json new file mode 100644 index 00000000..a4c73d51 --- /dev/null +++ b/tests/fixtures/transcription/openrouter-catalog.json @@ -0,0 +1,52 @@ +{ + "data": [ + { + "id": "deepgram/nova-4", + "name": "Deepgram: Nova 4", + "created": 1794500000, + "expiration_date": null + }, + { + "id": "x-ai/grok-stt-2.0", + "name": "SpaceXAI: Grok STT 2.0", + "created": 1794329167, + "expiration_date": "2026-11-01" + }, + { + "id": "elevenlabs/scribe-v2", + "name": "ElevenLabs: Scribe v2", + "created": 1791386190, + "expiration_date": null + }, + { + "id": "elevenlabs/scribe-v2-medical", + "name": "ElevenLabs: Scribe v2 Medical", + "created": 1791385619, + "expiration_date": null + }, + { + "id": "google/chirp-3", + "name": "Google: Chirp 3", + "created": 1777997783, + "expiration_date": null + }, + { + "id": "openai/gpt-transcribe", + "name": "OpenAI: GPT Transcribe", + "created": 1785973897, + "expiration_date": null + }, + { + "id": "x-ai/grok-stt-1.0", + "name": "SpaceXAI: Grok STT 1.0", + "created": 1784817014, + "expiration_date": "2026-11-15" + }, + { + "id": "deepgram/nova-3", + "name": "Deepgram: Nova 3", + "created": 1784075761, + "expiration_date": null + } + ] +} diff --git a/tests/tooling/modelReview.test.ts b/tests/tooling/modelReview.test.ts new file mode 100644 index 00000000..d9b4b8ad --- /dev/null +++ b/tests/tooling/modelReview.test.ts @@ -0,0 +1,89 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { curatedModels, renderReport, review } from "../../scripts/model-review.mjs"; + +const root = process.cwd(); +const script = join(root, "scripts/model-review.mjs"); +const catalog = JSON.parse( + readFileSync(join(root, "tests/fixtures/transcription/openrouter-catalog.json"), "utf8"), +).data; +const curated = [ + { provider: "openrouter", id: "x-ai/grok-stt-2.0" }, + { provider: "openrouter", id: "elevenlabs/scribe-v2" }, + { provider: "openrouter", id: "microsoft/mai-transcribe-2" }, + { provider: "deepgram", id: "nova-3" }, + { provider: "elevenlabs", id: "scribe_v3" }, + { provider: "openai", id: "gpt-transcribe" }, +]; +const result = review({ curated, catalog, today: "2026-11-14" }); + +describe("model review", () => { + it("reports a curated model past its end date", () => { + expect(result.ending).toEqual([ + { id: "x-ai/grok-stt-2.0", ends: "2026-11-01", past: true }, + ]); + expect( + renderReport(result, { catalogSize: catalog.length, today: "2026-11-14" }), + ).toContain("`x-ai/grok-stt-2.0` ended 2026-11-01."); + }); + + it("reports a curated model the catalog no longer lists", () => { + expect(result.gone).toEqual(["microsoft/mai-transcribe-2"]); + }); + + it("reports a model newer than a curated model of its family", () => { + expect(result.newer.map(({ model, than }) => [model.id, than])).toEqual([ + ["deepgram/nova-4", "nova-3"], + ]); + }); + + it("reports a model newer than the curated OpenRouter model even when a direct provider curates it", () => { + const { newer, candidates } = review({ + curated: [ + { provider: "openrouter", id: "elevenlabs/scribe-v2" }, + { provider: "elevenlabs", id: "scribe_v3" }, + ], + catalog: [{ id: "elevenlabs/scribe-v2" }, { id: "elevenlabs/scribe-v3" }], + today: "2026-11-14", + }); + expect(newer.map(({ model, than }) => [model.id, than])).toEqual([ + ["elevenlabs/scribe-v3", "elevenlabs/scribe-v2"], + ]); + expect(candidates).toEqual([]); + }); + + it("lists a model of no curated family once as a candidate", () => { + expect(result.candidates.map(({ id }) => id)).toEqual([ + "elevenlabs/scribe-v2-medical", + "google/chirp-3", + ]); + }); +}); + +describe("model review CLI", () => { + const directory = mkdtempSync(join(tmpdir(), "podnotes-model-review-")); + afterAll(() => rmSync(directory, { recursive: true, force: true })); + + function run(models: { id: string }[]) { + const file = join(directory, "catalog.json"); + writeFileSync(file, JSON.stringify({ data: models })); + return spawnSync(process.execPath, [script, "--catalog", file], { encoding: "utf8" }); + } + + it("passes when the catalog lists every curated OpenRouter model", () => { + const listed = curatedModels() + .filter(({ provider }) => provider === "openrouter") + .map(({ id }) => ({ id })); + expect(listed.length).toBeGreaterThan(0); + expect(run(listed).status).toBe(0); + }); + + it("fails when the catalog lists none of them", () => { + const result = run([]); + expect(result.status).toBe(1); + expect(result.stdout).toContain("is not in the catalog."); + }); +});