chore(tooling): review transcription models weekly

.github/workflows/model-review.yml

@@ -0,0 +1,53 @@
+name: Model review
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * 1"
+
+permissions:
+  contents: read
+
+jobs:
+  review:
+    name: Review models
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version-file: ".nvmrc"
+
+      - name: Review models
+        id: review
+        run: |
+          status=0
+          node scripts/model-review.mjs > "$RUNNER_TEMP/report.md" || status=$?
+          # A catalog that could not be read leaves no report, and fails the job here.
+          test -s "$RUNNER_TEMP/report.md"
+          cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY"
+          echo "status=$status" >> "$GITHUB_OUTPUT"
+
+      - name: Update the review issue
+        run: |
+          gh label create model-review --description "Weekly transcription model review" --force
+          number=$(gh issue list --label model-review --state open --limit 1 --json number --jq '.[0].number // empty')
+          if [ -n "$number" ]; then
+            gh issue edit "$number" --body-file "$RUNNER_TEMP/report.md"
+          elif grep -qE '^## (Curated models gone or ending|Newer models in a curated family)$' "$RUNNER_TEMP/report.md"; then
+            # Uncurated candidates alone do not reopen a review the maintainer closed.
+            gh issue create --title "Review transcription models" --label model-review --body-file "$RUNNER_TEMP/report.md"
+          fi
+        env:
+          GH_REPO: ${{ github.repository }}
+
+      - name: Fail when a curated model is gone or past its end date
+        if: steps.review.outputs.status != '0'
+        run: exit 1

AGENTS.md

@@ -48,6 +48,7 @@ root symlinks for local Obsidian loading.
 - `npm run test:tooling`: run the real-process Svelte checker failure tests and the build's syntax floor test, including rejection of tagged templates that disable Obsidian's eval cache. CI runs these separately on every PR.
 - `npm run build`: type-check and produce the production plugin bundle.
 - `npm run plant -- <test name> <file> <search> <replace> [<file> <search> <replace>...] [-- <command...>]`: plant a defect and pass only when the exactly named test fails on it and the command passes again once restored. Each search must match exactly once, and the plant must type-check. The files are restored on exit or interrupt. The command defaults to the unit suite filtered to the test name.
+- `npm run review:models`: compare the curated transcription models in `src/services/transcription/providers/` with OpenRouter's public catalog, report models that are gone, ending, newer in a curated family, or not curated, and exit 1 when a curated OpenRouter model is gone or past its end date. The `Model review` workflow runs it weekly and keeps one `model-review` issue current. It is not a required check.
 - `npm run docs:build`: build the Starlight documentation and check its internal links.
 
 `@typescript/native` aliases TypeScript 7. `typecheck` and `lint:tsconfig` call

package.json

@@ -17,6 +17,7 @@
 		"size:data": "node scripts/persistence-size.mjs",
 		"count:writes": "node scripts/write-counts.mjs",
 		"plant": "node scripts/plant.mjs",
+		"review:models": "node scripts/model-review.mjs",
 		"typecheck": "node node_modules/@typescript/native/bin/tsc --project tsconfig.json --noEmit && node node_modules/@typescript/native/bin/tsc --project tsconfig.test.json --noEmit",
 		"lint": "oxlint --deny-warnings src tests/e2e tests/tooling scripts .claude/skills/verify && svelte-policy",
 		"lint:tsconfig": "node node_modules/@typescript/native/bin/tsc --project tsconfig.json --noEmit --noCheck && node node_modules/@typescript/native/bin/tsc --project tsconfig.test.json --noEmit --noCheck",

scripts/model-review.mjs

@@ -0,0 +1,190 @@
+// Reports whether the transcription models PodNotes curates are still current. The curated
+// IDs are read from the provider files, and the models OpenRouter's public catalog lists
+// are compared with them. Exits 1 when a curated OpenRouter model is gone or past its end date.
+//
+//   npm run review:models [-- --catalog <saved catalog.json>]
+import { readdirSync, readFileSync } from "node:fs";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const PROVIDERS = fileURLToPath(
+	new URL("../src/services/transcription/providers/", import.meta.url),
+);
+const CATALOG_URL = "https://openrouter.ai/api/v1/models?output_modalities=transcription";
+
+/**
+ * @typedef {{ provider: string, id: string }} Curated
+ * @typedef {{ id: string, name?: string, created?: number, expiration_date?: string | null }} CatalogModel
+ */
+
+/**
+ * Every model ID in the provider files, with the provider whose file lists it.
+ * @param {string} [directory]
+ * @returns {Curated[]}
+ */
+export function curatedModels(directory = PROVIDERS) {
+	return readdirSync(directory)
+		.filter((file) => file.endsWith(".ts") && !file.endsWith(".test.ts"))
+		.sort()
+		.flatMap((file) => {
+			const source = readFileSync(new URL(file, pathToFileURL(`${directory}/`)), "utf8");
+			return [...source.matchAll(/^\s+id: "([^"]+)",$/gm)].map(([, id]) => ({
+				provider: file.slice(0, -".ts".length),
+				id,
+			}));
+		});
+}
+
+/**
+ * A model's family is its vendor and the words of its name that are not a version, so
+ * `elevenlabs/scribe-v2` and the direct `scribe_v3` belong to one family.
+ * @param {string} provider the vendor of an ID that names none
+ * @param {string} id
+ */
+function parse(provider, id) {
+	const slash = id.indexOf("/");
+	const vendor = slash < 0 ? provider : id.slice(0, slash);
+	const words = id
+		.slice(slash + 1)
+		.toLowerCase()
+		.split(/[-_]/);
+	const isVersion = (/** @type {string} */ word) => /^v?\d+(\.\d+)*$/.test(word);
+	return {
+		family: [vendor, ...words.filter((word) => !isVersion(word))].join("-"),
+		version: words
+			.filter(isVersion)
+			.flatMap((word) => word.replace(/^v/, "").split(".").map(Number)),
+	};
+}
+
+/**
+ * @param {number[]} a
+ * @param {number[]} b
+ */
+function compare(a, b) {
+	for (let i = 0; i < Math.max(a.length, b.length); i++) {
+		const difference = (a[i] ?? 0) - (b[i] ?? 0);
+		if (difference !== 0) return difference;
+	}
+	return 0;
+}
+
+/**
+ * @param {{ curated: Curated[], catalog: CatalogModel[], today: string }} input
+ * @returns {{
+ *   gone: string[],
+ *   ending: { id: string, ends: string, past: boolean }[],
+ *   newer: { model: CatalogModel, than: string }[],
+ *   candidates: CatalogModel[],
+ * }}
+ */
+export function review({ curated, catalog, today }) {
+	const listed = new Map(catalog.map((model) => [model.id, model]));
+	const gone = [];
+	const ending = [];
+	for (const { provider, id } of curated) {
+		if (provider !== "openrouter") continue;
+		const model = listed.get(id);
+		if (!model) gone.push(id);
+		else if (model.expiration_date) {
+			const ends = model.expiration_date.slice(0, 10);
+			ending.push({ id, ends, past: ends <= today });
+		}
+	}
+
+	/** @type {Map<string, { provider: string, id: string, version: number[] }[]>} */
+	const families = new Map();
+	for (const { provider, id } of curated) {
+		const { family, version } = parse(provider, id);
+		families.set(family, [...(families.get(family) ?? []), { provider, id, version }]);
+	}
+
+	/**
+	 * The newest curated model a catalog model is measured against: the OpenRouter one when
+	 * the family has any, since a direct provider's version says nothing about it.
+	 * @param {{ provider: string, id: string, version: number[] }[]} own
+	 */
+	const latest = (own) => {
+		const viaOpenRouter = own.filter(({ provider }) => provider === "openrouter");
+		return (viaOpenRouter.length > 0 ? viaOpenRouter : own).reduce((a, b) =>
+			compare(b.version, a.version) > 0 ? b : a,
+		);
+	};
+
+	const newer = [];
+	const candidates = [];
+	const fresh = catalog
+		.filter((model) => !model.expiration_date)
+		.sort((a, b) => (b.created ?? 0) - (a.created ?? 0));
+	for (const model of fresh) {
+		const { family, version } = parse("", model.id);
+		const own = families.get(family);
+		if (!own) candidates.push(model);
+		else if (compare(version, latest(own).version) > 0) {
+			newer.push({ model, than: latest(own).id });
+		} else if (!own.some((curatedModel) => compare(version, curatedModel.version) === 0)) {
+			candidates.push(model);
+		}
+	}
+	return { gone, ending, newer, candidates };
+}
+
+/**
+ * @param {ReturnType<typeof review>} result
+ * @param {{ catalogSize: number, today: string }} context
+ */
+export function renderReport({ gone, ending, newer, candidates }, { catalogSize, today }) {
+	/** @type {string[]} */
+	const lines = [
+		"# Transcription model review",
+		"",
+		`Checked ${today} against the OpenRouter catalog (${catalogSize} models).`,
+	];
+	/**
+	 * @param {string} heading
+	 * @param {string[]} items
+	 */
+	const section = (heading, items) => {
+		if (items.length > 0) lines.push("", `## ${heading}`, "", ...items.map((i) => `- ${i}`));
+	};
+	section("Curated models gone or ending", [
+		...gone.map((id) => `\`${id}\` is not in the catalog.`),
+		...ending.map(({ id, ends, past }) => `\`${id}\` ${past ? "ended" : "ends"} ${ends}.`),
+	]);
+	section(
+		"Newer models in a curated family",
+		newer.map(({ model, than }) => `\`${model.id}\` is newer than \`${than}\`.`),
+	);
+	section(
+		"Candidates, not curated",
+		candidates.map(({ id, name }) => `\`${id}\`${name ? ` (${name})` : ""}`),
+	);
+	if (gone.length + ending.length + newer.length + candidates.length === 0) {
+		lines.push("", "Nothing to review.");
+	}
+	return `${lines.join("\n")}\n`;
+}
+
+/** @param {string[]} args */
+async function main(args) {
+	const file = args[args.indexOf("--catalog") + 1];
+	const body =
+		args.includes("--catalog") && file
+			? JSON.parse(readFileSync(file, "utf8"))
+			: await fetch(CATALOG_URL, { signal: AbortSignal.timeout(30_000) }).then((response) => {
+					if (!response.ok) throw new Error(`The catalog answered ${response.status}.`);
+					return response.json();
+				});
+	/** @type {CatalogModel[]} */
+	const catalog = body.data;
+	const today = new Date().toISOString().slice(0, 10);
+	const result = review({ curated: curatedModels(), catalog, today });
+	process.stdout.write(renderReport(result, { catalogSize: catalog.length, today }));
+	if (result.gone.length > 0 || result.ending.some(({ past }) => past)) process.exitCode = 1;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	main(process.argv.slice(2)).catch((error) => {
+		console.error(`Model review failed: ${error instanceof Error ? error.message : error}`);
+		process.exitCode = 1;
+	});
+}

scripts/model-review.test.ts

@@ -0,0 +1,12 @@
+import { expect, it } from "vitest";
+import { PROVIDERS } from "src/services/transcription/registry";
+import { curatedModels } from "./model-review.mjs";
+
+it("reads the model IDs the providers offer", () => {
+	const offered = Object.entries(PROVIDERS).flatMap(([provider, { models }]) =>
+		models.map(({ id }) => ({ provider, id })),
+	);
+	const byProvider = (a: { provider: string }, b: { provider: string }) =>
+		a.provider.localeCompare(b.provider);
+	expect(curatedModels().sort(byProvider)).toEqual(offered.sort(byProvider));
+});

tests/fixtures/transcription/openrouter-catalog.json

@@ -0,0 +1,52 @@
+{
+	"data": [
+		{
+			"id": "deepgram/nova-4",
+			"name": "Deepgram: Nova 4",
+			"created": 1794500000,
+			"expiration_date": null
+		},
+		{
+			"id": "x-ai/grok-stt-2.0",
+			"name": "SpaceXAI: Grok STT 2.0",
+			"created": 1794329167,
+			"expiration_date": "2026-11-01"
+		},
+		{
+			"id": "elevenlabs/scribe-v2",
+			"name": "ElevenLabs: Scribe v2",
+			"created": 1791386190,
+			"expiration_date": null
+		},
+		{
+			"id": "elevenlabs/scribe-v2-medical",
+			"name": "ElevenLabs: Scribe v2 Medical",
+			"created": 1791385619,
+			"expiration_date": null
+		},
+		{
+			"id": "google/chirp-3",
+			"name": "Google: Chirp 3",
+			"created": 1777997783,
+			"expiration_date": null
+		},
+		{
+			"id": "openai/gpt-transcribe",
+			"name": "OpenAI: GPT Transcribe",
+			"created": 1785973897,
+			"expiration_date": null
+		},
+		{
+			"id": "x-ai/grok-stt-1.0",
+			"name": "SpaceXAI: Grok STT 1.0",
+			"created": 1784817014,
+			"expiration_date": "2026-11-15"
+		},
+		{
+			"id": "deepgram/nova-3",
+			"name": "Deepgram: Nova 3",
+			"created": 1784075761,
+			"expiration_date": null
+		}
+	]
+}