From 4df3f8e5c038eefb71a01e991cd9777bd378a83b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 14:32:06 +0000 Subject: [PATCH] Every prose word against a dictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/spelling.test.mjs answers WHICH spelling of a word - it names British forms from a list and rewrites them. What it cannot do is notice a word that is neither spelling of anything. "thant", and two days later "havte", "fullfll" and "requoirememnts", went in through the browser editor, past all fourteen gates, through the deploy and out to the published site. The second set stood in the FIRST SENTENCE of a page, which is where scripts/lib/pages.mjs takes the page description from - so it read wrong in the search index, in llms.txt and under the card as well. So: every prose word is looked up in dictionary-en (the Hunspell en_US), and what a dictionary cannot know about this project is committed beside it in scripts/lib/vocabulary.txt - 215 words, grouped and commented, built from the corpus in one pass. The prose comes from scripts/lib/prose.mjs, the same reader the spelling test uses, so an ABAP keyword, a CSS property or a path is never a word here. The tokenizer is most of the work, and the manual proved each rule: - \p{L}, not [A-Za-z]: the latter starts a new token at the "ller" of "Müller" and reports it. - emphasis inside a word is joined first: `**f**rontend` was read as "rontend". - a token with a digit or an underscore is an identifier (abap2UI5, z2ui5_if_app), so is an acronym (ICF, APIs) and so is camelCase (liveChange) - none of them is English to be spelled. Without this the first probe reported "UI" 1059 times, from abap2UI5 split at the digit. - a hyphenated compound is judged part by part, minus the fragments a hyphen leaves behind ("un-build" is not a word "un"). Each rule cut the unknown words: 821 distinct, then 266, then 249 - and those 249 are all real, which is why the list is exactly that long and no page changed. Fifteenth gate: named in the check script, both workflows and the four documents that count them. --unused names entries no page uses any more. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0182AiPodwfGRKPNZG9D5epe --- .github/workflows/check.yml | 12 ++ .github/workflows/deploy.yml | 3 + AGENTS.md | 13 +- CLAUDE.md | 2 +- CONTRIBUTING.md | 2 +- README.md | 8 +- package-lock.json | 47 +++++++ package.json | 5 +- scripts/check-vocabulary.mjs | 77 +++++++++++ scripts/lib/vocabulary.mjs | 86 +++++++++++++ scripts/lib/vocabulary.txt | 239 +++++++++++++++++++++++++++++++++++ test/gates.test.mjs | 10 +- test/vocabulary.test.mjs | 73 +++++++++++ 13 files changed, 560 insertions(+), 17 deletions(-) create mode 100644 scripts/check-vocabulary.mjs create mode 100644 scripts/lib/vocabulary.mjs create mode 100644 scripts/lib/vocabulary.txt create mode 100644 test/vocabulary.test.mjs diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 947b4377..9c35e8cc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -196,6 +196,18 @@ jobs: if: ${{ !cancelled() }} run: npm run check:line-length + # every prose word against a dictionary, and then against the words this + # project uses that a dictionary does not carry. test/spelling.test.mjs + # answers WHICH spelling of a word (it names British forms from a list); + # it cannot see a word that is neither spelling of anything. "thant", + # "havte", "fullfll" and "requoirememnts" went through the browser + # editor, through this workflow and out to the published site in two + # days, one of them in the first sentence of a page - which is where the + # search index, llms.txt and the card subtitle take their description. + - name: vocabulary + if: ${{ !cancelled() }} + run: npm run check:vocabulary + # every complete app class either carries a Run button or a marker on its # page saying why it cannot run in the playground. The rules that offer # the button fail towards NOT offering one, so without this an example diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6d61d22c..c2df3920 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -168,6 +168,9 @@ jobs: - name: line length if: ${{ !cancelled() }} run: npm run check:line-length + - name: vocabulary + if: ${{ !cancelled() }} + run: npm run check:vocabulary - name: Run-button coverage if: ${{ !cancelled() }} run: npm run check:playground diff --git a/AGENTS.md b/AGENTS.md index 35adce5d..730a67a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,7 @@ person reads the page. Do not put "as an AI, …" prose back into `docs/`. | `docs/.vitepress/playground.mjs` | Decides which fenced ABAP example gets a **Run** button, and wraps the fence; `theme/playground.js` is the browser half | | `scripts/list-runnable.mjs` | The measurement's worklist: every fenced example that carries a Run button, out of the same `playground.mjs` that decides the button. `--json` adds each example's ABAP verbatim - what the button sends - so the measurement below can be driven rather than clicked | | `scripts/check-playground.mjs` | The Run-button bookkeeping: every complete app class either gets a button from `playground.mjs` or carries a `` marker above its fence saying why it cannot run; a stale marker fails as loudly as a missing one. `--list` prints the deliberate exclusions with both reasons | +| `scripts/check-vocabulary.mjs` | Every prose word against a dictionary and against `scripts/lib/vocabulary.txt` (grouped and commented, one word per line). `scripts/lib/vocabulary.mjs` is the tokenizer - what is a word here and what is an identifier - and is pinned by `test/vocabulary.test.mjs`, whose first case is the four typos that reached the published site | | `scripts/check-line-length.mjs` | The wrapped-page rule and `--fix` (`npm run fix:line-length`), which rewraps a drifted paragraph. `scripts/lib/line-length.mjs` is what a prose line IS here and what a rewrap may touch; pinned by `test/line-length.test.mjs`, which holds the manual to the rule as its last case | | `scripts/check-conventions.mjs` | The two house conventions the sample corpora gate and this one did not: the view-chain layout in every fenced chain (the linter's `chain-house-layout`, which is opt-in — `check-examples.mjs` writes its config without a `rules` block, so the rule was never emitted), and the three class section blocks in every fenced app class. `--fix` (`npm run fmt:chains`) reformats a drifted chain; the sections are a judgement and stay by hand | | `scripts/lib/catalogue.mjs` | Parses and counts a sample catalogue, for `link-samples.mjs` and for the figures `generate-llms.mjs` writes into `llms.txt` — from a sibling checkout when one is here, else from the `catalogue.json` each sample repository commits at its root; pinned by `test/catalogue.test.mjs`, because it has stopped matching twice and both times answered wrongly instead of failing | @@ -57,8 +58,8 @@ person reads the page. Do not put "as an AI, …" prose back into `docs/`. npm run check # test + check:version + docs:build + check:cross-site + check:design + check:images + check:examples + check:conventions + check:playground + check:api-names + check:api-reference + check:samples ``` -A documentation repository has no compiler for its prose, but fourteen things -in it are decidable, and all fourteen are decided before a merge: +A documentation repository has no compiler for its prose, but fifteen things +in it are decidable, and all fifteen are decided before a merge: | | | |---|---| @@ -70,6 +71,7 @@ in it are decidable, and all fourteen are decided before a merge: | `check:api-reference` | the committed client API reference — the generated block in `resources/api.md` and `docs/public/api/client-api.json` — regenerated from `z2ui5_if_client` on `main` and compared byte for byte. Goes stale whenever the interface changes over there and the committed reference still describes the shape before it. `npm run generate:api` rewrites both | | `check:conventions` | the fenced ABAP against the house style the reader meets next: the view-chain layout, and the three section blocks of an app class. `check:examples` asks whether an example compiles and names real API — both questions about the framework; neither can see that a snippet is written in a different style from every sample. Measured against [samples-controls](https://github.com/abap2UI5/samples-controls) (637 classes, gated, at zero): five chains here showed the reader a different tree than the one that renders, and 57 of 86 app classes carried neither `PROTECTED SECTION.` nor `PRIVATE SECTION.`. What this gate deliberately does NOT take over is the blank-line and `t_arg` continuation rules — those are pattern-lint *warnings* over there and that corpus carries 382 of them | | `check:line-length` | **a page that is wrapped has to stay wrapped.** The manual is not written to one column and should not be: 39 pages are wrapped, 59 are one line per paragraph, 64 are in between, so a site-wide column would be a reformat of two thirds of the pages rather than a gate. The drift is the decidable part - a page whose prose already sits inside 80 characters may not acquire a line outside it. That is what an editor does to a paragraph it rewrites: it hands it back as ONE long line, invisible in the editor, and from then on every diff of that paragraph is one changed line instead of three, so each later correction reads as a rewrite. It happened twice in two days on the same page. Held: 66 pages; the other 97 are left alone and start being held the day somebody wraps them. Not prose, and the reason each one is not: fenced code, frontmatter, tables (they wrap at the cell), HTML, headings and image alt text (one line by construction), the caption under an image when it repeats that alt text word for word (the two are meant to be comparable at a glance), a line that is only long because of one unbreakable URL, and the generated `samples:` and `api:` blocks (wrapping one by hand is a change the next regeneration undoes). `npm run fix:line-length` rewraps a drifted paragraph - whitespace only, and verified as such: the words are identical and the rendered article is identical, which is the check that caught the two bugs in the rewrapper (it tore the punctuation off a code span, and it swallowed a `:::` container into the paragraph) | +| `check:vocabulary` | every prose word of the manual, against a dictionary (`dictionary-en`, the Hunspell en_US) and then against `scripts/lib/vocabulary.txt`, the 215 words this project uses that a dictionary does not carry. **`test/spelling.test.mjs` answers which SPELLING of a word; this one answers whether it is a word at all**, and nothing did: that test names British forms from a list, so a word that is neither spelling reads as prose it has no opinion about. `thant`, and then `havte`, `fullfll` and `requoirememnts`, went in through the browser editor, past all fourteen other gates and out to the published site in two days - and the second set stood in the FIRST SENTENCE of a page, which is where `scripts/lib/pages.mjs` takes the description from, so it read wrong in the search index, in `llms.txt` and under the card as well. The prose is `scripts/lib/prose.mjs`'s, the same reader the spelling test uses, so an ABAP keyword, a CSS property or a path is never a word here. Not a word either, each for a reason: a token carrying a digit or an underscore (`abap2UI5`, `z2ui5_if_app`), an acronym (`ICF`, `APIs`), camelCase (`liveChange`) - all identifiers, not English; a hyphenated compound is judged part by part, and emphasis inside a word (`**f**rontend`) is joined back up first. A word goes on the list when it is right and the dictionary simply does not have it; a word you are unsure about does not, because a list that absorbs doubt is how a gate like this stops catching anything. `--unused` names entries no page uses any more | | `check:playground` | every complete app class on the site either carries a **Run** button or a marker on its page saying why it cannot run. The rules that offer the button fail towards *not* offering one, so without this an example nobody ever measured is indistinguishable from an example that can never run — which is exactly how the coverage ledger below went stale. What stays undecidable by CI — does a *buttoned* example actually start — is the measurement the Run-button section describes | | `check:cross-site` | every link that leaves this deployment for a neighbouring one on the same origin — the playground, the catalogue, the linter's rule pages — carries a `target`. Without it VitePress's router treats the link as a route of THIS site, finds no page behind `/playground/` and renders the 404 *at that URL*, which reads as the other site being broken. Every way out of the manual was in that state at once: both bar items, the Linter rules row in the menu and the Run bar's link. Judges the built HTML, so it runs straight after `docs:build`. What it cannot see is the Run bar's link — built in a browser, in no built page — which is why `test/cross-site.test.mjs` pins that one as source | | `check:design` | the values the four bars are made of — the seven palette colours, the two type stacks, the two radii — against the copy [abap2UI5/playground](https://github.com/abap2UI5/playground) keeps, in **both** schemes. They are copied by hand on purpose (a stylesheet fetched across two deployments is a request in front of the first paint), and until this gate nothing compared the copies: they agreed because whoever touched one remembered the other. One had already drifted — two font stacks leading with different families, which is the same face on macOS and Windows and two different ones on Linux, so the same four words in the same bar measured 59/122/78/97px here and 65/141/87/110 there. It compares the EFFECTIVE value (a property the dark block does not redeclare keeps its light one), because the two sides switch schemes differently: `.dark` here, `[data-theme]` over there. @@ -78,18 +80,19 @@ in it are decidable, and all fourteen are decided before a merge: | `check:images` | every image under `docs/public`, against the three things a page can afford and the one it cannot: a screenshot is WebP (the PNG captures were 200 to 335 kB each, 2.9 MB across the manual, on pages of 20 kB of text; the same captures as WebP are a fifth of that), a deliverable is one of the PNGs the logo page hands out, nothing is over its budget, and the build can measure every one - an image it cannot size gets no width and height and moves the page when it lands | | `check:samples` | the **Working Samples** blocks and the source links a page writes by hand, against [abap2UI5/samples](https://github.com/abap2UI5/samples), [samples-controls](https://github.com/abap2UI5/samples-controls) and [samples-stack](https://github.com/abap2UI5/samples-stack) — a page may declare a class of any of the three | -**All five walking gates carry a floor.** A gate that checked nothing reports +**All six walking gates carry a floor.** A gate that checked nothing reports the same shape as a gate that found nothing wrong — which is precisely how `check:examples` passed for years on an abaplint config with no rules in it. So `check:examples`, `check:conventions` and `check:playground` each exit 1 when their walk finds no example at all, and say which of the fence language, the page layout or the builder name is the likely cause. `check:line-length` exits 1 when no page of the site counts as wrapped, which would mean its fence handling -or its glob stopped matching rather than that the manual went loose. `check:cross-site` carries +or its glob stopped matching rather than that the manual went loose, and +`check:vocabulary` when it walked no page or read an empty word list. `check:cross-site` carries the same floor twice over: no HTML in `dist` at all, and no cross-site link on a site whose bar carries three of them on every page. -The fourteen are written out in **three** places, and all three have to name the +The fifteen are written out in **three** places, and all three have to name the same set: `package.json`'s `check` script, `.github/workflows/check.yml` for a pull request, and `.github/workflows/deploy.yml` before the site is published. `check.yml` runs them in the script's order; `deploy.yml` cannot, because it diff --git a/CLAUDE.md b/CLAUDE.md index a6f86925..0d4f4b0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # CLAUDE.md All project guidance lives in **[AGENTS.md](AGENTS.md)** — the single source of -truth for this repository (how the site is built, the fourteen gates, the playground rule engine, and what may be written where). +truth for this repository (how the site is built, the fifteen gates, the playground rule engine, and what may be written where). Read `AGENTS.md` before making any change. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d7210b8..84aef0a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,4 +21,4 @@ Nothing here is a source of truth about the framework: the API reference, the sample catalogue and the corpus figures are generated or checked against `abap2UI5`, `abap2UI5/samples` and its siblings. If a fact is wrong, it is usually wrong there first. [AGENTS.md](AGENTS.md) is the full contract — how -the site is built, the fourteen gates, and what may be written where. +the site is built, the fifteen gates, and what may be written where. diff --git a/README.md b/README.md index c6ae0041..9b34a9cf 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ Every contribution makes the documentation better for the community! ```sh npm ci npm run docs:dev # the site, with hot reload -npm run check # what CI runs, all fourteen steps +npm run check # what CI runs, all fifteen steps ``` ### What CI checks -A documentation repository has no compiler for its prose, but fourteen things in -it are decidable, and `npm run check` decides all fourteen before a merge — the +A documentation repository has no compiler for its prose, but fifteen things in +it are decidable, and `npm run check` decides all fifteen before a merge — the prose builds (`docs:build`), the four bars are still made of the same palette, type and radii as the playground's (`check:design`), every link into a neighbouring site on this origin @@ -45,7 +45,7 @@ passed. Several of these go stale without anybody touching this repository (a release is published elsewhere, a sample class is renamed elsewhere), which is why the deploy re-runs them rather than trusting the merge. -**[AGENTS.md](AGENTS.md) describes each of the fourteen**, what a failure means and +**[AGENTS.md](AGENTS.md) describes each of the fifteen**, what a failure means and which of them need a sibling checkout to say anything at all — read it before changing anything beyond prose. diff --git a/package-lock.json b/package-lock.json index 16200598..d8779189 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "devDependencies": { "@abap2ui5/linter": "^0.6.1", "@abaplint/cli": "^2.120.52", + "dictionary-en": "^4.0.0", "esbuild": "^0.28.2", + "nspell": "^2.1.5", "vitepress": "^1.6.4" }, "engines": { @@ -1783,6 +1785,17 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/dictionary-en": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/dictionary-en/-/dictionary-en-4.0.0.tgz", + "integrity": "sha512-3NHnE1uq33ZE/CIwaZ6gqxa4BnglHnxeAcTM0GJ7cmtRGcvX9InMK/IqLtYcUMFCUpUNgybH+DzkqdgAo3F1zg==", + "dev": true, + "license": "(MIT AND BSD)", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/emoji-regex-xs": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/emoji-regex-xs/-/emoji-regex-xs-1.0.0.tgz", @@ -1933,6 +1946,30 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/is-what": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/is-what/-/is-what-5.5.0.tgz", @@ -2112,6 +2149,16 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/nspell": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/nspell/-/nspell-2.1.5.tgz", + "integrity": "sha512-PSStyugKMiD9mHmqI/CR5xXrSIGejUXPlo88FBRq5Og1kO5QwQ5Ilu8D8O5I/SHpoS+mibpw6uKA8rd3vXd2Sg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-buffer": "^2.0.0" + } + }, "node_modules/oniguruma-to-es": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-3.1.1.tgz", diff --git a/package.json b/package.json index 8fed1c6d..55e11c64 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "check:examples": "node scripts/check-examples.mjs", "check:conventions": "node scripts/check-conventions.mjs", "check:line-length": "node scripts/check-line-length.mjs", + "check:vocabulary": "node scripts/check-vocabulary.mjs", "fix:line-length": "node scripts/check-line-length.mjs --fix", "fmt:chains": "node scripts/check-conventions.mjs --fix", "check:playground": "node scripts/check-playground.mjs", @@ -31,7 +32,7 @@ "link:samples": "node scripts/link-samples.mjs", "check:samples": "node scripts/link-samples.mjs --check", "test": "node --test test/*.test.mjs", - "check": "npm run test && npm run check:version && npm run build && npm run docs:build && npm run check:cross-site && npm run check:design && npm run check:images && npm run check:examples && npm run check:conventions && npm run check:line-length && npm run check:playground && npm run check:api-names && npm run check:api-reference && npm run check:samples", + "check": "npm run test && npm run check:version && npm run build && npm run docs:build && npm run check:cross-site && npm run check:design && npm run check:images && npm run check:examples && npm run check:conventions && npm run check:line-length && npm run check:vocabulary && npm run check:playground && npm run check:api-names && npm run check:api-reference && npm run check:samples", "llms": "node scripts/generate-llms.mjs", "search": "node scripts/generate-search.mjs", "check:version": "node scripts/check-version.mjs", @@ -42,7 +43,9 @@ "devDependencies": { "@abap2ui5/linter": "^0.6.1", "@abaplint/cli": "^2.120.52", + "dictionary-en": "^4.0.0", "esbuild": "^0.28.2", + "nspell": "^2.1.5", "vitepress": "^1.6.4" }, "engines": { diff --git a/scripts/check-vocabulary.mjs b/scripts/check-vocabulary.mjs new file mode 100644 index 00000000..91f82e44 --- /dev/null +++ b/scripts/check-vocabulary.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +// Every prose word of the manual, against a dictionary and against the words +// this project uses that a dictionary does not carry. See +// scripts/lib/vocabulary.mjs for what counts as a word here. +// +// node scripts/check-vocabulary.mjs fail on a word nothing knows +// node scripts/check-vocabulary.mjs --unused name entries no page uses + +import { readFile } from 'node:fs/promises'; +import { manualPages } from './lib/prose.mjs'; +import { findUnknown, projectWords, WORDS_FILE } from './lib/vocabulary.mjs'; + +const unused = process.argv.includes('--unused'); + +const words = projectWords(); +const pages = manualPages('docs'); +const seen = new Set(); +const findings = []; +let checked = 0; + +for (const file of pages) { + const source = await readFile(file, 'utf8'); + for (const { line, word } of findUnknown(source, { words })) { + findings.push({ file, line, word }); + } + // what the list is actually earning + for (const { word } of findUnknown(source, { words: [] })) seen.add(word.toLowerCase()); + checked++; +} + +// The floor. A walk that matched no page would report the same shape as a walk +// that found nothing wrong. +if (checked === 0 || words.length === 0) { + console.error( + `check-vocabulary: walked ${checked} page(s) against ${words.length} project word(s).\n` + + ` That is not a corpus this gate can judge - the likely cause is the page\n` + + ` walk in scripts/lib/prose.mjs or an empty ${WORDS_FILE}.`, + ); + process.exit(1); +} + +if (unused) { + const stale = words.filter((w) => !seen.has(w.toLowerCase())); + console.log( + stale.length + ? `${stale.length} entr(ies) in vocabulary.txt that no page uses any more:\n ${stale.join('\n ')}` + : 'every entry in vocabulary.txt is earning its line.', + ); + process.exit(0); +} + +if (findings.length) { + const byWord = new Map(); + for (const f of findings) { + if (!byWord.has(f.word)) byWord.set(f.word, []); + byWord.get(f.word).push(`${f.file}:${f.line}`); + } + console.error(`check-vocabulary: ${byWord.size} word(s) that neither the dictionary nor this project knows:\n`); + for (const [word, places] of byWord) { + console.error(` ${word}`); + for (const place of places.slice(0, 4)) console.error(` ${place}`); + if (places.length > 4) console.error(` …and ${places.length - 4} more`); + } + console.error( + `\nIf it is a typo, fix the page. If it is a word this project uses and a\n` + + `dictionary would not carry - project vocabulary, one of SAP's own words, the\n` + + `name of a tool or a person - add it to scripts/lib/vocabulary.txt, in the group\n` + + `it belongs to. A word you are unsure about does not go on the list: the list\n` + + `absorbing doubt is how a gate like this stops catching anything.`, + ); + process.exit(1); +} + +console.log( + `check-vocabulary: every prose word on ${checked} page(s) is one the dictionary or\n` + + ` scripts/lib/vocabulary.txt (${words.length} entries) knows.`, +); diff --git a/scripts/lib/vocabulary.mjs b/scripts/lib/vocabulary.mjs new file mode 100644 index 00000000..94c9db4f --- /dev/null +++ b/scripts/lib/vocabulary.mjs @@ -0,0 +1,86 @@ +/* Is this a WORD - as opposed to which spelling of it. + * + * test/spelling.test.mjs answers the second question: it names British forms + * from a list and rewrites them. What it cannot do is notice a word that is + * neither spelling of anything. "thant", "havte", "fullfll" and + * "requoirememnts" all went in through the browser editor, through CI and out + * to the published site in two days, and one of them stood in the first + * sentence of a page - which is also what the search index, llms.txt and the + * card subtitle take their description from, so it read wrong in four places. + * + * So: every prose word is looked up in a dictionary (dictionary-en, the + * Hunspell en_US), and what a dictionary cannot know about this project is + * committed beside it in vocabulary.txt. + * + * The prose comes from scripts/lib/prose.mjs, which already blanks fenced + * code, inline code, link targets, HTML, URLs, the generated blocks and the + * configuration half of the frontmatter - so an ABAP keyword, a CSS property + * or a path is never a word here. + */ + +import { readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import nspell from 'nspell'; +import en from 'dictionary-en'; +import { proseLines } from './prose.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +export const WORDS_FILE = join(HERE, 'vocabulary.txt'); + +/** The project's own words, one per line; `#` starts a comment. */ +export function projectWords(file = WORDS_FILE) { + return readFileSync(file, 'utf8') + .split('\n') + .map((l) => l.replace(/#.*$/, '').trim()) + .filter(Boolean); +} + +// A token is a word and everything that may sit INSIDE one. The Unicode +// classes matter: [A-Za-z] would start a new token at the "ller" of "Müller" +// and report it. +const TOKEN = /[\p{L}][\p{L}\p{N}_'’-]*/gu; + +// Fragments a hyphen leaves behind that are not words on their own. +const AFFIX = new Set([ + 'un', 're', 'pre', 'non', 'co', 'de', 'mid', 'sub', 'cross', 'multi', 'semi', + 'anti', 'self', 'off', 'on', 'in', 'out', 'over', 'under', 'post', 'pro', + 'well', 'half', 'side', 'by', 'to', 'up', 'so', 'no', 'per', 'ever', 'like', + 'free', 'wide', 'less', 'ish', 'est', +]); + +/** + * Every word of `markdown` that no dictionary and no list of ours knows, as + * {line, word}. Ordered by where it stands. + */ +export function findUnknown(markdown, { words = projectWords() } = {}) { + const spell = nspell(en); + const known = new Set(words.map((w) => w.toLowerCase())); + const inDictionary = (w) => spell.correct(w) || spell.correct(w.toLowerCase()); + const ok = (w) => known.has(w.toLowerCase()) || inDictionary(w); + + const found = []; + proseLines(markdown).forEach((line, i) => { + // `**f**rontend` - emphasis INSIDE a word. Markdown joins it back up; a + // tokenizer that does not would report "rontend". + for (const m of line.replace(/\*+/g, '').matchAll(TOKEN)) { + const token = m[0].replace(/[-'’]+$/, ''); + // An identifier, not a word: it carries a digit or an underscore + // (`z2ui5_cl_app`, `abap2UI5`), is an acronym (`ICF`, `APIs`), or is + // camelCase (`liveChange`). None of them is English to be spelled. + if (/\p{N}/u.test(token) || token.includes('_')) continue; + if (/^\p{Lu}{2,}s?$/u.test(token)) continue; + if (/\p{Ll}\p{Lu}/u.test(token)) continue; + if (ok(token)) continue; + + // A hyphenated compound the dictionary does not carry whole is judged + // part by part: "design-time" is two words it knows. + for (const part of token.split('-')) { + const word = part.replace(/[’']s$/, ''); + if (word.length < 3 || AFFIX.has(word.toLowerCase()) || ok(word)) continue; + found.push({ line: i + 1, word }); + } + } + }); + return found; +} diff --git a/scripts/lib/vocabulary.txt b/scripts/lib/vocabulary.txt new file mode 100644 index 00000000..2e4ee71f --- /dev/null +++ b/scripts/lib/vocabulary.txt @@ -0,0 +1,239 @@ +# Words this manual uses that a dictionary does not carry. +# +# scripts/check-vocabulary.mjs looks every prose word up in dictionary-en +# (Hunspell en_US) and then in here. A word belongs here when it is right and +# the dictionary simply does not have it - project vocabulary, SAP's own +# words, the names of tools and people. A word that is merely long, or that +# you are unsure about, does not: the gate exists because four typos reached +# the published site in two days, and a list that absorbs doubt stops +# catching them. +# +# One word per line, lower case, grouped. Case does not matter when it is +# read - Frontend and frontend are one entry. + +# The project, and the things it is made of +abap +abaplint +abapmerge +ajson +classrun +dotabap +downport +downportable +downported +downporting +srtti +transpile +transpiled +transpiler +transpiles + +# SAP +alv +btp +cds +dynpro +eml +fiori +floorplans +flp +hana +kerberos +odata +persistency +samlassertion +sapconnect +searchhelp +steampunkification +subaccount +worklists +xsdbool + +# The web, and the tools around it +api +backend +backends +bundler +cdnjs +css +devtools +emmet +eslint +etag +favicon +frontend +frontends +gzip +hotwire +htmx +http +ico +iframe +json +livewire +mcp +npm +npx +png +preprocessor +reconciler +renderer +roundtrip +roundtrips +sdk +stylesheet +theia +unpoly +url +viewport +viewports +vscode +vscodium +webapp + +# Words the dictionary does not carry but a developer writes +activations +addon +allowlist +arg +async +auth +automagic +backtick +barcode +barcodes +bookmarkable +booleans +centric +changelog +clickjacking +cloudification +codebase +config +configs +debounce +denylist +deserialization +dev +dialogs +diffable +dropdown +editability +enum +filesystem +findable +formatter +formatters +fullscreen +geolocation +github +gitignored +globals +glyphs +greppable +hardcoded +init +inlined +instantiation +integrations +invariants +keypress +lifecycle +lifecycles +lightbulb +linter +linters +mistyped +mjs +multiselect +mutator +namespace +namespaces +natively +nav +numerics +ons +paddings +personalizable +personalization +positionally +preload +preloaded +preloading +pressable +programmatically +pushdown +quickstart +readme +relinking +renameable +repo +resync +resynced +rollout +runnable +sandboxing +sanitization +scrollable +selectable +serializer +severities +sortable +spacings +splitter +stateful +statefulness +stringified +subclasses +sublicense +subpath +superset +systemless +teardown +templated +templating +theming +toolchain +tooltip +truthy +unclosed +unlinked +unregister +unstyled +untrusted +vendored +walkthrough +whitespace + +# Names +decabase +eiserman +emineo +krono +larshp +logali +müller +nerdcast +nerdletter +nuve +sandrarossi +sbcgua + +# Everything else +backport +breakpoint +bwip +claude +ctrl +cust +customizable +deprecations +ele +fcl +kpi +reviewable +rfc +rtti +rulebook +sql +wordmark diff --git a/test/gates.test.mjs b/test/gates.test.mjs index 2d4aaf7b..cd7a41c6 100644 --- a/test/gates.test.mjs +++ b/test/gates.test.mjs @@ -1,5 +1,5 @@ /* - * The fourteen gates are written out in three places. Do all three name the + * The fifteen gates are written out in three places. Do all three name the * same set? * * `package.json`'s `check` script is what a contributor runs; check.yml is @@ -88,13 +88,13 @@ test('check.yml keeps the script order, so a green run locally is a green run th assert.deepEqual(workflowGates('.github/workflows/check.yml'), scriptGates()); }); -test('the documents say fourteen, and there are fourteen', () => { +test('the documents say fifteen, and there are fifteen', () => { /* AGENTS.md, README.md, CONTRIBUTING.md and CLAUDE.md all count them in - * prose. A fifteenth gate that left the four documents saying "fourteen" is the + * prose. A sixteenth gate that left the four documents saying "fifteen" is the * same drift as a gate missing from a workflow, one document over. */ const gates = scriptGates(); - assert.equal(gates.length, 14, `the count in the four documents is 14, the lists have ${gates.length}`); + assert.equal(gates.length, 15, `the count in the four documents is 15, the lists have ${gates.length}`); for (const doc of ['AGENTS.md', 'README.md', 'CONTRIBUTING.md', 'CLAUDE.md']) { - assert.match(read(doc), /fourteen/, `${doc} counts the gates`); + assert.match(read(doc), /fifteen/, `${doc} counts the gates`); } }); diff --git a/test/vocabulary.test.mjs b/test/vocabulary.test.mjs new file mode 100644 index 00000000..f5c25a9d --- /dev/null +++ b/test/vocabulary.test.mjs @@ -0,0 +1,73 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { manualPages } from '../scripts/lib/prose.mjs'; +import { findUnknown, projectWords } from '../scripts/lib/vocabulary.mjs'; + +const unknown = (markdown, words = []) => findUnknown(markdown, { words }).map((f) => f.word); +const page = (body) => `# Title\n\n${body}\n`; + +test('the four typos that reached the published site are caught', () => { + /* Two hand edits, two days, one page - and all fourteen other gates green. + This is the case the gate exists for. */ + assert.deepEqual( + unknown(page('To use a framework, you always havte to fullfll certain requoirememnts.')), + ['havte', 'fullfll', 'requoirememnts'], + ); + assert.deepEqual(unknown(page('that can be a structure, thant can be another.')), ['thant']); +}); + +test('ordinary English is not reported', () => { + assert.deepEqual(unknown(page('To use a framework, you always have to fulfill certain requirements.')), []); +}); + +test('an identifier is not a word to be spelled', () => { + assert.deepEqual(unknown(page('abap2UI5 calls z2ui5_if_app and the liveChange wire, per the ICF and its APIs.')), []); +}); + +test('a name with a letter outside A-Z is one token', () => { + /* [A-Za-z] starts a new token at the "ller" of "Müller" and reports it. */ + assert.deepEqual(unknown(page('edited by Müller'), ['müller']), []); + assert.deepEqual(unknown(page('edited by Müller')), ['Müller']); +}); + +test('emphasis inside a word does not split it', () => { + /* `**f**rontend` renders as one word and has to be read as one. Reported + as "frontend" (which the project list carries) and never as "rontend". */ + assert.deepEqual(unknown(page('The UI5 **f**rontend, embedded as constants.')), ['frontend']); + assert.deepEqual(unknown(page('The UI5 **f**rontend, embedded as constants.'), ['frontend']), []); +}); + +test('a hyphenated compound is judged part by part', () => { + assert.deepEqual(unknown(page('a design-time artifact and a runtime-typed model'), ['typed']), []); + assert.deepEqual(unknown(page('a design-tyme artifact')), ['tyme']); +}); + +test('code, link targets and generated blocks are not prose', () => { + const body = [ + '```abap', 'DATA(havte) = fullfll.', '```', '', + 'See `zcl_havte_fullfll` and [the page](/advanced/havte-fullfll).', '', + '', + 'havte fullfll requoirememnts', + '', + ].join('\n'); + assert.deepEqual(unknown(page(body)), []); +}); + +test('a project word is known through the list, and the list is read', () => { + assert.deepEqual(unknown(page('one roundtrip per click'), ['roundtrip']), []); + assert.deepEqual(unknown(page('one roundtrip per click')), ['roundtrip']); + assert.ok(projectWords().length > 100, 'vocabulary.txt carries the project words'); + assert.ok(projectWords().includes('roundtrip')); +}); + +test('every prose word of the manual is one something knows', () => { + const words = projectWords(); + const found = []; + for (const file of manualPages('docs')) { + for (const f of findUnknown(readFileSync(file, 'utf8'), { words })) { + found.push(`${file}:${f.line} ${f.word}`); + } + } + assert.deepEqual(found, []); +});