Spec 001 T030 follow-up: long-token breaks

ci/measure/breaks.ts

@@ -0,0 +1,123 @@
+// T030 (research R8): where Chromium breaks a line, and what can still be wider than the
+// text column once long tokens are given places to break (src/layout/html.ts).
+//
+//   node ci/measure/breaks.ts
+//
+// 1. Spaces: each character JavaScript's `\s` matches, and U+180E, U+200B and U+2060, set
+//    between two runs of eight letters in a box too narrow for both: whether Chromium puts
+//    them on two lines. Each is compared with `BREAKABLE_SPACE`; any disagreement is exit 1.
+// 2. Tables: a summation's table of two rows, every cell one token, laid out by Chromium
+//    with the book's stylesheets and print media, in a `.summation` as wide as the text
+//    column (A4 less two 20 mm margins, 170 mm); the token given its breaks by
+//    `markupBreaks`, as the layout does. For each token length, the fewest columns whose
+//    table is wider than the column. The letters are given: a widest case of `m` alone,
+//    and lowercase text cut from biochemical words, as a summation's long words are. The
+//    figures depend on the letters, so they are approximate for any other text.
+//
+// The output is numbers only.
+import { chromium } from 'playwright-core';
+
+import { pageStyles } from '../../src/layout/document.ts';
+import { BREAKABLE_SPACE, markupBreaks } from '../../src/layout/html.ts';
+
+const hex = (code: number) => `U+${code.toString(16).toUpperCase().padStart(4, '0')}`;
+
+const browser = await chromium.launch();
+let disagree = 0;
+try {
+  const page = await browser.newPage();
+
+  // 1. spaces
+  const codes: number[] = [];
+  for (let code = 0; code <= 0xffff; code++)
+    if (/\s/u.test(String.fromCharCode(code))) codes.push(code);
+  codes.push(0x180e, 0x200b, 0x2060);
+  await page.setContent(
+    '<div id="box" style="width: 60px; font: 16px monospace; white-space: normal"></div>'
+  );
+  // run in the page, as a string: this project's types have no DOM
+  const broken = await page.evaluate<boolean[]>(
+    `(() => {
+      const box = document.getElementById('box');
+      box.textContent = 'a';
+      const one = box.getBoundingClientRect().height;
+      return ${JSON.stringify(codes)}.map((code) => {
+        box.textContent = 'aaaaaaaa' + String.fromCharCode(code) + 'bbbbbbbb';
+        return box.getBoundingClientRect().height > one * 1.5;
+      });
+    })()`
+  );
+  console.log(
+    'Spaces: whether Chromium breaks a line there (DISAGREE: BREAKABLE_SPACE says otherwise)'
+  );
+  codes.forEach((code, i) => {
+    const chromiumBreaks = broken[i] === true;
+    const ends = BREAKABLE_SPACE.test(String.fromCharCode(code));
+    if (chromiumBreaks !== ends) disagree++;
+    const mark = chromiumBreaks === ends ? '' : '  DISAGREE';
+    console.log(`  ${hex(code)}  ${chromiumBreaks ? 'breaks' : 'joins '}  ${mark}`);
+  });
+
+  // 2. tables
+  const css = pageStyles();
+  const words = [
+    'phosphatidylinositol',
+    'glycosyltransferase',
+    'ubiquitin',
+    'mitochondrial',
+    'dehydrogenase',
+    'acetylglucosamine',
+    'carboxykinase',
+    'phosphofructokinase',
+    'transcription',
+    'cytochrome',
+    'oxidoreductase',
+    'aminoacyl',
+    'synthetase',
+    'glutathione',
+    'peroxidase',
+  ].join('');
+  const letters: Record<string, (length: number, column: number) => string> = {
+    'm alone': (length) => 'm'.repeat(length),
+    'biochemical words': (length, column) =>
+      (words + words).slice((column * 37) % words.length).slice(0, length),
+  };
+  const lengths = [20, 21, 25, 30, 35, 40, 41, 50, 60, 100];
+  console.log('\nTables: the fewest columns wider than the text column, by token length');
+  for (const [name, token] of Object.entries(letters)) {
+    console.log(`  ${name}`);
+    for (const length of lengths) {
+      let found = 'none of 1 to 12';
+      for (let columns = 1; columns <= 12; columns++) {
+        const cells = Array.from(
+          { length: columns },
+          (_, c) => `<td>${token(length, c)}</td>`
+        ).join('');
+        const table = markupBreaks(`<table><tr>${cells}</tr><tr>${cells}</tr></table>`);
+        await page.setContent(
+          `<!doctype html><html><head><style>${css.fonts}</style><style>${css.theme}</style>` +
+            `<style>${css.pages}</style></head><body class="book">` +
+            `<div class="summation" style="width: 170mm">${table}</div></body></html>`
+        );
+        await page.emulateMedia({ media: 'print' });
+        const [column, wide] = await page.evaluate<[number, number]>(
+          `document.fonts.ready.then(() => [
+            document.querySelector('.summation').getBoundingClientRect().width,
+            document.querySelector('table').scrollWidth,
+          ])`
+        );
+        if (wide > column + 0.5) {
+          found = `${String(columns)} (${wide.toFixed(0)} px > ${column.toFixed(0)} px)`;
+          break;
+        }
+      }
+      console.log(`    ${String(length).padStart(3)} characters: ${found}`);
+    }
+  }
+} finally {
+  await browser.close();
+}
+if (disagree > 0) {
+  console.log(`\n${String(disagree)} spaces disagree`);
+  process.exitCode = 1;
+}

ci/measure/results/2026-11-04/breaks/output.txt

@@ -0,0 +1,53 @@
+Spaces: whether Chromium breaks a line there (DISAGREE: BREAKABLE_SPACE says otherwise)
+  U+0009  breaks  
+  U+000A  breaks  
+  U+000B  joins   
+  U+000C  joins   
+  U+000D  breaks  
+  U+0020  breaks  
+  U+00A0  joins   
+  U+1680  breaks  
+  U+2000  breaks  
+  U+2001  breaks  
+  U+2002  breaks  
+  U+2003  breaks  
+  U+2004  breaks  
+  U+2005  breaks  
+  U+2006  breaks  
+  U+2007  joins   
+  U+2008  breaks  
+  U+2009  breaks  
+  U+200A  breaks  
+  U+2028  breaks  
+  U+2029  breaks  
+  U+202F  joins   
+  U+205F  breaks  
+  U+3000  breaks  
+  U+FEFF  joins   
+  U+180E  joins   
+  U+200B  breaks  
+  U+2060  joins   
+
+Tables: the fewest columns wider than the text column, by token length
+  m alone
+     20 characters: 3 (685 px > 643 px)
+     21 characters: 3 (718 px > 643 px)
+     25 characters: 3 (849 px > 643 px)
+     30 characters: 2 (675 px > 643 px)
+     35 characters: 2 (784 px > 643 px)
+     40 characters: 2 (893 px > 643 px)
+     41 characters: 4 (653 px > 643 px)
+     50 characters: 4 (783 px > 643 px)
+     60 characters: 3 (685 px > 643 px)
+    100 characters: 3 (685 px > 643 px)
+  biochemical words
+     20 characters: 5 (647 px > 643 px)
+     21 characters: 5 (677 px > 643 px)
+     25 characters: 5 (795 px > 643 px)
+     30 characters: 4 (748 px > 643 px)
+     35 characters: 3 (663 px > 643 px)
+     40 characters: 3 (754 px > 643 px)
+     41 characters: 7 (674 px > 643 px)
+     50 characters: 6 (691 px > 643 px)
+     60 characters: 5 (675 px > 643 px)
+    100 characters: 5 (685 px > 643 px)

specs/001-pathway-pdfs/contracts/cli.md

@@ -203,9 +203,14 @@ FR-011's check, alone. Also run by `book`.
     figure's link (a diagram's labels). Measured against the section, not in points, so a
     part Chromium printed scaled down still reads its names, its scale not a fault: the
     layout keeps realistic content inside the text column (long tokens may break, a table's
-    columns narrow to their longest words), but a table of very many columns can still be
-    wider; `no-stable-id`: its text holds no `Stable identifier` followed by its stable
-    identifier;
+    columns narrow to their longest words), but a table whose longest words, side by side,
+    are wider than the column can still be wider. A token of up to 40 characters does not
+    break, and a longer one breaks into pieces of 13 to 20; how many columns are too many
+    depends on the letters. Measured (`ci/measure/breaks.ts`, lowercase text cut from
+    biochemical words), roughly: five columns of 20- to 25-character words, four of 30,
+    three of 35 to 40, and five to seven of longer tokens; of wider letters (`m` alone),
+    two to four;
+    `no-stable-id`: its text holds no `Stable identifier` followed by its stable identifier;
   - `missing-figure`: a pathway's or reaction's section with a figure by the Figure rules,
     whose source is in `--files` (its diagram's two files, its EHLD; a reaction's is the
     graph), has no link to that figure's interactive form below its heading. The figure is

specs/001-pathway-pdfs/research.md

@@ -954,7 +954,15 @@ embedded.
     28 human events have a `<table>` in their summation, so a table wider than the text
     column, which Chromium printed by scaling its whole part down below the theme's sizes,
     would have reached the real book: the layout now keeps realistic content inside the
-    column (pages.css; a table of very many columns can still be wider), and the check
+    column (pages.css; a table whose longest words, side by side, are wider than the
+    column can still be wider: a token of up to 40 characters does not break, a longer one
+    breaks into pieces of 13 to 20. Measured by `ci/measure/breaks.ts`, a table of two
+    rows laid out by Chromium with the book's stylesheets and print media in a
+    `.summation` 170 mm wide, the text column; its output in
+    `ci/measure/results/2026-11-04/breaks/`. With lowercase text cut from biochemical
+    words, roughly five columns of 20- to 25-character words are wider, four of 30,
+    three of 35 to 40, and five to seven of longer tokens; with `m` alone, two to four.
+    The figures depend on the letters, so they are approximate), and the check
     reads headings against the section's own body text, so a part printed scaled down is
     not a fault. No human summation holds a
     `<pre>` block, and no human event's name holds a soft hyphen, a zero-width or

specs/001-pathway-pdfs/tasks.md

@@ -536,6 +536,11 @@ passes.
       - **`overflow-wrap: anywhere` broke short columns' words (high).** In a table it let every column narrow to a character, so a table wider than the column broke its short columns' words (`Da`/`te`). `body` now has `overflow-wrap: break-word`, which keeps each column at its longest word, and the layout gives a token longer than 40 characters a `<wbr>` every 20 (`text()` in `src/layout/html.ts`, grapheme by grapheme, for every text it writes; `markupBreaks` for a summation's text runs, a character reference one unit; an attribute's value and the page's `<title>` never, through `plainText`), which adds no character to what is read. An edit's date is kept whole (`white-space: nowrap`), as a column narrowed to its longest word broke `2026-01-04` at its hyphens. Test: an edit's names, 30 of them, and a summation table of `Phosphofructokinase` beside a column of prose: exit 0, `Date` and `Action` whole and no fragment of them, every date whole, `Phosphofructokinase` whole four times; on #160's CSS it read `Da`, `te`, `Ac`. The overflow test is kept: its table is now four columns of long words, as wide as release 98's are (six columns of 28-letter words are wider than the column whatever breaks, and a part holding one is printed scaled down, which the check tolerates), and its 140-character token also sits in a table cell, where only its `<wbr>`s let the column narrow.
       - **Lows.** The docs no longer say nothing overflows: the layout keeps realistic content inside the column, a table of very many columns can still be wider, and the check's measure against the section tolerates a part printed scaled down (contracts/cli.md, research R8). `folded()` builds its pattern from FR-017's `NEVER_DRAWN` (`src/layout/characters.ts`), not a copy. `identifies()` reads a candidate's section only to the next place a section may start (an event's first link to its page, or an outline entry with an event's name). Recorded: the failed-removal test (`test/print/temporary.test.ts`) relies on `chmod` keeping a folder from being emptied, so it does not hold when run as root.
       - **Tests:** 439 before, 441 after (the labels and the short-columns tests). Each of these mutants made one fail: the page's commonest size with the figures' labels counted, as before (the labels test); `overflow-wrap: anywhere` again, and the dates allowed to break (the short-columns test, each); no `<wbr>` at all, and none in summations (the overflow test, each). Survived: the label's size alone, and the page's size without the figures' labels alone (each covers the other on the fixture; both kept, the second as the fallback for a section with no label); `identifies()` unbounded (no fixture section lacks its label, the only case it changes); no `overflow-wrap` and the tables' `max-width: 100%` (with the `<wbr>`s, nothing tested is wider than a line; both kept, as the review asked); the facts' grid at `minmax(0, 1fr)` and the contents' links at `min-width: 0`, taken out again.
+    - **Follow-up (#164, replacing #163 after its review):** #162's review's five lows. A token ends only at a space Chromium breaks a line at, as measured (`ci/measure/breaks.ts`): tab, line feed, carriage return, space, U+1680, U+2000 to U+2006, U+2008 to U+200B, U+2028, U+2029, U+205F and U+3000, and a numeric reference to one; a no-break space, U+2007, U+202F, U+FEFF, U+000B and U+000C no longer end one. A summation's markup is read whole (`markupBreaks`, which `markupText` no longer feeds run by run), its tags never counted or broken into, and a word counted whole through the inline tags (`a`, `b`, `em`, `i`, `span`, `strong`, `sub`, `sup`, `u`) and comments, any other tag ending it; it relies on `sanitiseSummation`, which the model applies to every summation. A token over 40 breaks into equal pieces of at most 20 (41 is 14, 14 and 13, not 20, 20 and 1). The docs (contracts/cli.md, research R8) say what can still overflow, measured by `ci/measure/breaks.ts` (output in `ci/measure/results/2026-11-04/breaks/`) and approximate, as it depends on the letters: with lowercase text cut from biochemical words, five columns of 20- to 25-character words, four of 30, three of 35 to 40, five to seven of longer tokens (broken into pieces of 13 to 20); with `m` alone, two to four. Tests: `test/layout/html.test.ts` (11) for `text()` and `markupBreaks`, and in `test/model/build.test.ts` every built summation sanitised; 441 before, 452 after. Each of these mutants made one fail: the old `/^\s$/u` space (the spaces test, and the markup one through a no-break space); a carriage return and line feed not taken as a space (the spaces test); character references not split out (`&amp;` broken into, the references test); `text()` escaping before it breaks (the escaped test); tags taken as text (the attribute and inline-tags tests); every tag ending a word, and none (the inline-tags test, each); tags counted in a word's length (the same); a `<` before no tag name taken as a tag (the markup test); a break every 20 again (five tests).
+      - **From #163's review:** two mediums, each reproduced, and lows.
+        - **Breaking spaces dropped (medium, a regression).** #163 ended a token only at a space, tab, line feed or carriage return, so two words beside a thin space (or an en, em or ideographic space, or U+200B, which main also missed) counted as one token, and a long pair broke mid-word; 9 of release 98's human summations hold such spaces (measured on the private copy). `BREAKABLE_SPACE` is now the set Chromium was measured to break at, which `ci/measure/breaks.ts` checks (exit 1 on a disagreement); the review's rule, `\s` without U+00A0, U+202F and U+FEFF and with U+200B, would also have ended tokens at U+2007, U+000B and U+000C, at which Chromium does not break. Tests: two 28- and 17-letter words beside a thin space kept whole; every measured space, breaking and joining. Each of these mutants made one fail: #163's set (the thin-space, spaces and reference tests); the set without U+2000 to U+200B (the same); the review's rule, and U+2007 taken as a space (the spaces test, each); numeric references never taken as spaces (the reference test).
+        - **The overflow figures (medium).** #163's figures, measured with the one word `phosphatidylinositoltrisphosphatekinases`, did not hold for other letters, and said pieces of 14 to 20 where 41 gives 13. Re-measured by a script that can be re-run (`ci/measure/breaks.ts`), the letters and the method stated, the figures given as approximate (above).
+        - **Lows.** The tag reader's branches are tested: a quoted `>` in an attribute, a comment within a word, `a` and `b` inline, and a tag in capitals; each of these mutants made the test fail: quotes not read in a tag, comments ending a word, `a` and `b` not inline, tag names not lowercased. `markupBreaks`'s doc comment says it relies on `sanitiseSummation`, and that raw markup (CDATA, a doctype, a processing instruction, a script) is not read rightly; a test gives such raw markup through the sanitiser and finds no `<wbr>` inside a tag, and the model's test checks every summation it builds is the sanitiser's fixed point (with the model not sanitising, it fails, as the existing check for scripts does).
     - **Known gaps:** the rules are measured on the fixture alone, whose names are short and whose figures small: T031 runs the check on release 98's book, and its time against R7's 9 minutes. A figure is known by its link: a drawing lost with its link kept is not seen. A heading that lost its link and every entry naming it is found nowhere, and is then three faults. A link landing on a page but not on its target is not a fault (R8: every link landing on a page). Between `bookCommand` returning and the process ending there are no handlers, so a signal then is the default action, exit 128 and its number, the book and its report already in place.
 
 - [ ] T031 [US1] The book on a private copy of release 98's graph.