diff --git a/ci/measure/breaks.ts b/ci/measure/breaks.ts index 4dd27c0..7b9c2b1 100644 --- a/ci/measure/breaks.ts +++ b/ci/measure/breaks.ts @@ -3,9 +3,12 @@ // // node ci/measure/breaks.ts // -// 1. Spaces: each character JavaScript's `\s` matches, and U+180E, U+200B and U+2060, set -// between two runs of eight letters in a box too narrow for both: whether Chromium puts -// them on two lines. Each is compared with `BREAKABLE_SPACE`; any disagreement is exit 1. +// 1. Breaks: each character JavaScript's `\s` matches, U+0085, U+180E, U+200B and U+2060, +// ASCII's punctuation, and hyphens and dashes, set between two runs of eight characters +// (letters, digits, and each both ways round) in a box too narrow for both: whether +// Chromium breaks the line after it, before it, both or not at all, and the same for a +// space or a hyphen with a combining mark. Each is compared with `breakBeside` +// (src/layout/html.ts), a space expected to break after; any disagreement is exit 1. // 2. Tables: a summation's table of two rows, every cell one token, laid out by Chromium // with the book's stylesheets and print media, in a `.summation` as wide as the text // column (A4 less two 20 mm margins, 170 mm); the token given its breaks by @@ -18,7 +21,7 @@ import { chromium } from 'playwright-core'; import { pageStyles } from '../../src/layout/document.ts'; -import { BREAKABLE_SPACE, markupBreaks } from '../../src/layout/html.ts'; +import { breakBeside, markupBreaks } from '../../src/layout/html.ts'; const hex = (code: number) => `U+${code.toString(16).toUpperCase().padStart(4, '0')}`; @@ -27,35 +30,75 @@ let disagree = 0; try { const page = await browser.newPage(); - // 1. spaces + // 1. where a line breaks const codes: number[] = []; for (let code = 0; code <= 0xffff; code++) if (/\s/u.test(String.fromCharCode(code))) codes.push(code); - codes.push(0x180e, 0x200b, 0x2060); + codes.push(0x85, 0x180e, 0x200b, 0x2060); + for (let code = 0x21; code <= 0x7e; code++) + if (!/[a-z0-9]/i.test(String.fromCharCode(code))) codes.push(code); + codes.push(0xad, 0xb7, 0x58a, 0x1806, 0x2027, 0x2043, 0x2044, 0x2212, 0x2215, 0x2e17); + for (let code = 0x2010; code <= 0x2015; code++) codes.push(code); + codes.push(0x2e1a, 0x30fb); + // a combining acute accent after a space or a hyphen: one grapheme with it + const marked = [0x20, 0x2009, 0x2d, 0x2010].map( + (code) => String.fromCharCode(code) + String.fromCharCode(0x301) + ); + const units = [...codes.map((code) => String.fromCharCode(code)), ...marked]; await page.setContent( '
' ); - // run in the page, as a string: this project's types have no DOM - const broken = await page.evaluate( + // run in the page, as a string: this project's types have no DOM. For each unit between + // eight characters and eight more (letters, digits, both ways round), too wide for one + // line: whether the line breaks after it, before it, both, or not at all. + const measured = await page.evaluate( `(() => { const box = document.getElementById('box'); - box.textContent = 'a'; - const one = box.getBoundingClientRect().height; - return ${JSON.stringify(codes)}.map((code) => { - box.textContent = 'aaaaaaaa' + String.fromCharCode(code) + 'bbbbbbbb'; - return box.getBoundingClientRect().height > one * 1.5; - }); + const top = (node, from, to) => { + const range = document.createRange(); + range.setStart(node, from); + range.setEnd(node, to); + const rect = range.getClientRects()[0]; + return rect ? Math.round(rect.top) : NaN; + }; + return ${JSON.stringify(units)}.map((unit) => + [['a', 'b'], ['a', '1'], ['1', 'b'], ['1', '2']].map(([left, right]) => { + box.textContent = left.repeat(8) + unit + right.repeat(8); + const node = box.firstChild; + const before = top(node, 7, 8); + const after = top(node, 8 + unit.length, 9 + unit.length); + const lines = new Set(); + for (let i = 0; i < 16 + unit.length; i++) lines.add(top(node, i, i + 1)); + lines.delete(NaN); + if (lines.size === 1) return 'none'; + // the unit's own line, where it is drawn (a space at a line's end may not be) + let own = NaN; + for (let i = 8; i < 8 + unit.length && Number.isNaN(own); i++) own = top(node, i, i + 1); + if (Number.isNaN(own)) return 'after'; + if (own === before && own !== after) return 'after'; + if (own === after && own !== before) return 'before'; + // on neither line: on one of its own (both), or a soft hyphen's added hyphen + return lines.size > 2 ? 'both' : 'after'; + }) + ); })()` ); console.log( - 'Spaces: whether Chromium breaks a line there (DISAGREE: BREAKABLE_SPACE says otherwise)' + 'Breaks: where Chromium breaks a line beside each, between letters and digits' + + ' (DISAGREE: breakBeside says otherwise)' ); - codes.forEach((code, i) => { - const chromiumBreaks = broken[i] === true; - const ends = BREAKABLE_SPACE.test(String.fromCharCode(code)); - if (chromiumBreaks !== ends) disagree++; - const mark = chromiumBreaks === ends ? '' : ' DISAGREE'; - console.log(` ${hex(code)} ${chromiumBreaks ? 'breaks' : 'joins '} ${mark}`); + units.forEach((unit, i) => { + const found = [...new Set(measured[i])]; + const expected = breakBeside(unit); + const agrees = found.length === 1 && found[0] === (expected === 'space' ? 'after' : expected); + // a hyphen with a mark is taken to join, as decided (breakBeside): not a disagreement + const decided = !agrees && expected === 'none' && Array.from(unit).length > 1; + if (!agrees && !decided) disagree++; + const name = Array.from(unit, (char) => hex(char.codePointAt(0) ?? 0)).join(' '); + // by context (letter-letter, letter-digit, digit-letter, digit-digit) where they differ + const what = found.length === 1 ? (found[0] ?? '') : (measured[i] ?? []).join(' '); + const mark = agrees ? '' : decided ? ' taken to join, as decided' : ' DISAGREE'; + console.log(` ${name.padEnd(13)} ${what.padEnd(7)} ${mark}`); }); // 2. tables @@ -118,6 +161,6 @@ try { await browser.close(); } if (disagree > 0) { - console.log(`\n${String(disagree)} spaces disagree`); + console.log(`\n${String(disagree)} disagree`); process.exitCode = 1; } diff --git a/ci/measure/results/2026-11-04/breaks/output.txt b/ci/measure/results/2026-11-04/breaks/output.txt index 13b8d9f..41d3f8c 100644 --- a/ci/measure/results/2026-11-04/breaks/output.txt +++ b/ci/measure/results/2026-11-04/breaks/output.txt @@ -1,32 +1,87 @@ -Spaces: whether Chromium breaks a line there (DISAGREE: BREAKABLE_SPACE says otherwise) - U+0009 breaks - U+000A breaks - U+000B joins - U+000C joins - U+000D breaks - U+0020 breaks - U+00A0 joins - U+1680 breaks - U+2000 breaks - U+2001 breaks - U+2002 breaks - U+2003 breaks - U+2004 breaks - U+2005 breaks - U+2006 breaks - U+2007 joins - U+2008 breaks - U+2009 breaks - U+200A breaks - U+2028 breaks - U+2029 breaks - U+202F joins - U+205F breaks - U+3000 breaks - U+FEFF joins - U+180E joins - U+200B breaks - U+2060 joins +Breaks: where Chromium breaks a line beside each, between letters and digits (DISAGREE: breakBeside says otherwise) + U+0009 after + U+000A after + U+000B none + U+000C none + U+000D after + U+0020 after + U+00A0 none + U+1680 after + U+2000 after + U+2001 after + U+2002 after + U+2003 after + U+2004 after + U+2005 after + U+2006 after + U+2007 none + U+2008 after + U+2009 after + U+200A after + U+2028 after + U+2029 after + U+202F none + U+205F after + U+3000 after + U+FEFF none + U+0085 after + U+180E none + U+200B after + U+2060 none + U+0021 none + U+0022 none + U+0023 none + U+0024 none + U+0025 none + U+0026 none + U+0027 none + U+0028 none + U+0029 none + U+002A none + U+002B none + U+002C none + U+002D after + U+002E none + U+002F none + U+003A none + U+003B none + U+003C none + U+003D none + U+003E none + U+003F after + U+0040 none + U+005B none + U+005C none + U+005D none + U+005E none + U+005F none + U+0060 none + U+007B none + U+007C none + U+007D none + U+007E none + U+00AD after + U+00B7 none + U+058A after + U+1806 before + U+2027 after + U+2043 none + U+2044 none + U+2212 none + U+2215 none + U+2E17 after + U+2010 after + U+2011 none + U+2012 after + U+2013 after + U+2014 both + U+2015 none + U+2E1A none + U+30FB after + U+0020 U+0301 after + U+2009 U+0301 after + U+002D U+0301 after none after none taken to join, as decided + U+2010 U+0301 after taken to join, as decided Tables: the fewest columns wider than the text column, by token length m alone diff --git a/specs/001-pathway-pdfs/contracts/cli.md b/specs/001-pathway-pdfs/contracts/cli.md index bdfc2fa..0fc4f55 100644 --- a/specs/001-pathway-pdfs/contracts/cli.md +++ b/specs/001-pathway-pdfs/contracts/cli.md @@ -204,8 +204,9 @@ FR-011's check, alone. Also run by `book`. part Chromium printed scaled down still reads its names, its scale not a fault: the layout keeps realistic content inside the text column (long tokens may break, a table's columns narrow to their longest words), but a table whose longest words, side by side, - are wider than the column can still be wider. A token of up to 40 characters does not - break, and a longer one breaks into pieces of 13 to 20; how many columns are too many + are wider than the column can still be wider. A run of up to 40 characters between + places Chromium breaks a line (a space, or after a hyphen) does not break, and a longer + one breaks into pieces of 13 to 20; how many columns are too many depends on the letters. Measured (`ci/measure/breaks.ts`, lowercase text cut from biochemical words), roughly: five columns of 20- to 25-character words, four of 30, three of 35 to 40, and five to seven of longer tokens; of wider letters (`m` alone), diff --git a/specs/001-pathway-pdfs/research.md b/specs/001-pathway-pdfs/research.md index 76b7aa8..a18b0d3 100644 --- a/specs/001-pathway-pdfs/research.md +++ b/specs/001-pathway-pdfs/research.md @@ -955,8 +955,9 @@ embedded. column, which Chromium printed by scaling its whole part down below the theme's sizes, would have reached the real book: the layout now keeps realistic content inside the column (pages.css; a table whose longest words, side by side, are wider than the - column can still be wider: a token of up to 40 characters does not break, a longer one - breaks into pieces of 13 to 20. Measured by `ci/measure/breaks.ts`, a table of two + column can still be wider: a run of up to 40 characters between places Chromium + breaks a line (a space, or after a hyphen) does not break, a longer one breaks into + pieces of 13 to 20. Measured by `ci/measure/breaks.ts`, a table of two rows laid out by Chromium with the book's stylesheets and print media in a `.summation` 170 mm wide, the text column; its output in `ci/measure/results/2026-11-04/breaks/`. With lowercase text cut from biochemical diff --git a/specs/001-pathway-pdfs/tasks.md b/specs/001-pathway-pdfs/tasks.md index 359a779..ad710a5 100644 --- a/specs/001-pathway-pdfs/tasks.md +++ b/specs/001-pathway-pdfs/tasks.md @@ -541,6 +541,7 @@ passes. - **Breaking spaces dropped (medium, a regression).** #163 ended a token only at a space, tab, line feed or carriage return, so two words beside a thin space (or an en, em or ideographic space, or U+200B, which main also missed) counted as one token, and a long pair broke mid-word; 9 of release 98's human summations hold such spaces (measured on the private copy). `BREAKABLE_SPACE` is now the set Chromium was measured to break at, which `ci/measure/breaks.ts` checks (exit 1 on a disagreement); the review's rule, `\s` without U+00A0, U+202F and U+FEFF and with U+200B, would also have ended tokens at U+2007, U+000B and U+000C, at which Chromium does not break. Tests: two 28- and 17-letter words beside a thin space kept whole; every measured space, breaking and joining. Each of these mutants made one fail: #163's set (the thin-space, spaces and reference tests); the set without U+2000 to U+200B (the same); the review's rule, and U+2007 taken as a space (the spaces test, each); numeric references never taken as spaces (the reference test). - **The overflow figures (medium).** #163's figures, measured with the one word `phosphatidylinositoltrisphosphatekinases`, did not hold for other letters, and said pieces of 14 to 20 where 41 gives 13. Re-measured by a script that can be re-run (`ci/measure/breaks.ts`), the letters and the method stated, the figures given as approximate (above). - **Lows.** The tag reader's branches are tested: a quoted `>` in an attribute, a comment within a word, `a` and `b` inline, and a tag in capitals; each of these mutants made the test fail: quotes not read in a tag, comments ending a word, `a` and `b` not inline, tag names not lowercased. `markupBreaks`'s doc comment says it relies on `sanitiseSummation`, and that raw markup (CDATA, a doctype, a processing instruction, a script) is not read rightly; a test gives such raw markup through the sanitiser and finds no `` inside a tag, and the model's test checks every summation it builds is the sanitiser's fixed point (with the model not sanitising, it fails, as the existing check for scripts does). + - **Follow-up (#166):** #164's review's lows. **Breaks beside hyphens (low 6):** a token is now measured by its longest run between places Chromium breaks a line, so a hyphenated name is not broken mid-run: `1-phosphatidyl-1D-myo-inositol-4,5-bisphosphate` was `1-phosphatidyl-1D-myo-inositol-4,5-bisphosphate`, and is now whole. `ci/measure/breaks.ts`'s first part measures ASCII's punctuation and the hyphens and dashes too, between letters and digits both ways round, and checks `breakBeside` (`src/layout/html.ts`) against it (exit 1 on a disagreement): Chromium breaks after `-`, U+2010, U+00AD, U+058A, U+2012, U+2013, U+2027, U+2E17, U+30FB and `?`, before U+1806, and on both sides of U+2014; not beside U+2011, U+2015, U+2212, `/`, `|` or other punctuation. A hyphen is counted in the run it closes; breaks still go only inside a run longer than 40. **U+2011:** FR-017's substitution (`printedDocument`, and the on-request mapping) runs on the document before the layout, so a non-breaking hyphen reaches `text()` as `-`, and Chromium breaks after the printed `-` whatever the layout does: decided, it is counted as the break it is in print. Keeping the author's no-break would need a word joiner or a `nowrap` span in the printed text, which the check and a reader's search would then meet; not done. **NEL:** U+0085 measured as a break, and added to the spaces. **Combining marks:** a space with a combining mark is one grapheme, and now ends a token as the space does (classed by its first character; Chromium breaks before the mark after U+0020, after it after U+2009, so a mark may go uncounted: one character in a run, at most); a hyphen with a mark is taken to join (Chromium breaks after it before a letter, not before a digit), at most a `` too many, and the script reports it as decided. **The comment:** U+2002 and U+2003 are the en and em spaces, in U+2000 to U+2006; U+2008 to U+200A hold the thin and hair spaces. **Corrections to the line above:** a numeric reference to a space is dead code after the sanitiser, which writes every character but `&`, `<`, `>` and `"` as itself (` ` comes out as U+2009; checked), so only a direct call reaches it, and its comment says so. The model test's sanitiser assertion already bites: the fixture's Alpha pathway summation holds a script, a `javascript:` link and an `onerror` image, which the sanitiser changes; with the model not sanitising and the older script check taken out, the assertion alone fails the test. Tests: `test/layout/html.test.ts` 3 more (14), 452 before, 455 after. Each of these mutants made one fail: no break-after characters (the hyphen and break-beside tests); the hyphen alone breaking after, no soft hyphen, U+2011 taken to break after, and nothing breaking before (the break-beside test, each); a hyphen counted in the next run (the hyphen test); a space tested as the whole grapheme, and a hyphen with a mark breaking as the hyphen (the combining-mark test, each); no NEL (the spaces test); the model not sanitising with the script check taken out (the model's parts test). - **Known gaps:** the rules are measured on the fixture alone, whose names are short and whose figures small: T031 runs the check on release 98's book, and its time against R7's 9 minutes. A figure is known by its link: a drawing lost with its link kept is not seen. A heading that lost its link and every entry naming it is found nowhere, and is then three faults. A link landing on a page but not on its target is not a fault (R8: every link landing on a page). Between `bookCommand` returning and the process ending there are no handlers, so a signal then is the default action, exit 128 and its number, the book and its report already in place. - [ ] T031 [US1] The book on a private copy of release 98's graph. diff --git a/src/layout/html.ts b/src/layout/html.ts index e865ee7..eedb12f 100644 --- a/src/layout/html.ts +++ b/src/layout/html.ts @@ -24,14 +24,49 @@ export const BREAK_EVERY = 20; /** * The spaces a token ends at: those Chromium breaks a line at, measured * (`ci/measure/breaks.ts`): tab, line feed, carriage return (a carriage return and line - * feed are one grapheme), space, U+1680, U+2000 to U+2006, U+2008 to U+200A (the en, em, - * thin and hair spaces among them), U+200B (zero width), U+2028, U+2029, U+205F and U+3000. - * The others `\s` holds join their neighbours, and a token runs through them: U+000B, - * U+000C, U+00A0 (no-break), U+2007 (figure), U+202F and U+FEFF. A space left out of this - * list would join two words into one token, and a long one would break mid-word. + * feed are one grapheme), space, U+0085 (next line), U+1680, U+2000 to U+2006 (the en and + * em spaces among them), U+2008 to U+200A (the thin and hair spaces among them), U+200B + * (zero width), U+2028, U+2029, U+205F and U+3000. The others `\s` holds join their + * neighbours, and a token runs through them: U+000B, U+000C, U+00A0 (no-break), U+2007 + * (figure), U+202F and U+FEFF. A space left out of this list would join two words into one + * token, and a long one would break mid-word. */ export const BREAKABLE_SPACE = - /^[\t\n\r \u1680\u2000-\u2006\u2008-\u200b\u2028\u2029\u205f\u3000]+$/u; + /^[\t\n\r \u0085\u1680\u2000-\u2006\u2008-\u200b\u2028\u2029\u205f\u3000]+$/u; + +/** + * The characters other than spaces Chromium breaks a line beside, measured + * (`ci/measure/breaks.ts`) between letters and digits alike: after a hyphen (`-`, U+2010), + * U+058A, U+2012 and U+2013 (figure and en dashes), U+2027, U+2E17, U+30FB, `?` and U+00AD + * (the soft hyphen, which prints a hyphen where it breaks); before + * U+1806; before and after U+2014 (the em dash). Not beside U+2011, the non-breaking + * hyphen, which FR-017 prints as `-` (characters.ts) before the layout sees it: Chromium + * then breaks after it as after any hyphen, so it is counted as one. Nor beside `/`, `|` + * or U+2212 (minus). Ideographs, between which Chromium breaks too, are not + * listed: a long run of them is only given ``s where a break already was. + */ +const BREAKS_AFTER: ReadonlySet = new Set([ + 0x2d, 0x3f, 0xad, 0x58a, 0x2010, 0x2012, 0x2013, 0x2014, 0x2027, 0x2e17, 0x30fb, +]); +const BREAKS_BEFORE: ReadonlySet = new Set([0x1806, 0x2014]); + +/** + * Where a line may break beside a character as a reader sees it: a space ends a token and + * is not counted in it; a hyphen ends the run it closes; U+1806 starts one. A space with a + * combining mark is one grapheme, and breaks as the space does (Chromium ends the line + * after the mark, or for U+0020 before it, which leaves the mark uncounted: one character + * in a run, at most). A hyphen with a mark is taken to join: Chromium breaks after it + * before a letter but not before a digit, and a `` too many is harmless. + */ +export function breakBeside(unit: string): 'space' | 'after' | 'before' | 'both' | 'none' { + const first = String.fromCodePoint(unit.codePointAt(0) ?? 0); + if (BREAKABLE_SPACE.test(first)) return 'space'; + if (first !== unit) return 'none'; + const point = first.codePointAt(0) ?? 0; + const after = BREAKS_AFTER.has(point); + const before = BREAKS_BEFORE.has(point); + return after && before ? 'both' : after ? 'after' : before ? 'before' : 'none'; +} /** * A unit of text: a character as a reader sees it, a character reference, or a tag, which @@ -40,8 +75,8 @@ export const BREAKABLE_SPACE = interface Unit { readonly text: string; readonly tag?: 'joins' | 'ends'; - /** A character reference to a space a line breaks at. */ - readonly space?: boolean; + /** The character a numeric character reference stands for, where a line may break. */ + readonly stands?: string; } /** @@ -78,11 +113,17 @@ function breakTokens(units: readonly Unit[], escape: (unit: string) => string): } else if (unit.tag === 'ends') { flush(); out += unit.text; - } else if (unit.space === true || BREAKABLE_SPACE.test(unit.text)) { - flush(); - out += escape(unit.text); } else { + const at = breakBeside(unit.stands ?? unit.text); + if (at === 'space') { + flush(); + out += escape(unit.text); + continue; + } + // a run ends after a hyphen, which is counted in it + if (at === 'before' || at === 'both') flush(); token.push(unit); + if (at === 'after' || at === 'both') flush(); } } flush(); @@ -115,8 +156,9 @@ const TAG = /(|$)|<\/?([a-z][a-z0-9]*)(?:[^>"']|"[^"]*"|'[^']* const REFERENCE = /(&(?:#[0-9]+|#x[0-9a-f]+|[a-z][a-z0-9]*);)/giu; /** - * The character a numeric reference stands for; a named one is left as it is, as the - * sanitiser writes every character but `&`, `<`, `>` and `"` as itself. + * The character a numeric reference stands for; a named one is left as it is. Defensive: + * the sanitiser writes every character but `&`, `<`, `>` and `"` as itself, so no + * summation it has read holds a numeric reference, and only a direct call reaches this. */ function referenced(reference: string): string { const hex = /^&#x([0-9a-f]+);$/iu.exec(reference)?.[1]; @@ -140,7 +182,7 @@ export function markupBreaks(markup: string): string { for (let i = 0; i < parts.length; i += 3) { const between = parts[i] ?? ''; between.split(REFERENCE).forEach((part, j) => { - if (j % 2 === 1) units.push({ text: part, space: BREAKABLE_SPACE.test(referenced(part)) }); + if (j % 2 === 1) units.push({ text: part, stands: referenced(part) }); else units.push(...graphemes(part)); }); const tag = parts[i + 1]; diff --git a/test/layout/html.test.ts b/test/layout/html.test.ts index 7c715f6..ecd6d3e 100644 --- a/test/layout/html.test.ts +++ b/test/layout/html.test.ts @@ -28,7 +28,7 @@ describe("text()'s breaks", () => { const breaking = [' ', '\t', '\n', '\r', '\r\n', '\u1680', '\u200b', '\u2028', '\u2029']; for (let code = 0x2000; code <= 0x200a; code++) if (code !== 0x2007) breaking.push(String.fromCharCode(code)); - breaking.push('\u205f', '\u3000'); + breaking.push('\u205f', '\u3000', String.fromCharCode(0x85)); for (const space of breaking) assert.equal(text(`${half}${space}${half}`), `${half}${space}${half}`, JSON.stringify(space)); const joining = [ @@ -54,6 +54,42 @@ describe("text()'s breaks", () => { assert.equal(text(name), name); }); + test('a run ends after a hyphen: a hyphenated name is kept whole, a long run broken', () => { + const name = '1-phosphatidyl-1D-myo-inositol-4,5-bisphosphate'; + assert.equal(text(name), name); + assert.equal(markupBreaks(`

${name}

`), `

${name}

`); + // the hyphen counted in the run it closes: 45 and a hyphen in three pieces, then 10 + const long = `${'a'.repeat(45)}-${'b'.repeat(10)}`; + assert.deepEqual(pieces(text(long)), [16, 15, 25]); + assert.equal(text(long).replaceAll('', ''), long); + }); + + test('a run ends where Chromium breaks beside a character, and nowhere else', () => { + // as measured by ci/measure/breaks.ts, between two runs of 30 + const half = 'a'.repeat(30); + const kept = (c: string) => text(`${half}${c}${half}`) === `${half}${c}${half}`; + const after = [0x2d, 0x3f, 0xad, 0x58a, 0x2010, 0x2012, 0x2013, 0x2027, 0x2e17, 0x30fb]; + for (const code of [...after, 0x1806, 0x2014]) + assert.ok(kept(String.fromCharCode(code)), code.toString(16)); + // U+2011 (non-breaking hyphen), U+2015, U+2212 (minus), and punctuation that joins + for (const code of [0x2011, 0x2015, 0x2212, 0x2f, 0x7c, 0x2c, 0x2e, 0x2b, 0x5f, 0xb7]) + assert.deepEqual( + pieces(text(`${half}${String.fromCharCode(code)}${half}`)), + [16, 15, 15, 15] + ); + // a run that starts with U+1806 counts it there: 30, then 1 and 45 + const before = `${'a'.repeat(30)}${String.fromCharCode(0x1806)}${'b'.repeat(45)}`; + assert.deepEqual(pieces(text(before)), [46, 15, 15]); + }); + + test('a space with a combining mark ends a token; a hyphen with one is taken to join', () => { + const half = 'a'.repeat(30); + const mark = String.fromCharCode(0x301); + for (const space of [' ', String.fromCharCode(0x2009)]) + assert.equal(text(`${half}${space}${mark}${half}`), `${half}${space}${mark}${half}`); + assert.deepEqual(pieces(text(`${half}-${mark}${half}`)), [16, 16, 15, 15]); + }); + test('a character escaped is one unit, never broken into', () => { const out = text('&'.repeat(41)); assert.equal(out, ['&'.repeat(14), '&'.repeat(14), '&'.repeat(13)].join(''));