diff --git a/package.json b/package.json index 3372541..f902338 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "typecheck": "tsc -p tsconfig.json", "lint": "eslint .", "format": "prettier --check .", - "test": "node --test \"test/**/*.test.ts\"", + "test": "node --test --test-concurrency=4 --test-timeout=300000 \"test/**/*.test.ts\"", "check": "npm run typecheck && npm run lint && npm run format && npm test && node ci/licences.ts" }, "devDependencies": { diff --git a/specs/001-pathway-pdfs/contracts/cli.md b/specs/001-pathway-pdfs/contracts/cli.md index 4033e5f..0f87159 100644 --- a/specs/001-pathway-pdfs/contracts/cli.md +++ b/specs/001-pathway-pdfs/contracts/cli.md @@ -64,9 +64,9 @@ the figures already drawn by `figures`. | `--site` | `REACTOME_SITE` | the website's address, for links to its pages; never fetched | | `--species` | | default `Homo sapiens` (spec: the book's scope) | | `--out` | | the folder written; default `TheReactomeBook` | -| `--parallel` | | parts printed and PDFs joined at once, the largest PDFs first; default 4 | -| `--joins` | | of those, PDFs joined at once; default 1 (research R4: T029 raises it if 2 fit) | -| `--report` | | where the run report (data-model.md) is written, as JSON | +| `--parallel` | | parts printed and PDFs joined at once, largest PDFs first; default 4, at most 64 | +| `--joins` | | of those, PDFs joined at once; default 1, at most `--parallel` (R4: T031 settles) | +| `--report` | | where the run report (data-model.md) is written, as JSON, however the run ends | - **The preflight,** before anything is printed (and the character check's second half, on reading each figure, below): from the graph and the EHLD summary it @@ -104,8 +104,11 @@ the figures already drawn by `figures`. - **Exit 1:** a fault from the preflight or FR-011. The report names each one by stable identifier or file name, never a person. - **Exit 2:** bad arguments; an `--out` that is not absent or empty, as for `figures` - (an old book is never replaced in place); the graph cannot be reached at the start; or - any other error during the preflight, as `preflight` exits on one (below). + (an old book is never replaced in place); the graph cannot be reached at the start; + any other error during the preflight, as `preflight` exits on one (below); or any other + error while the book is printed (a print or a join that fails, or a PDF refused at its + contents, links, outline or stamps), after printing the faults found before it. A + refused PDF is never saved, and the run writes none (Exit 0). ## `pdf-export preflight` diff --git a/specs/001-pathway-pdfs/data-model.md b/specs/001-pathway-pdfs/data-model.md index 035e819..1369a38 100644 --- a/specs/001-pathway-pdfs/data-model.md +++ b/specs/001-pathway-pdfs/data-model.md @@ -170,6 +170,12 @@ cause (`unanswered`, `failed`, `refused`, `invalid` or `label`, contracts/figure - Release files missing, by name, from the preflight. - Characters FR-017's list replaced, by code point and count; characters off the list, by code point and stable identifier (a fault in the book, U+FFFD in a PDF made on request). -- Each stage's time, against research R7's budget. -- Each FR-011 check, and its result. +- Each stage's time, against research R7's budget: the preflight's and the parts' wall + time, and the figures' reads, the prints, the joins (with links, outline, stamps and + save) and the contents' second prints each summed over the slots, so above their wall + time where they overlap (T029). +- The join's settings: `--joins`, `--parallel` and whether object streams are on + (research R4). +- Each FR-011 check, and its result (none until `check-book`, T030). +- The book's faults, each as `preflight` prints it, and the error that stopped it, if any. - No names (FR-013). diff --git a/specs/001-pathway-pdfs/tasks.md b/specs/001-pathway-pdfs/tasks.md index a0f3c98..d50c11d 100644 --- a/specs/001-pathway-pdfs/tasks.md +++ b/specs/001-pathway-pdfs/tasks.md @@ -399,7 +399,7 @@ passes. - a substitution in an inlined figure leaves its elements, ids and attributes as they were; - the fixture summary's entry for an absent event is not a fault. - - **Done (#153, replacing #149 and #150 after their reviews):** the preflight is `src/preflight/`, its checks exported one by one for T029 and T046, and `pdf-export preflight` (`src/cli.ts`, `src/preflight/command.ts`) runs it alone with `book`'s flags, one `fault ` line each, exit 0, 1 or 2 (contracts/cli.md gains the command). `preflight.ts` reads the EHLD summary (its absence the one fault `missing-file ehld/svgsummary.txt`, as the Figure rules cannot be followed without it), builds the book (T048's `buildBook`) and runs, in order: the file names (`names.ts`: `/`, NUL, over 255 bytes with `.pdf`, counted in bytes, and every name of a group equal ignoring case); the release's files (`files.ts`: `diagram/{stId}.json` and `.graph.json` for a figure drawn from a diagram, its own or an ancestor's, `ehld/{stId}.svg` for an illustration, none for a reaction, from the Figure rules over the graph's events, so the summary's `R-HSA-99999999` is never looked for); the folder (`folder.ts`); and the characters before drawing (`characters.ts`). **The folder:** `folder.ts` holds the manifest's line formats for T046 (`manifestLine`, each a JSON object with a `type`: `fingerprint`, `figure`, `complete`; contracts/figures.md now gives them), the expected figures (`expectedFigures`, one per event with a figure, by its stable id, from the built book) and `checkFolder`: `no-checksum`; then `incomplete-folder` for no manifest, a torn last line, a line not a JSON object of its type, no fingerprint first, a fingerprint or `complete` anywhere else, no `complete` last, a `complete` differing from the first line, or two lines for one figure; `stale-folder` and `stale-summary` (the summary's hash is `sha256:` and the hex digest of its bytes); and only on a whole, current folder, each figure's line (`stale-figure`: none, or another kind, source or selection; or a line for an event with no figure) and file (`missing-file {stId}.svg`). A folder refused as a whole is not checked figure by figure, so a stale folder is a line, not thousands. **The checksum:** `releaseInfo` (`src/graph/queries.ts`, beside `release`, which is unchanged) reads `toString(d.checksum)`, as a long can pass 2^53, where the driver's numbers lose digits; the fixture's is 2^53 + 1, and read as a number it fails the reader's test. **Characters:** `src/layout/characters.ts` gains `BookCharacters`, the book's mode (appended, nothing before it changed): each substitution made and counted by code point (`replaced`, for T029's report: the preflight's own instance is discarded), and any other character no font draws a fault `uncovered-character `, kept as it is; `text`, `markupText` (numeric references too), `markup` and `strings` (a summation as markup), as the on-request mode has. `src/figures/labels.ts` gains `labelsInBook` (an SVG's text alone, after T014's prefixing, its fault naming the figure's event), for T029 and T046 after drawing. `src/preflight/characters.ts` checks before drawing: every string of each section (naming its event), each cover (its PDF's top-level pathway) and each short entry (its event); the names the reactions' figures label, from a new reader query, `figureNames` (each reaction's inputs, outputs, catalysts' and regulators' entities, by display name and names, and their compartments), each blamed on the first event in document order whose own figure is its reaction's; and the labels of the diagram files the Figure rules draw (`src/preflight/labels.ts`: the `displayName`s of `nodes`, `compartments`, `notes`, `shadows` and `links`), naming the diagram's pathway, an illustrated pathway's file and a non-human one's never read. A character in several places is a fault for each stable id it is blamed on (a top-level pathway's name in its sections' locations, too). A diagram file that is not JSON is a fault, `bad-file ` (from #149's review, below). `openGraph` gains `retryTimeMs` (the driver's `maxTransactionRetryTime`), 0 for the command's first read, which otherwise took 65 s to say a graph on a closed port could not be reached (its other reads keep the driver's default, from #149's review). **The fixture** gains `DBInfo.checksum` and `placeholder membrane`, a compartment only Reaction one's output holds (81 nodes, 128 relationships; reloaded once, as ci/fixture/README.md says, which lists both). **Tests:** `test/preflight/preflight.test.ts` (18 after #149's and #150's reviews, on the fixture, with a figures' folder each test writes and characters planted through a graph that rewrites its rows, so the shared fixture is untouched), `test/preflight/units.test.ts` (10, no graph), `test/graph/reader.test.ts` (2 more). 304 tests before, 329 after. Each of these mutants made one fail: a torn last line read; two lines for one figure allowed; `complete` not compared with the first line; other lines in the middle skipped (a second fingerprint, a `complete` in the middle); the checksum, the release or the summary not compared; `no-checksum` not raised; the selection, or the kind, not compared; a line for no figure allowed; figure files not looked for; the checksum read as a number; every summary entry looked for (the fixture's own run then fails); `.graph.json`, or illustrations, not looked for; participants read without their compartments (the membrane's test, and the reader's); the figures' names not checked; the last event blamed for a name; an illustration's diagram file read; diagram files not read; sections not checked; covers not checked (an undated edit's author, planted in the edits alone, is printed only on the cover, as a section's history leaves undated edits out); the substitutions not made, or not counted; numeric references not read; labels read as plain text (attributes then changed); a `/`, a NUL, characters counted for bytes, 256 bytes allowed, case not folded, and only the second of two equal names; the file names not checked by the preflight; faults exit 0; no retry bound (the 15 s bound); an unreachable graph not exit 2. Two survive as equivalent on any book: an ancestor's diagram file not read (an ancestor with a diagram draws its own figure from the same file, and is in the book above the pathway it draws), and the short entries' strings not checked (a short entry's name and location are printed again in its event's full section, at its first place in the same PDF). Left for T029 and T046: wiring `labelsInBook` after drawing, and the run report's counts. + - **Done (#153, replacing #149 and #150 after their reviews):** the preflight is `src/preflight/`, its checks exported one by one for T029 and T046, and `pdf-export preflight` (`src/cli.ts`, `src/preflight/command.ts`) runs it alone with `book`'s flags, one `fault ` line each, exit 0, 1 or 2 (contracts/cli.md gains the command). `preflight.ts` reads the EHLD summary (its absence the one fault `missing-file ehld/svgsummary.txt`, as the Figure rules cannot be followed without it), builds the book (T048's `buildBook`) and runs, in order: the file names (`names.ts`: `/`, NUL, over 255 bytes with `.pdf`, counted in bytes, and every name of a group equal ignoring case); the release's files (`files.ts`: `diagram/{stId}.json` and `.graph.json` for a figure drawn from a diagram, its own or an ancestor's, `ehld/{stId}.svg` for an illustration, none for a reaction, from the Figure rules over the graph's events, so the summary's `R-HSA-99999999` is never looked for); the folder (`folder.ts`); and the characters before drawing (`characters.ts`). **The folder:** `folder.ts` holds the manifest's line formats for T046 (`manifestLine`, each a JSON object with a `type`: `fingerprint`, `figure`, `complete`; contracts/figures.md now gives them), the expected figures (`expectedFigures`, one per event with a figure, by its stable id, from the built book) and `checkFolder`: `no-checksum`; then `incomplete-folder` for no manifest, a torn last line, a line not a JSON object of its type, no fingerprint first, a fingerprint or `complete` anywhere else, no `complete` last, a `complete` differing from the first line, or two lines for one figure; `stale-folder` and `stale-summary` (the summary's hash is `sha256:` and the hex digest of its bytes); and only on a whole, current folder, each figure's line (`stale-figure`: none, or another kind, source or selection; or a line for an event with no figure) and file (`missing-file {stId}.svg`). A folder refused as a whole is not checked figure by figure, so a stale folder is a line, not thousands. **The checksum:** `releaseInfo` (`src/graph/queries.ts`, beside `release`, which is unchanged) reads `toString(d.checksum)`, as a long can pass 2^53, where the driver's numbers lose digits; the fixture's is 2^53 + 1, and read as a number it fails the reader's test. **Characters:** `src/layout/characters.ts` gains `BookCharacters`, the book's mode (appended, nothing before it changed): each substitution made and counted by code point (`replaced`, for T029's report: the preflight's own instance is discarded), and any other character no font draws a fault `uncovered-character `, kept as it is; `text`, `markupText` (numeric references too), `markup` and `strings` (a summation as markup), as the on-request mode has. `src/figures/labels.ts` gains `labelsInBook` (an SVG's text alone, after T014's prefixing, its fault naming the figure's event), for T029 and T046 after drawing. `src/preflight/characters.ts` checks before drawing: every string of each section (naming its event), each cover (its PDF's top-level pathway) and each short entry (its event); the names the reactions' figures label, from a new reader query, `figureNames` (each reaction's inputs, outputs, catalysts' and regulators' entities, by display name and names, and their compartments), each blamed on the first event in document order whose own figure is its reaction's; and the labels of the diagram files the Figure rules draw (`src/preflight/labels.ts`: the `displayName`s of `nodes`, `compartments`, `notes`, `shadows` and `links`), naming the diagram's pathway, an illustrated pathway's file and a non-human one's never read. A character in several places is a fault for each stable id it is blamed on (a top-level pathway's name in its sections' locations, too). A diagram file that is not JSON is a fault, `bad-file ` (from #149's review, below). `openGraph` gains `retryTimeMs` (the driver's `maxTransactionRetryTime`), 0 for the command's first read, which otherwise took 65 s to say a graph on a closed port could not be reached (its other reads keep the driver's default, from #149's review). **The fixture** gains `DBInfo.checksum` and `placeholder membrane`, a compartment only Reaction one's output holds (81 nodes, 128 relationships; reloaded once, as ci/fixture/README.md says, which lists both). **Tests:** `test/preflight/preflight.test.ts` (18 after #149's and #150's reviews, on the fixture, with a figures' folder each test writes and characters planted through a graph that rewrites its rows, so the shared fixture is untouched), `test/preflight/units.test.ts` (10, no graph), `test/graph/reader.test.ts` (2 more). 322 tests before (#153's base, main with T027), 352 after, the reviews' tests included (362 on main once T028's were merged beside them). Each of these mutants made one fail: a torn last line read; two lines for one figure allowed; `complete` not compared with the first line; other lines in the middle skipped (a second fingerprint, a `complete` in the middle); the checksum, the release or the summary not compared; `no-checksum` not raised; the selection, or the kind, not compared; a line for no figure allowed; figure files not looked for; the checksum read as a number; every summary entry looked for (the fixture's own run then fails); `.graph.json`, or illustrations, not looked for; participants read without their compartments (the membrane's test, and the reader's); the figures' names not checked; the last event blamed for a name; an illustration's diagram file read; diagram files not read; sections not checked; covers not checked (an undated edit's author, planted in the edits alone, is printed only on the cover, as a section's history leaves undated edits out); the substitutions not made, or not counted; numeric references not read; labels read as plain text (attributes then changed); a `/`, a NUL, characters counted for bytes, 256 bytes allowed, case not folded, and only the second of two equal names; the file names not checked by the preflight; faults exit 0; no retry bound (the 15 s bound); an unreachable graph not exit 2. Two survive as equivalent on any book: an ancestor's diagram file not read (an ancestor with a diagram draws its own figure from the same file, and is in the book above the pathway it draws), and the short entries' strings not checked (a short entry's name and location are printed again in its event's full section, at its first place in the same PDF). Left for T029 and T046: wiring `labelsInBook` after drawing, and the run report's counts. - **From #149's review:** (1) an error during the preflight was exit 1 with no fault line, the faults already found lost (a diagram file not JSON, a graph failing after its retries, `buildBook`'s "not read from the graph"); exit 1 is to mean faults. A diagram file that cannot be read as JSON is now a fault, `bad-file ` (contracts/cli.md), its labels unchecked and the run going on; any other error ends the command with exit 2, its error on stderr, after the faults found before it are printed (`preflight` adds each check's faults to a list its caller holds). Both seen failing first: a corrupt `diagram/R-HSA-90000130.json` with an empty `--figures` gave exit 1 and nothing printed, and a graph failing at the reactions' names exit 1 with `incomplete-folder` lost. (2) `retryTimeMs: 0` still makes one retry, and it applied to every read: the start's read now has a driver of its own with it (said within seconds), and the checks' reads the driver's default; the comment and contract say "bounds the retrying". (3) The website's diagram draws a node's attachments' `label`s (a modification's lettering); `src/preflight/labels.ts` now reads them. `figureNames` does not read a reaction figure's equivalent (its entities' modified residues' PSI-MOD labels), recorded, not built: the fixture has no modification to test it on. Recorded too: a normal pathway's diagram drawn beneath a disease diagram (`forNormalDraw` false) is not read apart from its file. (4) Tests: a stale folder that also has a figure with no line and a missing file gives the one fault; a selection that is the right one's beginning is `stale-figure`; a figure line whose selection is not all strings is not whole. (5) Recorded, not changed: the fingerprint's `content` is not compared with the graph's release by `book`, defence in depth left to T046's check before drawing; "ignoring case" is `toLowerCase`, not full case folding or Unicode normalisation (final sigma, ß, NFC against NFD); the text check reads strings that are never printed (web addresses, and whatever the model holds beside what the layout writes), so a fault can be false; T047 needs the fingerprint compared and `none` told from `incomplete` (now in T047's text); a truncated `.graph.json` is not found, only its presence checked. 347 tests before (on main with T027), 350 after. Each of these mutants made one fail: the refusal as a whole removed (the stale folder's test); the selection's length not compared (the beginning's case); the selection's ids not checked as strings (the unit case); attachments not read; a file not JSON thrown again (the bad-file test); the error not caught (the exit 2 test); the faults found not printed on an error (the same); the start's read without its bound (the 15 s bound). Of the earlier mutants, 26 were run again on this code and each still fails a test; the other 12 (among them the file-name rule's, the substitutions' and the unreachable graph's) were not re-run; the two equivalents still survive. - **From #150's review:** the start's driver was opened outside the `try`, and the driver throws at once on an address it cannot parse, so `NEO4J_URI=127.0.0.1:17700` gave exit 1 with a stack trace; it is opened inside it now, exit 2 with the error on stderr (seen failing: `Unknown scheme: null`; tested only with addresses that fail to parse, never one falling back to the default port). On an error the faults are printed first, then the error (seen failing, `err` before `out`). The faults are kept check by check, as contracts/cli.md now says: those of the check an error stops are lost. A test runs the command as a child process and holds it to 15 s, so a driver left open is caught. contracts/cli.md adds `bad-file` to `book`'s preflight and to `figures`' exit 1, and says `book` exits 2 on an error during its preflight. Each of these mutants made one fail: the start's driver not closed (the child process's test); the checks' driver not closed (the same, and the test file's run then never ends); the start's driver opened outside the `try` (the address test); the error written before the faults (the exit 2 test's order). @@ -463,7 +463,7 @@ passes. - **Tests:** `test/print/contents.test.ts` (7) prints the fixture's book with the threshold at 1 page, as T025's to T027's do, and in `joined` saves each PDF as joined, reprints its front part, saves it, rewrites its links, writes its outline and stamps it: every contents number, read from the finished PDF's contents pages, is the page on which the entry's title is first a heading (found in the text, not from the probes) and that page's stamped number, the targets spread over the front part and at least two parts of sections, every number a placeholder `0` as joined; only the front part printed again, once, every other part once, the swapped PDF's pages as joined but for the front part's numbers; a reprint a page longer (a block pushing the introduction on) and one of the same length with its headings moved (its cover moved to its end) refused, naming the pages and the anchors, the PDF unchanged, and an entry with no probe, the parts not the PDF's (each way) and an anchor probed twice refused; the reprinted front part's contents links, matched by page and rectangle, landing on their headings, one for each entry, the outline the contents list, each entry's destination its link's, and no placeholder or probe address left in the file; the front pages stamped, the cover bare, `Introduction` and `Contents` and their numbers; an object the first print's front shares with a later page (planted, as parts share none) kept while its own annotations go; and no object left that nothing reaches, as joined, swapped and finished. 322 tests before, 330 after (8: these 7, and the stamping race below). Each of these mutants made one fail: no swap (the numbers, reprint and shared-object tests); the front part printed again without its numbers (the numbers and reprint tests); every number a page low (the numbers test); the first print's objects left in the file (the links test, by the placeholder address left, the shared-object and nothing-left tests); every object the old pages reach deleted, live or not (the shared-object test); moved pages not refused, and only the page count compared (the refusal test, each); entries with no probe not refused, the parts' pages unchecked, an anchor probed twice not refused (the refusal test, each); the reprinted pages inserted in reverse (every test, through the setup's run failing). In the shared-object test, an `assert.equal` of a found object against `undefined` never finished printing the object (the run ended with a promise pending), so it asserts with `assert.ok`. - **From #148's review:** the stamp-twice guard raced (`stamped.add` came after an `await`, so two stampings at once both passed); the PDF is now held once its stamps are checked, before the first `await`, and a test stamps it twice at once: one stamps, the other is refused, the pages as stamped once. `marginText` collapsed JavaScript's `\s`, which takes in a no-break, thin or ideographic space that Chromium keeps; it now collapses only space, tab, line feed and carriage return (and form feed, until #151's review, below), leaves out only controls and default-ignorable characters (FR-017's `NEVER_DRAWN` but its white space), trims only spaces, and is described as close to a margin box, not as one. A name that leaves no header (all white space and characters never drawn) is refused, naming it, not stamped bare. `ci/measure/join.ts` now embeds Roboto's WOFF file, as `stamp.ts` does, so its stamp stage draws outlines. T027's note is corrected (its test count, and which pages are rendered). Each of these mutants made one fail: the guard after the first `await` (the race test, and the stamp-twice test); `\s` collapsed, `trim()`, `NEVER_DRAWN` left out whole, and a form feed not collapsed, as it then was (`marginText`'s test, each); the empty name not refused (`marginText`'s test through `pageStamps`, and the refusal test). - **From #151's review:** collapsing only CSS's white space let an ideographic space, a line separator or an Ogham space mark (U+3000, U+2028, U+1680) through to the stamp's faces, none of which draws them, so a name holding one refused its whole PDF (`\s` had collapsed them before), where its sections print it and Chromium's margin box draws a space. `marginText` now keeps a space a stamp face draws (a no-break or thin space) and stamps any other as U+0020, each its own; a stamping test plants all three (`Alpha pathway one two` read back), which failed with "holds U+3000, which no font draws" before the fix. A form feed is not CSS's white space and Chromium does not collapse it: it is no longer collapsed but left out, as the control it is. A name refused as leaving no header is now one of white space alone, not only an empty one. Known gaps against Chromium, in `marginText`'s comment: the Hangul fillers U+3164 and U+115F are default-ignorable and left out, where Chromium may draw them as blanks; a kept space is as wide as U+0020. The stamped-twice guard was set before work that can still refuse (each face's file read, the space's face found), so a refusal there left the PDF held as stamped; those now come first, and `stampPages` takes each face's file from a function (`stampFile` by default) so a test can refuse one: the PDF is then stamped as if never tried. Recorded: the reprint's length cannot change through wrapping, as the numbers sit in boxes 4 em wide, so T028's refusal guards against the two prints loading fonts differently. 330 tests before, 332 after. Each of these mutants made one fail: the spaces no face draws passed through (the new stamping test, and `marginText`'s); every space mapped, a no-break space among them (`marginText`'s test); a form feed collapsed (the same); only an empty header refused (the same, through `pageStamps`); the guard set before the files are read (the refused-file test). -- [ ] T029 [US1] `book` end to end, `src/cli.ts` (contracts/cli.md). +- [x] T029 [US1] `book` end to end, `src/cli.ts` (contracts/cli.md). - **Flags and their environment variables** (contracts/cli.md): `--graph` (`NEO4J_URI`), `--database` (`NEO4J_DATABASE`), `--figures`, `--site` (`REACTOME_SITE`), `--files`, `--species`, `--out`, `--parallel`, `--joins` and `--report`. Credentials come from `NEO4J_USER` and `NEO4J_PASSWORD`, never the command line. - **Writing:** `--out` must be absent or empty (exit 2); the folder is written beside its name and moved into place; each PDF is named as today's (the pathway's name and `.pdf`). - **Memory (research R4, T011):** the join's memory, object streams on and off, measured on the real book; the join limit's default (1, raised to 2 only if two real joins fit beside printing), given a flag `--joins` in contracts/cli.md, and the object streams' setting chosen from it; if one real join does not fit beside printing, T012 is done before T029 closes. @@ -480,6 +480,13 @@ passes. - the run report lists the characters replaced, by code point and count. The test plants them itself, in a copy of the fixture: one listed character in a summation, one in the name of Reaction eight (`R-HSA-90000132`, at one place, with no events before or after it and no children, so its name prints only in its heading and its contents entry), and, in a figures' folder the test writes itself (a valid manifest, `complete`), one in the label of Reaction eight's own reaction figure (drawn once; a pathway's diagram can be drawn again for its subpathways). The counts are of occurrences in the finished book: the summation's 1, the name's 2, the label's 1, each plant a different one of the list's characters. What is not printed text is not counted: the bookmark's title (T026) and the figure's `aria-label`, which an untagged part does not carry. A count of the preflight's substitutions too, or of the front part's first print (T028), fails it; and the fourth listed character, planted in a top-level pathway's name, is counted where that name prints as text (its heading, its contents entry and the locations beneath it) and not for its stamped headers (T027), seen failing with the stamps counted; - with T025's threshold lowered and, in a figures' folder the test writes itself with a valid manifest, a character off the list planted in one SVG of a later part (the backstop for a folder changed by hand; `figures` would have refused it): exit 1 after earlier parts are printed, and no folder, PDF, temporary file or part file left. + - **Done (#156):** `pdf-export book` is `src/book/`: `command.ts` (the flags and their variables, exit codes, the folder, the run report), `book.ts` (the printing) and `figures.ts` (the figures' folder), wired into `src/cli.ts` by one branch. **The run:** the arguments (`--parallel` and `--joins` whole numbers from 1 to 64, `--joins` at most `--parallel`; `--site` or `REACTOME_SITE` and `--files` and `--figures` required; `--out` `TheReactomeBook` by default); `--out` absent or an empty folder, and its parent a folder, or exit 2; the graph's release read by a driver of its own with bounded retries, as `preflight` reads it (exit 2 unreachable or unparsable); the preflight (T024), its faults exit 1 and any error exit 2 after the faults; then the PDFs printed into `.-XXXXXX` beside `--out`, which is renamed into place only once every PDF is written (onto an empty folder too). Any failure removes that folder, so no folder, PDF or part is left; the printer's pages' folder goes with the printer. A run stopped by SIGINT, SIGTERM or SIGHUP removes the folder and exits 128 plus the signal's number, so the exit handlers run: `Printer` now removes its pages' folder on the process's exit too, registered before the browser is launched (found when a test run killed for memory left them, below). **Each joined PDF,** in `printBook`'s `joined`, in T029's order: `reprintFront`, `rewriteLinks`, `writeOutline`, `stampPages`, then `save({ useObjectStreams: false })` (`OBJECT_STREAMS`) and written as its pathway's name, as the graph has it, plus `.pdf`. A throw anywhere there fails the run and the PDF is never saved (contracts/cli.md's exit 2 now says so: an error while printing, a refused PDF among them, is exit 2, faults exit 1). **Figures** are read from `--figures` only (`FolderFigures`): each `{stId}.svg`, named for the first event in document order with that figure's key, read when the layout asks for it, never held, with `labelsInBook` applied; no render setting is taken. `LayoutOptions.figures` is now `FigureSvgs` (a `get` alone, `src/layout/figure.ts`), which a `Map` still is, so the folder is not read into memory; two tests type their options as maps. **The label check after drawing:** `printBook` gains `laying`, called in a part's slot before the part is laid out (the front part with `null`); `book` checks there each figure the part draws (the cover's for the front part), a fault (`uncovered-character `, or `missing-file .svg` for a file gone since the preflight) failing the run before that part prints, after earlier parts have. It is not done when the figures are first read, as `printBook` measures every figure before it prints anything. **Characters (FR-017):** counted in what is printed, once each time it is printed: `printedDocument` makes the substitutions in each text the layout prints and counts them in one `BookCharacters` (not the preflight's): the cover's title and names, the contents' titles (which the outline reuses, uncounted), each full section's printed fields, and each short entry's name; a short entry's location, which is not printed, and fields never printed (species, a reference's address) are left as they are. Each figure's substitutions are counted for each drawing, in `laying`, so a pathway's figure on its cover and in its section counts twice; T028's second print of the front part is not laid out through `laying`, so is not counted. Decided: the cover's title is printed text and is counted there, beside the heading, contents entry and locations T029 names; the stamped headers are not (T027). **The run report** (`--report`, JSON, written beside its name and moved into place however the run ends once its arguments are read): the release and checksum; the outcome, faults and error; the time; each stage against R7's budget (the preflight's and the parts' wall time; the figures' reads, the prints, the joins with links, outline, stamps and save, and the contents' second prints summed over the slots); each PDF's file, stable identifier, pages and bytes, and the totals; `--joins`, `--parallel` and object streams; figures by kind, missing and stale (by event), below 1 pt and unmeasured; release files missing; characters replaced by code point and those uncovered; and `checks: null` until `check-book` (T030). Exit 0 is provisional until T030 runs the check. + - **Memory: not done here.** The join's memory, object streams on and off, needs the real book (release 98's graph), which is not this task's: it is left to T031, with whether `--joins` goes to 2 and T012 is needed. Until then the defaults stay `--joins 1` and object streams off. + - **Folded in from earlier reviews:** `package.json`'s `test` runs with `--test-timeout=300000`, so a leaked handle fails the run instead of hanging it, and `--test-concurrency=4`: with the book's tests, the suite's peak memory (page cache included) was 3,143 MiB at the default concurrency (7 on this host), over a 3 GiB cap that killed it twice (main's was 2,678–2,729 MiB); at 4 it was 1,962 MiB, at 5 2,311, the run as long (73 s), as the longest file sets it. `src/preflight/command.ts` opens the checks' graph inside the `try`, and closes both graphs through `closing`, which says a failure to close on stderr and never replaces the exit code. `src/print/stamp.ts`: a space mapped to U+0020 is not trimmed at either end, as Chromium's box keeps it (`marginText`'s test gains `['　a', ' a']`, its comment why); the doc says a PDF is held as stamped once its checks pass, so a failure while embedding leaves it held (`book` discards it); the unreachable `?? load(...)` is gone; the comment on a kept space's width says which spaces are stamped at U+0020's width. T024's stale test count is corrected. + - **Tests:** `test/book/book.test.ts` (8, on the fixture, with a figures' folder each test writes from `ci/fixture/figures/`, names and characters planted through a graph that rewrites its rows, each run's TMPDIR a folder of its own, checked empty after): exit 0, one PDF per top-level pathway and nothing else in the parent, no request to a render stub named in `RENDER_SERVICE`, each PDF's bytes and pages as reported, no placeholder or probe left, no object stream, an outline, its last page's number stamped, and its contents numbered (every number past the cover), the report's figures by kind, stages and join settings; exit 1 on a figure file missing, nothing left; exit 2 on an `--out` not empty or a file, left as it was; exit 2 on bad arguments (an unknown flag, `--render` among them, `--figures` or `--site` missing, `--parallel 0`, `--joins` over `--parallel`, a flag twice, no graph) and on a graph that cannot be read at the start (an address that cannot be parsed, never one that could reach another database, and reads that fail); the characters: a summation's U+FB01 (1), Reaction eight's name's U+2011 (2), U+02D9 in Reaction eight's figure (1) and in beta's own figure (cover and section, 2), and U+05F3 in Alpha pathway's name, counted as often as its prime is printed outside the stamps (11), the PDF named with U+05F3; U+E000 in Reaction eight's figure with the threshold at 1 page and one slot: exit 1 after at least three parts printed, the fault line, nothing left in the parent or the run's TMPDIR; a PDF refused at its stamps (Gamma's name planted as a space, a joiner and a space): exit 2, nothing left; and a run sent SIGTERM while printing, as a child process: nothing left in the parent or its TMPDIR. `test/preflight/preflight.test.ts` (1 more): the checks' graph throwing on open is exit 2, and graphs rejecting on close keep exit 0 and 1, each failure said. 400 tests before (main with T046), 409 after. Each of these mutants made one fail: `reprintFront` left out (the contents' numbers), `rewriteLinks` before it, no outline, no stamps (the exit-0 test, each); object streams on (the same); no move into place (the same); the folder not removed on failure (the label and refused tests); `--out` not checked; no `laying` (the label test, exit 0); a label's fault as an error (the same, exit 2); the preflight's faults ignored (the missing-file test); the stamped headers counted, the preflight's text substitutions counted too, the cover's figure counted again at the reprint, a figure counted once however often drawn, short entries' locations counted, and the cover's title, the contents' titles or the locations not counted (the characters test, each); the summations not substituted (the same); the PDF named for its printed name (the same); an unreachable graph exit 1, the start's driver opened outside the `try`, and `--joins` over `--parallel` allowed (the bad-arguments test, each); a refused PDF's stamps skipped and the PDF saved (the refused test); no signal handler, the printer's exit hook removed, and the handler re-raising the signal instead of exiting (the signal test, each); in `preflight`, the checks' graph opened outside the `try`, and either close unguarded (the new preflight test, each); a mapped space trimmed in `marginText` (its test). + - **A test run killed for memory leaves its folders:** the first runs of the full suite under the 3 GiB cap were killed (systemd's OOM policy stops the whole scope), and each left its test folders and the printer's pages' folders in the temporary directory, as `after` and `close` never ran. The book's tests now put every folder of theirs under one folder per file, and `Printer` removes its own on exit; a process killed outright (SIGKILL) still leaves them. + - **Known gaps:** a file's label check names the first event with that figure; a refused PDF's error does not say which PDF unless its message does (stamping's names the name); a run killed outright leaves `.-XXXXXX` beside `--out`, which a later run ignores and does not remove. + - [ ] T030 [US1] `check-book`, `src/check/book.ts` (FR-011, contracts/cli.md). - **Flags:** `--graph`, `--species` and `--files`. - **Output:** one line per fault, in contracts/cli.md's three forms: `fault missing-pathway `, `fault extra-file ""`, and `fault "" ` for the rest. @@ -492,6 +499,7 @@ passes. - [ ] T031 [US1] The book on a private copy of release 98's graph. - Run `figures` first and time it for SC-010; then time each of `book`'s stages against research R7's budget, check memory, and run `check-book`. Record the results in research R7. + - T029's memory measurement, left here: the join's memory on the real book, object streams on and off, beside printing; raise `--joins` to 2 only if two real joins fit, choose object streams from it (off until then), and do T012 first if one real join does not fit. - Open every one of the book's PDFs in the three readers of SC-004, and follow a sample of bookmarks and links in each, the largest PDF's among them. - T026's test (`landing()` in `test/print/links.test.ts`) finds a link's heading only as one text item equal to the whole title; a long name that wraps, or that pdf.js splits, would not match. On the real book, check the landings another way (the heading's first line, or the probe's anchor beside the text), and loosen the test if it is brittle there. diff --git a/src/book/book.ts b/src/book/book.ts new file mode 100644 index 0000000..3d08e1b --- /dev/null +++ b/src/book/book.ts @@ -0,0 +1,232 @@ +// The book printed and written (spec 001 T029, contracts/cli.md `book`, research R4): each +// of the book's PDFs, as the preflight built it, with FR-017's substitutions made in what it +// prints, printed in parts (T025), and each joined PDF then, in its join's slot and in this +// order, given its contents' numbers (T028's `reprintFront`, which reads the probes), its +// links (T026's `rewriteLinks`, which removes them), its outline (`writeOutline`) and its +// stamps (T027's `stampPages`), and saved with object streams off into the folder the +// command moves into place at the end. A PDF refused at any of these is never saved: those +// steps change it in place and do not undo a part done, so the run fails whole. +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { pageStyles, type LayoutOptions, type Report } from '../layout/document.ts'; +import type { BookCharacters } from '../layout/characters.ts'; +import { figureKey } from '../layout/figure.ts'; +import type { Document, Figure, Place } from '../model/types.ts'; +import { fileName } from '../preflight/names.ts'; +import { bookPathways } from '../preflight/preflight.ts'; +import { reprintFront } from '../print/contents.ts'; +import { rewriteLinks } from '../print/links.ts'; +import { writeOutline } from '../print/outline.ts'; +import { joinParts, printBook, type PartPrinter, type PartsSettings } from '../print/parts.ts'; +import { stampPages } from '../print/stamp.ts'; +import { FolderFigures } from './figures.ts'; + +/** + * pdf-lib's object streams, off unless memory binds (research R4, T011): on synthetic files + * a save took five to eight times as long with them on, for files 6% smaller, and memory + * peaked lower, perhaps only because slow saves overlapped less. The real book's join is + * measured with both in T031, which settles this, as it settles `--joins`' default of 1. + */ +export const OBJECT_STREAMS = false; + +/** Faults found while the book is printed, each a line as the preflight's are. */ +export class BookFaults extends Error { + override name = 'BookFaults'; + readonly faults: readonly string[]; + + constructor(faults: readonly string[]) { + super(`The book's faults: ${faults.join(', ')}`); + this.faults = faults; + } +} + +/** + * A book's PDF as it is printed: FR-017's substitutions made in every text the layout + * prints, and counted in `characters` once for each place it is printed, so the counts are + * the finished book's: a name printed in a heading and a contents entry counts twice, and + * once only where one string of the model is printed once and read again unprinted (the + * outline's titles, a figure's label for readers, which an untagged part does not carry). + * The cover's title is printed on the cover and counted there; the running headers stamped + * from it are not counted (T027). What the model holds and the layout does not print (a + * short entry's location, an event's species, a reference's address) is left as it is. + * A character no font draws is a fault in `characters`, as the preflight found it. + */ +export function printedDocument(doc: Document, characters: BookCharacters): Document { + const root = bookPathways([doc])[0]?.stId ?? ''; + const text = (value: string, subject: string) => characters.text(value, subject); + return { + ...doc, + cover: { + ...doc.cover, + title: text(doc.cover.title, root), + names: doc.cover.names.map((name) => text(name, root)), + }, + contents: doc.contents?.map((entry) => ({ ...entry, title: text(entry.title, root) })) ?? null, + places: doc.places.map((place): Place => { + if (!place.full) { + const { event } = place; + return { ...place, event: { ...event, name: text(event.name, event.stId) } }; + } + const { section } = place; + const { event } = section; + const s = event.stId; + return { + full: true, + section: { + ...section, + event: { + ...event, + name: text(event.name, s), + category: event.category === null ? null : text(event.category, s), + compartments: characters.strings(event.compartments, s), + before: characters.strings(event.before, s), + after: characters.strings(event.after, s), + }, + path: characters.strings(section.path, s), + diseases: characters.strings(section.diseases, s), + inferredFrom: characters.strings(section.inferredFrom, s), + summations: section.summations.map((summation) => characters.markup(summation, s)), + // a reference's address is a link's, not printed + references: section.references.map((r) => ({ + ...characters.strings({ ...r, url: null }, s), + url: r.url, + })), + edits: characters.strings(section.edits, s), + }, + }; + }), + }; +} + +/** One PDF written: its top-level pathway, its file, its pages and bytes. */ +export interface PdfWritten { + stId: string; + file: string; + pages: number; + bytes: number; +} + +/** What writing the book gives the run report. */ +export interface Written { + /** In the book's order (FR-002). */ + pdfs: PdfWritten[]; + /** The layout's report over every part, by stable identifier, each once. */ + layout: Report; + /** FR-017's substitutions made in the finished book, by code point. */ + replaced: Map; + /** Milliseconds, summed over the slots: so above the wall time where they overlap. */ + ms: { figures: number; print: number; contents: number; join: number; parts: number }; +} + +export interface WriteSettings { + /** The folder `figures` wrote. */ + figures: string; + site: string; + /** Where the PDFs are written: the folder the command moves into place. */ + folder: string; + parts: PartsSettings; + printer: PartPrinter; + date: Date; +} + +/** + * The book's PDFs, printed and written into `settings.folder`, from its documents as the + * preflight built them; `characters` counts the substitutions in what is printed. Throws a + * `BookFaults` for a character no font draws in the texts (which the preflight has + * checked), and printBook's AggregateError for anything that stops the run, among them a + * `BookFaults` for a figure's labels; what was written is then the caller's to remove. + */ +export async function writeBook( + docs: readonly Document[], + characters: BookCharacters, + settings: WriteSettings +): Promise { + const files = new Map(); + const printed = docs.map((doc) => { + const done = printedDocument(doc, characters); + const [pathway] = bookPathways([doc]); + if (!pathway) throw new Error('A PDF of the book holds no pathway'); + files.set(done, { stId: pathway.stId, file: fileName(pathway.name), pages: 0, bytes: 0 }); + return done; + }); + if (characters.faults.length > 0) throw new BookFaults(characters.faults); + const figures = new FolderFigures(settings.figures, printed); + const layout: LayoutOptions = { + site: settings.site, + date: settings.date, + figures, + styles: pageStyles(), + }; + const replaced = new Map(characters.replaced); + const ms = { figures: 0, print: 0, contents: 0, join: 0, parts: 0 }; + const reports: Report[] = []; + const timed = async (stage: keyof typeof ms, work: () => Promise): Promise => { + const started = performance.now(); + try { + return await work(); + } finally { + ms[stage] += performance.now() - started; + } + }; + // the parts' prints timed; the front part's second print is the contents' stage + const printer: PartPrinter = { + part: (html) => timed('print', () => settings.printer.part(html)), + }; + const started = performance.now(); + await printBook(printed, { + printer, + layout, + settings: settings.parts, + join: (parts) => timed('join', () => joinParts(parts)), + // each figure a part draws checked before the part is laid out, and its substitutions + // counted for each drawing, the cover's among them; the front part's second print + // (T028) is not laid out here, so nothing in it is counted twice + laying: (doc, places) => { + const drawn: (Figure | null)[] = + places === null + ? [doc.cover.figure] + : places.map((place) => (place.full ? place.section.figure : null)); + const checks = drawn.flatMap((figure) => (figure ? [figures.check(figureKey(figure))] : [])); + const faults = [...new Set(checks.flatMap((check) => check.faults))]; + if (faults.length > 0) throw new BookFaults(faults); + for (const check of checks) { + for (const [point, count] of check.replaced) { + replaced.set(point, (replaced.get(point) ?? 0) + count); + } + } + }, + joined: async (joined) => { + const { doc, pdf } = joined; + const written = files.get(doc); + if (!written) throw new Error('A joined PDF that is not one of the book'); + await timed('contents', () => reprintFront(joined, settings.printer, layout)); + const bytes = await timed('join', async () => { + const targets = rewriteLinks(pdf); + writeOutline(pdf, doc.contents ?? [], targets); + await stampPages(joined, targets); + return pdf.save({ useObjectStreams: OBJECT_STREAMS }); + }); + writeFileSync(join(settings.folder, written.file), bytes); + written.pages = pdf.getPageCount(); + written.bytes = bytes.length; + reports.push(joined.report); + }, + }); + ms.parts = performance.now() - started; + ms.figures = figures.ms; + const each = (list: (r: Report) => string[]) => [...new Set(reports.flatMap(list))].sort(); + return { + pdfs: printed.flatMap((doc) => { + const written = files.get(doc); + return written ? [written] : []; + }), + layout: { + belowFloor: each((r) => r.belowFloor), + missing: each((r) => r.missing), + unmeasured: each((r) => r.unmeasured), + }, + replaced, + ms, + }; +} diff --git a/src/book/command.ts b/src/book/command.ts new file mode 100644 index 0000000..4b8a6a3 --- /dev/null +++ b/src/book/command.ts @@ -0,0 +1,416 @@ +// `pdf-export book` (spec 001 T029, contracts/cli.md): the release's book, one PDF per +// top-level pathway, from the graph, the release's files and the figures' folder `figures` +// drew; it asks no render service. The preflight (T024) runs first; then the PDFs are +// printed into a folder beside `--out`'s name, moved into place only once every PDF is +// written, so a failed run leaves no book, part or temporary folder. Exit 0 with the book +// written, 1 on faults (the preflight's, or a figure's labels before its part is printed), +// 2 on bad arguments, an `--out` that is not absent or empty, a graph that cannot be read +// at the start, or any other error, after printing the faults found before it. The run +// report (data-model.md) is written to `--report` however the run ends, once its arguments +// are read. FR-011's check (`check-book`, T030) is not yet run: exit 0 is provisional. +import { + existsSync, + mkdtempSync, + readdirSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { constants as osConstants } from 'node:os'; +import { basename, dirname, join, resolve } from 'node:path'; + +import { graphSettings, openGraph, type Graph, type GraphSettings } from '../graph/connection.ts'; +import { releaseInfo, type ReleaseInfo } from '../graph/queries.ts'; +import { BookCharacters } from '../layout/characters.ts'; +import { codePointName } from '../layout/fonts.ts'; +import type { Figure } from '../model/types.ts'; +import { closing, message, SettingsError } from '../preflight/command.ts'; +import { expectedFigures } from '../preflight/folder.ts'; +import { preflight } from '../preflight/preflight.ts'; +import { PARTS, type PartPrinter } from '../print/parts.ts'; +import { BookFaults, OBJECT_STREAMS, writeBook, type PdfWritten, type Written } from './book.ts'; + +export interface BookSettings { + graph: GraphSettings; + species: string; + files: string; + figures: string; + site: string; + out: string; + parallel: number; + joins: number; + report: string | null; +} + +const FLAGS = [ + '--graph', + '--database', + '--figures', + '--site', + '--files', + '--species', + '--out', + '--parallel', + '--joins', + '--report', +]; + +/** + * The settings from `book`'s flags and the environment (contracts/cli.md): a flag wins over + * its variable; the graph's credentials come from the environment only. + */ +export function bookSettings( + args: readonly string[], + env: Readonly> +): BookSettings { + const flags = new Map(); + for (let i = 0; i < args.length; i++) { + const [name = '', inline] = (args[i] ?? '').split(/=(.*)/s, 2); + if (!FLAGS.includes(name)) throw new SettingsError(`Unknown argument: ${name}`); + const value = inline ?? args[++i]; + if (value === undefined || value === '') throw new SettingsError(`${name} needs a value`); + if (flags.has(name)) throw new SettingsError(`${name} is given twice`); + flags.set(name, value); + } + const setting = (flag: string, variable?: string) => + flags.get(flag) ?? (variable === undefined ? undefined : env[variable] || undefined); + const required = (flag: string, variable?: string) => { + const value = setting(flag, variable); + if (value === undefined) { + throw new SettingsError(`${flag}${variable ? ` (or ${variable})` : ''} is needed`); + } + return value; + }; + const whole = (flag: string, fallback: number) => { + const value = flags.get(flag) ?? String(fallback); + if (!/^[0-9]+$/.test(value) || Number(value) < 1 || Number(value) > 64) { + throw new SettingsError(`${flag} must be a whole number from 1 to 64`); + } + return Number(value); + }; + const graphEnv = { ...env }; + const uri = setting('--graph'); + if (uri !== undefined) graphEnv['NEO4J_URI'] = uri; + const database = setting('--database'); + if (database !== undefined) graphEnv['NEO4J_DATABASE'] = database; + let graph: GraphSettings; + try { + graph = graphSettings(graphEnv); + } catch { + throw new SettingsError('--graph (or NEO4J_URI) is needed'); + } + const parallel = whole('--parallel', PARTS.parallel); + const joins = whole('--joins', PARTS.joins); + // a join holds one of --parallel's slots, so more joins than slots is never reached + if (joins > parallel) throw new SettingsError('--joins cannot be more than --parallel'); + return { + graph, + species: setting('--species') ?? 'Homo sapiens', + files: required('--files'), + figures: required('--figures'), + site: required('--site', 'REACTOME_SITE'), + out: setting('--out') ?? 'TheReactomeBook', + parallel, + joins, + report: setting('--report') ?? null, + }; +} + +/** Why `--out` cannot be written, or null: it must be absent or an empty folder. */ +export function outProblem(out: string): string | null { + const parent = dirname(resolve(out)); + if (!existsSync(parent) || !statSync(parent).isDirectory()) { + return `--out ${out}: its folder ${parent} does not exist`; + } + if (!existsSync(out)) return null; + if (!statSync(out).isDirectory()) return `--out ${out} is not a folder`; + if (readdirSync(out).length > 0) { + return `--out ${out} is not empty: an old book is never replaced in place`; + } + return null; +} + +/** A stage's time against research R7's budget, in seconds. */ +export interface StageTime { + stage: string; + /** Null for a stage not run. */ + seconds: number | null; + budget: number | null; + /** Wall time, or the time summed over the slots, above the wall time where they overlap. */ + measured: 'wall' | 'summed'; +} + +/** The run report (data-model.md). Everything is named by stable identifier or file. */ +export interface RunReport { + release: number | null; + checksum: string | null; + outcome: 'written' | 'faults' | 'error'; + /** The faults, each a line as `preflight` prints it: the preflight's, then the printing's. */ + faults: string[]; + /** The error that stopped the run, if one did. */ + error: string | null; + seconds: number; + stages: StageTime[]; + pdfs: PdfWritten[]; + total: { pdfs: number; pages: number; bytes: number }; + /** The settings that shape the join's memory (research R4). */ + joins: { joins: number; parallel: number; objectStreams: boolean }; + figures: { + byKind: Record; + /** Missing from the figures' folder, or stale there, by the event's stable identifier. */ + missing: string[]; + stale: string[]; + /** Whose labels print below 1 pt even at their largest (research R4). */ + belowFloor: string[]; + unmeasured: string[]; + }; + /** The release's files missing, by name as in `--files`. */ + missingFiles: string[]; + characters: { + /** FR-017's substitutions in the finished book, by code point. */ + replaced: Record; + /** Characters no font draws, each as its fault: code point and stable identifier. */ + uncovered: string[]; + }; + /** FR-011's checks, each and its result: `check-book` (T030), not yet run. */ + checks: null; +} + +/** A printer that can be closed: the Printer (src/print/print.ts). */ +export type BookPrinter = PartPrinter & { close(): Promise }; + +export interface BookIo { + out: (text: string) => void; + err: (text: string) => void; + /** How the graph is opened (a test gives one that fails or rewrites rows). */ + open?: typeof openGraph; + /** How the printer is started (a test counts its prints). */ + startPrinter?: () => Promise; + /** The pages at which a part ends (T025): lowered by the tests, so a PDF splits. */ + threshold?: number; + /** The covers' date. */ + date?: Date; +} + +/** `pdf-export book`: its exit code; faults to `out`, errors to `err`. */ +export async function bookCommand( + args: readonly string[], + env: Readonly>, + io: BookIo +): Promise { + const { out, err, open = openGraph } = io; + let settings: BookSettings; + try { + settings = bookSettings(args, env); + } catch (error) { + if (!(error instanceof SettingsError)) throw error; + err(`${error.message}\n`); + return 2; + } + const started = performance.now(); + const report = emptyReport(settings); + const end = (code: number, error: unknown = null): number => { + for (const fault of report.faults) out(`fault ${fault}\n`); + if (error !== null) { + report.error = message(error); + err(`The book was not written: ${report.error}\n`); + } + report.outcome = code === 0 ? 'written' : code === 1 ? 'faults' : 'error'; + report.seconds = seconds(performance.now() - started); + if (settings.report !== null) { + try { + writeAtomically(settings.report, `${JSON.stringify(report, null, 2)}\n`); + } catch (failure) { + err(`The run report was not written: ${message(failure)}\n`); + return 2; + } + } + return code; + }; + + const problem = outProblem(settings.out); + if (problem) return end(2, problem); + + // the start's check, its own driver retrying briefly, as `preflight`'s (T024) + let start: Graph | undefined; + let release: ReleaseInfo; + try { + start = open(settings.graph, { connectionTimeoutMs: 5000, retryTimeMs: 0 }); + release = await releaseInfo(start); + } catch (error) { + await closing(start, err); + return end(2, `the graph cannot be read: ${message(error)}`); + } + await closing(start, err); + report.release = release.release; + report.checksum = release.checksum; + + // the preflight, with the driver's own retries; its faults kept check by check + const preflightStarted = performance.now(); + const faults: string[] = []; + let docs; + let stopped: { error: unknown } | null = null; + let graph: Graph | undefined; + try { + graph = open(settings.graph); + docs = (await preflight(graph, release, settings, faults)).docs; + } catch (error) { + stopped = { error }; + } finally { + await closing(graph, err); + } + report.stages.push({ + stage: 'preflight', + seconds: seconds(performance.now() - preflightStarted), + budget: 4 * 60, + measured: 'wall', + }); + noteFaults(report, faults); + if (stopped) return end(2, `the preflight stopped: ${message(stopped.error)}`); + if (faults.length > 0 || !docs) return end(1); + for (const record of expectedFigures(docs).values()) report.figures.byKind[record.kind] += 1; + + // the book, printed into a folder beside its name and moved into place at the end + const target = resolve(settings.out); + const folder = mkdtempSync(join(dirname(target), `.${basename(target)}-`)); + let printer: BookPrinter | undefined; + // a run stopped by a signal removes its folder too, and exits as one stopped by it does + // (128 and its number), so the exit handlers run: the printer's, which removes its pages' + // folder (src/print/print.ts), and the browser's launcher's, which ends the browser + const stop = (signal: NodeJS.Signals) => { + rmSync(folder, { recursive: true, force: true }); + process.exit(128 + osConstants.signals[signal]); + }; + const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'] as const; + for (const signal of signals) process.once(signal, stop); + let written: Written | undefined; + let failure: unknown = null; + const characters = new BookCharacters(); + try { + printer = await (io.startPrinter ?? startPrinter)(); + written = await writeBook(docs, characters, { + figures: settings.figures, + site: settings.site, + folder, + parts: { + threshold: io.threshold ?? PARTS.threshold, + parallel: settings.parallel, + joins: settings.joins, + }, + printer, + date: io.date ?? new Date(), + }); + renameSync(folder, target); + } catch (error) { + failure = error; + } finally { + try { + await printer?.close(); + } catch (error) { + err(`The printer did not close: ${message(error)}\n`); + } + if (failure !== null) rmSync(folder, { recursive: true, force: true }); + for (const signal of signals) process.off(signal, stop); + } + report.characters.replaced = counts(written?.replaced ?? characters.replaced); + if (failure !== null) { + // faults found while printing are exit 1, any other error exit 2, the faults printed first + const failures = failure instanceof AggregateError ? (failure.errors as unknown[]) : [failure]; + const others = failures.filter((f) => !(f instanceof BookFaults)); + for (const f of failures) if (f instanceof BookFaults) noteFaults(report, [...f.faults]); + if (others.length > 0) return end(2, others.map(message).join('; ')); + return end(1); + } + if (!written) return end(2, 'nothing was written'); + report.pdfs = written.pdfs; + report.total = { + pdfs: written.pdfs.length, + pages: written.pdfs.reduce((n, pdf) => n + pdf.pages, 0), + bytes: written.pdfs.reduce((n, pdf) => n + pdf.bytes, 0), + }; + report.figures.belowFloor = written.layout.belowFloor; + report.figures.unmeasured = written.layout.unmeasured; + report.figures.missing = [...new Set([...report.figures.missing, ...written.layout.missing])]; + const { ms } = written; + report.stages.push( + { stage: 'figures', seconds: seconds(ms.figures), budget: 44 * 60, measured: 'summed' }, + { stage: 'print', seconds: seconds(ms.print), budget: 13 * 60, measured: 'summed' }, + { stage: 'join', seconds: seconds(ms.join), budget: 6 * 60, measured: 'summed' }, + { stage: 'contents', seconds: seconds(ms.contents), budget: 2 * 60, measured: 'summed' }, + { stage: 'parts', seconds: seconds(ms.parts), budget: null, measured: 'wall' }, + { stage: 'check-book', seconds: null, budget: 9 * 60, measured: 'wall' } + ); + err( + `The book: ${String(report.total.pdfs)} PDFs, ${String(report.total.pages)} pages, in ${settings.out}\n` + ); + return end(0); +} + +const startPrinter = async (): Promise => { + // loaded only here: it starts a browser + const { Printer } = await import('../print/print.ts'); + return Printer.start(); +}; + +const seconds = (ms: number): number => Math.round(ms) / 1000; + +function emptyReport(settings: BookSettings): RunReport { + return { + release: null, + checksum: null, + outcome: 'error', + faults: [], + error: null, + seconds: 0, + stages: [], + pdfs: [], + total: { pdfs: 0, pages: 0, bytes: 0 }, + joins: { joins: settings.joins, parallel: settings.parallel, objectStreams: OBJECT_STREAMS }, + figures: { + byKind: { diagram: 0, ancestor: 0, ehld: 0, reaction: 0 }, + missing: [], + stale: [], + belowFloor: [], + unmeasured: [], + }, + missingFiles: [], + characters: { replaced: {}, uncovered: [] }, + checks: null, + }; +} + +/** Faults added to the report, and sorted into its lists by kind. */ +function noteFaults(report: RunReport, faults: readonly string[]): void { + for (const fault of faults) { + report.faults.push(fault); + const [kind = '', name = ''] = fault.split(' '); + if (kind === 'missing-file') { + // a figure's file is named for its event; a release file by its folder + if (name.includes('/')) report.missingFiles.push(name); + else report.figures.missing.push(name.replace(/\.svg$/, '')); + } else if (kind === 'stale-figure') { + report.figures.stale.push(name); + } else if (kind === 'uncovered-character') { + report.characters.uncovered.push(fault.slice(kind.length + 1)); + } + } +} + +/** Counts by code point, `U+XXXX`, in code point order. */ +function counts(replaced: ReadonlyMap): Record { + return Object.fromEntries( + [...replaced].sort(([a], [b]) => a - b).map(([point, n]) => [codePointName(point), n]) + ); +} + +/** A file written beside its name and moved into place. */ +function writeAtomically(path: string, text: string): void { + const beside = `${path}.${String(process.pid)}.part`; + try { + writeFileSync(beside, text); + renameSync(beside, path); + } finally { + rmSync(beside, { force: true }); + } +} diff --git a/src/book/figures.ts b/src/book/figures.ts new file mode 100644 index 0000000..afb31e2 --- /dev/null +++ b/src/book/figures.ts @@ -0,0 +1,91 @@ +// The book's figures, read from the folder `figures` drew (spec 001 T029, contracts/figures.md): +// the book asks no render service. Each figure's file is `{stId}.svg`, named for an event +// whose section shows it; one drawing serves every section with the same figure (figureKey). +// A file is read each time the layout asks for it, never held, as the release's figures +// together are far larger than any one part. Its labels are printed as the book prints them +// (FR-017, src/figures/labels.ts); the second half of the character check, over its labels' +// text, is made before each part that draws it is printed (contracts/cli.md), a fault +// there naming the event the file is named for. +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { labelsInBook } from '../figures/labels.ts'; +import { BookCharacters } from '../layout/characters.ts'; +import { figureKey, type FigureSvgs } from '../layout/figure.ts'; +import type { Document } from '../model/types.ts'; + +/** What one drawing of a figure gives: its faults, and the substitutions it makes. */ +export interface FigureCheck { + faults: readonly string[]; + replaced: ReadonlyMap; +} + +export class FolderFigures implements FigureSvgs { + readonly #folder: string; + /** Each figure's key, and the event whose file draws it: the first in document order. */ + readonly #files = new Map(); + readonly #checked = new Map(); + #ms = 0; + + constructor(folder: string, docs: readonly Document[]) { + this.#folder = folder; + for (const doc of docs) { + for (const place of doc.places) { + if (!place.full || !place.section.figure) continue; + const key = figureKey(place.section.figure); + if (!this.#files.has(key)) this.#files.set(key, place.section.event.stId); + } + } + } + + /** Milliseconds spent reading and checking the folder's files, summed. */ + get ms(): number { + return this.#ms; + } + + /** A figure's file and the event it is named for; its text null where it is missing. */ + #read(key: string): { stId: string; svg: string | null } | undefined { + const stId = this.#files.get(key); + if (stId === undefined) return undefined; + const started = performance.now(); + try { + const path = join(this.#folder, `${stId}.svg`); + return { stId, svg: existsSync(path) ? readFileSync(path, 'utf8') : null }; + } finally { + this.#ms += performance.now() - started; + } + } + + /** + * A figure's labels checked, once a figure: a character no font draws a fault + * `uncovered-character `, a file gone since the preflight + * `missing-file .svg`; and the substitutions one drawing of it makes, for the run + * report to count each time it is drawn. + */ + check(key: string): FigureCheck { + let checked = this.#checked.get(key); + if (!checked) { + const read = this.#read(key); + if (!read) throw new Error(`No event of the book has the figure ${key}`); + if (read.svg === null) { + checked = { faults: [`missing-file ${read.stId}.svg`], replaced: new Map() }; + } else { + const started = performance.now(); + const characters = new BookCharacters(); + labelsInBook(read.svg, read.stId, characters); + this.#ms += performance.now() - started; + checked = { faults: characters.faults, replaced: characters.replaced }; + } + this.#checked.set(key, checked); + } + return checked; + } + + /** A figure's SVG as the book prints it, read from its file; none for one not given. */ + get(key: string): string | undefined { + const read = this.#read(key); + if (!read || read.svg === null) return undefined; + // the counts are the run's, made by check() for each drawing, so these are let go + return labelsInBook(read.svg, read.stId, new BookCharacters()); + } +} diff --git a/src/cli.ts b/src/cli.ts index 641fb8f..0868f4a 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node // pdf-export's command line (contracts/cli.md). Its commands come with spec 001's tasks: // serve (T022) runs the single-event service; preflight (T024) runs the book's preflight -// alone; figures (T046) draws the book's figures into a folder; book, check-book and compare -// are to come. +// alone; figures (T046) draws the book's figures into a folder; book (T029) writes the +// book; check-book and compare are to come. export const USAGE = `Usage: pdf-export [options] @@ -52,6 +52,12 @@ if (import.meta.main) { const args = process.argv.slice(2); if (args[0] === 'serve') { process.exitCode = await serve(args.slice(1)); + } else if (args[0] === 'book') { + const { bookCommand } = await import('./book/command.ts'); + process.exitCode = await bookCommand(args.slice(1), process.env, { + out: (text) => process.stdout.write(text), + err: (text) => process.stderr.write(text), + }); } else if (args[0] === 'preflight') { const { preflightCommand } = await import('./preflight/command.ts'); process.exitCode = await preflightCommand(args.slice(1), process.env, { diff --git a/src/layout/document.ts b/src/layout/document.ts index 4118382..1e68dd3 100644 --- a/src/layout/document.ts +++ b/src/layout/document.ts @@ -6,6 +6,7 @@ import { readFileSync } from 'node:fs'; import type { AnalysisProperties, Document, Found, Place } from '../model/types.ts'; +import type { FigureSvgs } from './figure.ts'; import { fontFaceCss } from './fonts.ts'; import { FRONT, @@ -46,7 +47,7 @@ export interface LayoutOptions { /** The cover's date. */ date: Date; /** The figures' SVGs, by figureKey. A section whose figure is missing is written without it. */ - figures: ReadonlyMap; + figures: FigureSvgs; styles: Styles; /** A single event's analysis: its properties and what it found, by stable identifier. */ analysis?: { properties: AnalysisProperties; found: ReadonlyMap } | null; diff --git a/src/layout/figure.ts b/src/layout/figure.ts index 2531227..84ba6c3 100644 --- a/src/layout/figure.ts +++ b/src/layout/figure.ts @@ -32,6 +32,12 @@ export const FLOOR_PT = 1; /** No figure is drawn larger than the render service's own size: a CSS pixel, 0.75 pt. */ const NATURAL = 0.75; +/** + * The figures' SVGs by figureKey, as the layout reads them: a map, or, in the book, the + * figures' folder read as each is asked for (src/book/figures.ts). + */ +export type FigureSvgs = Pick, 'get'>; + /** What the figures are looked up by: one drawing serves every section it is the figure of. */ export function figureKey(figure: Figure): string { return figure.kind === 'ancestor' diff --git a/src/layout/front.ts b/src/layout/front.ts index 3d76793..3881ffc 100644 --- a/src/layout/front.ts +++ b/src/layout/front.ts @@ -4,7 +4,7 @@ import { readFileSync } from 'node:fs'; import { prefixIds } from '../figures/prefix.ts'; import type { AnalysisProperties, ContentsEntry, Document } from '../model/types.ts'; -import { coverFigureHtml, figureKey } from './figure.ts'; +import { coverFigureHtml, figureKey, type FigureSvgs } from './figure.ts'; import { Links, attribute, text } from './html.ts'; import { ABOUT_REACTOME, @@ -42,12 +42,7 @@ const LOGO = prefixIds(readFileSync(new URL('logo.svg', import.meta.url), 'utf8' ', - date: Date -): string { +export function coverHtml(doc: Document, links: Links, figures: FigureSvgs, date: Date): string { const { cover } = doc; const parts = [``, `

${text(cover.title)}

`]; if (cover.release !== null) { diff --git a/src/layout/section.ts b/src/layout/section.ts index 7223994..9eaf8c1 100644 --- a/src/layout/section.ts +++ b/src/layout/section.ts @@ -4,7 +4,7 @@ // Parts are set apart by space, type and thin rules, never boxes (FR-016). import type { Reference } from '../graph/queries.ts'; import type { Found, FoundRow, Place, Section } from '../model/types.ts'; -import { figureHtml, figureKey, type Placement } from './figure.ts'; +import { figureHtml, figureKey, type FigureSvgs, type Placement } from './figure.ts'; import { anchorOf } from '../model/build.ts'; import { Links, attribute, text } from './html.ts'; @@ -20,7 +20,7 @@ export interface Report { export interface SectionInput { links: Links; - figures: ReadonlyMap; + figures: FigureSvgs; /** What an analysis found, by the event's stable identifier. */ found: ReadonlyMap | null; report: Report; diff --git a/src/preflight/command.ts b/src/preflight/command.ts index c4c2641..6749d3a 100644 --- a/src/preflight/command.ts +++ b/src/preflight/command.ts @@ -92,18 +92,19 @@ export async function preflightCommand( err(`The graph cannot be read: ${message(error)}\n`); return 2; } finally { - await start?.close(); + await closing(start, err); } - // the checks' reads with the driver's own retries - const graph = open(settings.graph); + // the checks' reads with the driver's own retries (opened inside the try, as above) const faults: string[] = []; let failure: { error: unknown } | null = null; + let graph: Graph | undefined; try { + graph = open(settings.graph); await preflight(graph, release, settings, faults); } catch (error) { failure = { error }; } finally { - await graph.close(); + await closing(graph, err); } // the faults found first, then the error that stopped the checks, if one did for (const fault of faults) out(`fault ${fault}\n`); @@ -114,5 +115,17 @@ export async function preflightCommand( return faults.length > 0 ? 1 : 0; } -const message = (error: unknown): string => +/** + * A graph closed, a failure to close said on stderr and never thrown: a rejection in a + * `finally` would replace the command's exit code with an error. + */ +export async function closing(graph: Graph | undefined, err: (text: string) => void) { + try { + await graph?.close(); + } catch (error) { + err(`The graph did not close: ${message(error)}\n`); + } +} + +export const message = (error: unknown): string => error instanceof Error ? error.message : String(error); diff --git a/src/print/parts.ts b/src/print/parts.ts index d6f2312..17a9eed 100644 --- a/src/print/parts.ts +++ b/src/print/parts.ts @@ -17,7 +17,13 @@ import { type LayoutOptions, type Report, } from '../layout/document.ts'; -import { figureKey, measure, place as placeFigure, PORTRAIT } from '../layout/figure.ts'; +import { + figureKey, + measure, + place as placeFigure, + PORTRAIT, + type FigureSvgs, +} from '../layout/figure.ts'; import type { Document, Figure, Place } from '../model/types.ts'; export interface PartsSettings { @@ -68,6 +74,13 @@ export interface BookRun { joined: (joined: Joined) => Promise; /** How parts are joined; the tests count it. */ join?: typeof joinParts; + /** + * Called in a part's slot before it is laid out, with its places (null for the front + * part); a throw fails the part as a failed print does. `book` checks there the labels of + * the figures the part draws (T029), so a fault in one stops the run before its part is + * printed, not before the first. + */ + laying?: (doc: Document, places: readonly Place[] | null) => void; } /** A figure as it prints: on a landscape page of its own, or its height in the portrait text. */ @@ -81,7 +94,7 @@ export interface FigureSize { export type Sizes = (figure: Figure) => FigureSize | null; /** Sizes from the layout's figures, each SVG measured once however many sections draw it. */ -export function figureSizes(figures: ReadonlyMap): Sizes { +export function figureSizes(figures: FigureSvgs): Sizes { const sizes = new Map(); return (figure) => { const key = figureKey(figure); @@ -302,6 +315,7 @@ export async function printBook(docs: readonly Document[], run: BookRun): Promis let left = parts.length; const print = async (places: readonly Place[] | null, i: number) => { if (failures.length > 0) return; + run.laying?.(doc, places); const laid = places ? layoutBookPart(doc, places, layout) : layoutBookFront(doc, layout); printed[i] = await printer.part(laid.html); reports[i] = laid.report; diff --git a/src/print/print.ts b/src/print/print.ts index 49773ea..c0f3936 100644 --- a/src/print/print.ts +++ b/src/print/print.ts @@ -18,17 +18,27 @@ export class Printer { readonly #folder: string; #count = 0; - private constructor(browser: Browser, folder: string) { + readonly #onExit: () => void; + + private constructor(browser: Browser, folder: string, onExit: () => void) { this.#browser = browser; this.#folder = folder; + this.#onExit = onExit; } static async start(): Promise { const folder = mkdtempSync(join(tmpdir(), 'pdf-export-')); + // its folder removed as the process exits if it is not closed first (a run stopped by a + // signal), from before the browser is launched + const onExit = () => { + rmSync(folder, { recursive: true, force: true }); + }; + process.once('exit', onExit); try { - return new Printer(await chromium.launch(), folder); + return new Printer(await chromium.launch(), folder, onExit); } catch (error) { - rmSync(folder, { recursive: true, force: true }); + process.off('exit', onExit); + onExit(); throw error; } } @@ -113,6 +123,7 @@ export class Printer { await this.#browser.close(); } finally { rmSync(this.#folder, { recursive: true, force: true }); + process.off('exit', this.#onExit); } } } diff --git a/src/print/stamp.ts b/src/print/stamp.ts index aa117a3..a72698e 100644 --- a/src/print/stamp.ts +++ b/src/print/stamp.ts @@ -55,11 +55,16 @@ const LEFT_OUT = /[\p{Cc}\p{Default_Ignorable_Code_Point}]/u; * space at either end trimmed. Any other space is kept, as Chromium keeps it: as itself * where a stamp face draws it (a no-break or thin space), and otherwise as U+0020, which * Chromium's box draws in its place where it has no face for it (an ideographic space, a - * line separator, an Ogham space mark), not refused as a character no face draws. + * line separator, an Ogham space mark), not refused as a character no face draws. Such a + * space at either end is not trimmed, as Chromium does not trim it: CSS strips only its + * collapsible white space at a line's ends, so the box keeps it, a blank before or after + * the name, and so does the stamp, as U+0020. * * Gaps against Chromium, known: Hangul fillers (U+3164, U+115F) are default-ignorable and - * left out here, where Chromium may draw them as blanks; a kept space is as wide as - * U+0020, not its own width; and nothing here is shaped (above). + * left out here, where Chromium may draw them as blanks; a space no stamp face draws is + * stamped as U+0020, so as wide as a word space, where Chromium draws it at its own width + * (an ideographic space an em wide), while a no-break or thin space, which a face draws, + * is stamped at its own; and nothing here is shaped (above). */ export function marginText(text: string): string { let out = ''; @@ -244,7 +249,10 @@ const stamped = new WeakSet(); * `rewriteLinks` gives them, so it runs after that. Throws, stamping nothing, when the * PDF is stamped already, the parts' pages are not the PDF's, a front part's heading is * not in the front part, the name leaves no header, a character of a stamp is one no font - * draws, or a face's file cannot be read; a PDF refused is not held as stamped. + * draws, or a face's file cannot be read; a PDF refused for any of these is not held as + * stamped. Once those checks pass it is held as stamped before anything is embedded, so a + * failure after that (embedding a face) leaves it held, part stamped: `book` discards a + * PDF on any throw, never saving one (T029). * * @param fileOf each face's file (`stampFile`); the tests give one that is missing */ @@ -285,7 +293,7 @@ export async function stampPages( } // every face's file read, and the space's face found, before the PDF is held: a refusal // leaves it unmarked - const files = new Map([...used.keys()].map((face) => [face, load(face, fileOf)])); + const files = [...used].map(([face, chars]) => ({ face, chars, file: load(face, fileOf) })); // a margin box's line is the first face's `normal` line (the one drawing a space, // Roboto's), centred in the box: the margin's height at the top or bottom of the page const space = faceFor(0x20); @@ -295,8 +303,7 @@ export async function stampPages( stamped.add(pdf); pdf.registerFontkit(fontkit); const fonts = new Map(); - for (const [face, chars] of used) { - const file = files.get(face) ?? load(face, fileOf); + for (const { face, chars, file } of files) { const customName = `${subsetTag(face, chars)}+${file.postscriptName}`; fonts.set(face, await pdf.embedFont(file.bytes, { subset: true, customName })); } diff --git a/test/book/book.test.ts b/test/book/book.test.ts new file mode 100644 index 0000000..56e6c87 --- /dev/null +++ b/test/book/book.test.ts @@ -0,0 +1,480 @@ +// `pdf-export book` end to end (spec 001 T029, contracts/cli.md) on the fixture's graph and +// release files, with a figures' folder each test writes itself (as T024's tests do; T046 +// writes the real one): the book written whole, one PDF per top-level pathway, or nothing +// written at all; its exit codes; no render service asked; and the run report's characters. +// Characters and names are planted in what the graph answers, through a graph that rewrites +// its rows, so the shared fixture is unchanged. +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { after, before, describe, test } from 'node:test'; + +import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; + +import { bookCommand, type BookIo, type RunReport } from '../../src/book/command.ts'; +import { + graphSettings, + openGraph, + type Graph, + type GraphOptions, + type GraphSettings, +} from '../../src/graph/connection.ts'; +import { releaseInfo } from '../../src/graph/queries.ts'; +import { PAGE } from '../../src/layout/figure.ts'; +import { PLACEHOLDER, PROBE } from '../../src/layout/html.ts'; +import { buildBook } from '../../src/model/build.ts'; +import { readSummary } from '../../src/preflight/files.ts'; +import { + expectedFigures, + manifestLine, + MANIFEST, + type FigureRecord, + type Fingerprint, +} from '../../src/preflight/folder.ts'; +import { Printer } from '../../src/print/print.ts'; +import { startRenderStub, type RenderStub } from '../stubs/render.ts'; + +const skip = process.env['NEO4J_URI'] ? false : 'no fixture graph: set NEO4J_URI'; +const FILES = fileURLToPath(new URL('../../ci/fixture/files/', import.meta.url)); +const FIGURES = fileURLToPath(new URL('../../ci/fixture/figures/', import.meta.url)); +const SITE = 'https://site.example'; +const CLI = fileURLToPath(new URL('../../src/cli.ts', import.meta.url)); +const BOOK = ['Alpha pathway.pdf', 'Gamma pathway.pdf', 'beta pathway.pdf']; + +/** The graph, its rows' strings rewritten: a character or a name planted. */ +function rewriting(graph: Graph, text: (value: string) => string): Graph { + return { + async read(query: string, params?: Record): Promise { + const visit = (item: unknown): unknown => { + if (typeof item === 'string') return text(item); + if (Array.isArray(item)) return item.map(visit); + if (item !== null && typeof item === 'object') { + return Object.fromEntries(Object.entries(item).map(([k, v]) => [k, visit(v)])); + } + return item; + }; + return (await graph.read(query, params)).map(visit) as T[]; + }, + close: () => graph.close(), + }; +} + +/** How `book` opens its graph, each `from` planted as its `to`. */ +const planting = + (plants: readonly [string, string][]) => + (settings: GraphSettings, options?: GraphOptions): Graph => + rewriting(openGraph(settings, options), (value) => + plants.reduce((v, [from, to]) => v.split(from).join(to), value) + ); + +type PdfJs = Awaited['promise']>; + +async function reading(path: string, read: (doc: PdfJs) => Promise): Promise { + const task = getDocument({ data: new Uint8Array(readFileSync(path)) }); + try { + return await read(await task.promise); + } finally { + await task.destroy(); + } +} + +/** A PDF's text, page by page, without what is stamped in the top and bottom margins. */ +function bodyText(path: string): Promise { + return reading(path, async (doc) => { + const pages: string[] = []; + for (let i = 1; i <= doc.numPages; i++) { + const page = await doc.getPage(i); + const [, , , height = 0] = page.view; + const items = (await page.getTextContent()).items.flatMap((item) => { + if (!('str' in item)) return []; + const y = (item.transform as number[])[5] ?? 0; + return y > PAGE.margin && y < height - PAGE.margin ? [item.str] : []; + }); + pages.push(items.join('')); + } + return pages; + }); +} + +const occurrences = (text: string, of: string) => text.split(of).length - 1; + +describe('pdf-export book', { skip }, () => { + let expected: Map; + let print: Fingerprint; + let stub: RenderStub; + /** + * The test file's own folder, removed after: every folder a test writes is in it, so a run + * stopped before `after` (killed for memory, say) leaves one folder, not one a test. + */ + let root: string | undefined; + const scratch = (): string => { + root ??= mkdtempSync(join(tmpdir(), 'pdfx-book-')); + return mkdtempSync(join(root, 'test-')); + }; + + /** + * A figures' folder as `figures` would leave it for the fixture's book: each event's file + * the recorded SVG of what it draws, given to `edit` with its event first, and the manifest. + */ + const folder = (edit: (stId: string, svg: string) => string = (_, svg) => svg): string => { + const dir = scratch(); + const lines = [manifestLine({ type: 'fingerprint', ...print })]; + for (const record of expected.values()) { + const file = join( + FIGURES, + `${record.source}${record.kind === 'reaction' ? '.reaction' : ''}.svg` + ); + writeFileSync(join(dir, `${record.stId}.svg`), edit(record.stId, readFileSync(file, 'utf8'))); + lines.push(manifestLine({ type: 'figure', ...record })); + } + lines.push(manifestLine({ type: 'complete', ...print })); + writeFileSync(join(dir, MANIFEST), lines.join('')); + return dir; + }; + + /** + * `book` run in this process, its temporary folder (TMPDIR: the printer's pages, Chromium's + * profile) one of its own, and what is left there after it, which should be nothing. + */ + const book = async ( + args: string[], + io: Partial = {}, + env: Record = { + NEO4J_URI: process.env['NEO4J_URI'], + RENDER_SERVICE: stub.url, + } + ) => { + const out: string[] = []; + const err: string[] = []; + const temporary = scratch(); + const was = process.env['TMPDIR']; + process.env['TMPDIR'] = temporary; + let code: number; + try { + code = await bookCommand(args, env, { + out: (text) => out.push(text), + err: (text) => err.push(text), + date: new Date('2026-11-03T12:00:00Z'), + ...io, + }); + } finally { + if (was === undefined) delete process.env['TMPDIR']; + else process.env['TMPDIR'] = was; + } + return { code, out: out.join(''), err: err.join(''), left: readdirSync(temporary) }; + }; + + /** A parent folder for the book, and the arguments that write it there. */ + const place = (figures: string = folder()) => { + const parent = scratch(); + const out = join(parent, 'TheReactomeBook'); + const report = join(scratch(), 'report.json'); + const args = [ + ...['--files', FILES, '--figures', figures, '--site', SITE], + ...['--out', out, '--report', report], + ]; + return { parent, out, report, args }; + }; + + const readReport = (path: string) => JSON.parse(readFileSync(path, 'utf8')) as RunReport; + + before(async () => { + stub = await startRenderStub(); + const graph = openGraph(graphSettings(process.env)); + try { + const release = await releaseInfo(graph); + const summary = readSummary(FILES); + if (!summary) throw new Error('the fixture has no EHLD summary'); + expected = expectedFigures(await buildBook(graph, { illustrated: summary.illustrated })); + print = { + release: release.release, + checksum: release.checksum ?? '', + summary: summary.hash, + content: release.release, + build: 'test-build', + }; + } finally { + await graph.close(); + } + }); + after(async () => { + await stub.close(); + if (root) rmSync(root, { recursive: true, force: true }); + }); + + test('exit 0 on the fixture: one PDF per top-level pathway and no other, finished, with no render service asked', async () => { + // provisional until check-book (T030) checks the book + const { parent, out, report, args } = place(); + // two at once, not the default four, to spare the memory of a test run's other prints + const run = await book([...args, '--parallel', '2', '--joins', '2']); + assert.deepEqual(run.left, [], 'left in its temporary folder'); + assert.equal(run.code, 0, run.err); + assert.equal(run.out, ''); + // the folder moved into place, nothing beside it + assert.deepEqual(readdirSync(parent), ['TheReactomeBook']); + assert.deepEqual(readdirSync(out).sort(), BOOK); + assert.deepEqual(stub.requests, [], 'a render service was asked'); + const written = readReport(report); + assert.equal(written.outcome, 'written'); + assert.equal(written.release, 98); + assert.deepEqual( + written.pdfs.map((pdf) => [pdf.file, pdf.stId]), + [ + ['Alpha pathway.pdf', 'R-HSA-90000200'], + ['beta pathway.pdf', 'R-HSA-90000100'], + ['Gamma pathway.pdf', 'R-HSA-90000300'], + ] + ); + // every figure the book needs, by kind, as the folder's manifest lists them + const byKind = { diagram: 0, ancestor: 0, ehld: 0, reaction: 0 }; + for (const record of expected.values()) byKind[record.kind] += 1; + assert.deepEqual(written.figures.byKind, byKind); + assert.equal(expected.size, 18); + assert.deepEqual(written.characters, { replaced: {}, uncovered: [] }); + assert.deepEqual(written.faults, []); + assert.deepEqual(written.joins, { joins: 2, parallel: 2, objectStreams: false }); + assert.deepEqual( + written.stages.map((stage) => stage.stage), + ['preflight', 'figures', 'print', 'join', 'contents', 'parts', 'check-book'] + ); + for (const pdf of written.pdfs) { + const path = join(out, pdf.file); + const bytes = readFileSync(path); + assert.equal(bytes.length, pdf.bytes, pdf.file); + const text = bytes.toString('latin1'); + // its links rewritten, saved with object streams off + assert.ok(!text.includes(PLACEHOLDER) && !text.includes(PROBE), pdf.file); + assert.ok(!text.includes('/ObjStm'), `${pdf.file} has object streams`); + await reading(path, async (doc) => { + assert.equal(doc.numPages, pdf.pages, pdf.file); + // its outline written, its pages stamped: the last page's number at its foot + assert.ok(((await doc.getOutline()) as unknown[] | null)?.length, pdf.file); + const last = await doc.getPage(doc.numPages); + const foot = (await last.getTextContent()).items.flatMap((item) => + 'str' in item && ((item.transform as number[])[5] ?? 0) < PAGE.margin ? [item.str] : [] + ); + assert.equal(foot.join(''), String(doc.numPages), pdf.file); + // its contents numbered (T028): every number on the contents' page past the cover + let numbers: number[] = []; + for (let i = 1; i <= doc.numPages && numbers.length === 0; i++) { + const items = (await (await doc.getPage(i)).getTextContent()).items.flatMap((item) => + 'str' in item ? [{ str: item.str, transform: item.transform as number[] }] : [] + ); + const heading = items.some( + (item) => + item.str === 'Contents' && + Math.hypot(item.transform[2] ?? 0, item.transform[3] ?? 0) > 17 + ); + if (!heading) continue; + numbers = items.flatMap((item) => { + const y = item.transform[5] ?? 0; + const x = item.transform[4] ?? 0; + return /^\d+$/.test(item.str) && y > PAGE.margin && x > 300 ? [Number(item.str)] : []; + }); + } + assert.ok(numbers.length > 0, `${pdf.file}: no contents numbers`); + assert.ok( + numbers.every((n) => n > 1 && n <= doc.numPages), + `${pdf.file}: ${numbers.join(' ')}` + ); + }); + } + assert.equal( + written.total.pages, + written.pdfs.reduce((n, pdf) => n + pdf.pages, 0) + ); + }); + + test('exit 1 on a figure file missing from --figures, with nothing left', async () => { + const figures = folder(); + unlinkSync(join(figures, 'R-HSA-90000211.svg')); + const { parent, report, args } = place(figures); + const run = await book(args); + assert.deepEqual(run.left, [], 'left in its temporary folder'); + assert.equal(run.code, 1, run.err); + assert.equal(run.out, 'fault missing-file R-HSA-90000211.svg\n'); + assert.deepEqual(readdirSync(parent), []); + const written = readReport(report); + assert.equal(written.outcome, 'faults'); + assert.deepEqual(written.figures.missing, ['R-HSA-90000211']); + }); + + test('exit 2 on an --out that is not empty, or not a folder, which is left as it was', async () => { + const { parent, out, args } = place(); + mkdirSync(out); + writeFileSync(join(out, 'old.pdf'), 'an old book'); + const run = await book(args); + assert.deepEqual(run.left, [], 'left in its temporary folder'); + assert.equal(run.code, 2); + assert.match(run.err, /is not empty/); + assert.deepEqual(readdirSync(parent), ['TheReactomeBook']); + assert.deepEqual(readdirSync(out), ['old.pdf']); + rmSync(out, { recursive: true }); + writeFileSync(out, 'a file'); + assert.equal((await book(args)).code, 2); + assert.deepEqual(readdirSync(parent), ['TheReactomeBook']); + assert.equal(readFileSync(out, 'utf8'), 'a file'); + }); + + test('exit 2 on bad arguments, and on the graph unreachable at the start', async () => { + const { parent, args } = place(); + const bad = [ + [...args, '--render', stub.url], + args.filter((_, i) => i !== 2 && i !== 3), + [...args, '--parallel', '0'], + [...args, '--parallel', '2', '--joins', '3'], + [...args, '--joins', 'one'], + [...args, '--out', 'twice'], + ]; + for (const given of bad) assert.equal((await book(given)).code, 2, given.join(' ')); + // no site, and no graph + assert.equal((await book(args.filter((_, i) => i !== 4 && i !== 5))).code, 2); + assert.equal((await book(args, {}, {})).code, 2); + // an address that cannot be parsed (never one that could reach another database), and a + // graph whose reads fail + const unparsable = await book(args, {}, { NEO4J_URI: '127.0.0.1:17700' }); + assert.equal(unparsable.code, 2); + assert.match(unparsable.err, /the graph cannot be read/); + const failing = (): Graph => ({ + read: () => Promise.reject(new Error('the planted failure')), + close: () => Promise.resolve(), + }); + const unreachable = await book(args, { open: failing }); + assert.equal(unreachable.code, 2); + assert.deepEqual(unreachable.left, []); + assert.match(unreachable.err, /the graph cannot be read: the planted failure/); + assert.deepEqual(readdirSync(parent), []); + }); + + test("the run report counts FR-017's substitutions in the finished book, by code point", async () => { + // a summation printed once (Alpha's), a reaction's name printed in its heading and its + // contents entry (Reaction eight's), a label in its figure, drawn once, and a top-level + // pathway's name (Alpha's), printed on its cover, in its heading, its contents entry and + // the locations of the full sections beneath it, but neither in the short entries' + // locations, which are not printed (Two-parent sub's later place), nor in the headers + // stamped on its pages; and a label in beta's own figure, drawn on its cover and in its + // section, counted twice + const plants: [string, string][] = [ + ['A placeholder summation', 'A placeholder \uFB01summation'], + ['Reaction eight', 'Reaction\u2011eight'], + ['Alpha pathway', 'Alpha\u05F3pathway'], + ]; + const figures = folder((stId, svg) => + stId === 'R-HSA-90000132' || stId === 'R-HSA-90000100' + ? svg.replace(/(]*>)/, '$1\u02D9') + : svg + ); + const { out, report, args } = place(figures); + const run = await book([...args, '--parallel', '1'], { open: planting(plants) }); + assert.deepEqual(run.left, [], 'left in its temporary folder'); + assert.equal(run.code, 0, run.err); + // the file named for the name as the graph has it + assert.deepEqual(readdirSync(out).sort(), [ + 'Alpha\u05F3pathway.pdf', + 'Gamma pathway.pdf', + 'beta pathway.pdf', + ]); + const text = new Map(); + for (const file of readdirSync(out)) + text.set(file, (await bodyText(join(out, file))).join('\n')); + const all = [...text.values()].join('\n'); + // the prime replacing U+05F3, as printed outside the stamps + const primes = occurrences(all, '\u2032'); + assert.ok(primes >= 4, `${String(primes)} primes printed`); + assert.equal(occurrences(all, 'Alpha\u2032pathway'), primes); + assert.equal(occurrences(all, '\u05F3'), 0); + assert.equal(occurrences(all, 'Reaction-eight'), 2); + assert.equal(occurrences(all, 'A placeholder fisummation'), 1); + assert.deepEqual(readReport(report).characters, { + replaced: { 'U+02D9': 3, 'U+05F3': primes, 'U+2011': 2, 'U+FB01': 1 }, + uncovered: [], + }); + }); + + test("a character off the list in a later part's figure: exit 1 after earlier parts are printed, and nothing left", async () => { + // a backstop for a folder changed by hand, which `figures` would have refused + const figures = folder((stId, svg) => + stId === 'R-HSA-90000132' ? svg.replace(/(]*>)/, '$1\uE000') : svg + ); + const { parent, report, args } = place(figures); + let printed = 0; + const run = await book([...args, '--parallel', '1'], { + threshold: 1, + startPrinter: async () => { + const printer = await Printer.start(); + return { + part: async (html) => { + const pdf = await printer.part(html); + printed += 1; + return pdf; + }, + close: () => printer.close(), + }; + }, + }); + assert.equal(run.code, 1, run.err); + assert.equal(run.out, 'fault uncovered-character U+E000 R-HSA-90000132\n'); + assert.ok(printed >= 3, `${String(printed)} parts printed before it`); + assert.deepEqual(readdirSync(parent), []); + // nor the printer's pages, nor Chromium's profile, in the run's temporary folder + assert.deepEqual(run.left, []); + const written = readReport(report); + assert.equal(written.outcome, 'faults'); + assert.deepEqual(written.characters.uncovered, ['U+E000 R-HSA-90000132']); + assert.deepEqual(written.pdfs, []); + }); + + test('a PDF refused after its join is discarded, and the run fails whole', async () => { + // a name that leaves no header to stamp (white space and a joiner): the preflight passes + // it, and stamping refuses its PDF after its links and outline are done + const { parent, report, args } = place(); + const run = await book([...args, '--parallel', '1'], { + open: planting([['Gamma pathway', ' \u200D ']]), + }); + assert.equal(run.code, 2); + assert.match(run.err, /leaves no header to stamp/); + assert.deepEqual(run.left, [], 'left in its temporary folder'); + assert.equal(run.out, ''); + assert.deepEqual(readdirSync(parent), []); + assert.equal(readReport(report).outcome, 'error'); + assert.ok(!existsSync(join(parent, 'TheReactomeBook'))); + }); + test('a run stopped by a signal while printing leaves no folder, temporary or not', async () => { + const { parent, args } = place(); + const temporary = scratch(); + const child = spawn(process.execPath, [CLI, 'book', ...args, '--parallel', '1'], { + env: { ...process.env, TMPDIR: temporary }, + stdio: 'ignore', + }); + const ended = new Promise((resolve) => child.once('exit', resolve)); + try { + // stopped once its folder is there, beside the book's name, while it prints + const deadline = Date.now() + 60_000; + while (!readdirSync(parent).some((name) => name.startsWith('.TheReactomeBook-'))) { + assert.ok(Date.now() < deadline, 'the run never began printing'); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + child.kill('SIGTERM'); + const stop = setTimeout(() => child.kill('SIGKILL'), 30_000); + await ended; + clearTimeout(stop); + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + } + assert.notEqual(child.exitCode, 0); + assert.deepEqual(readdirSync(parent), []); + assert.deepEqual(readdirSync(temporary), []); + }); +}); diff --git a/test/layout/layout.test.ts b/test/layout/layout.test.ts index e12ad61..9419c58 100644 --- a/test/layout/layout.test.ts +++ b/test/layout/layout.test.ts @@ -44,7 +44,8 @@ const ids = (html: string) => new Set([...html.matchAll(/\sid="([^"]*)"/g)].map( describe('the layout', { skip: skipWithoutGraph }, () => { let built: Built; - let options: LayoutOptions; + // the fixture's figures as a map, which a test copies and changes + let options: LayoutOptions & { figures: ReadonlyMap }; before(async () => { built = await buildFixture(); options = { diff --git a/test/preflight/preflight.test.ts b/test/preflight/preflight.test.ts index a74ad83..0d53fcf 100644 --- a/test/preflight/preflight.test.ts +++ b/test/preflight/preflight.test.ts @@ -402,6 +402,52 @@ describe("the book's preflight", { skip }, () => { assert.deepEqual(both, ['out', 'err']); }); + test("the checks' graph failing to open is exit 2, and a graph failing to close keeps the exit code", async () => { + const call = async (open: (settings: GraphSettings) => Graph, figures: string) => { + const out: string[] = []; + const err: string[] = []; + const code = await preflightCommand( + ['--files', FILES, '--figures', figures], + { NEO4J_URI: process.env['NEO4J_URI'] }, + { out: (text) => out.push(text), err: (text) => err.push(text), open } + ); + return { code, out: out.join(''), err: err.join('') }; + }; + // the second open, the checks', throws at once: an error, not a rejection of the command + let opened = 0; + const secondThrows = (settings: GraphSettings): Graph => { + opened += 1; + if (opened === 2) throw new Error('the planted open failure'); + return openGraph(settings); + }; + const thrown = await call(secondThrows, folder()); + assert.equal(thrown.code, 2); + assert.equal(thrown.out, ''); + assert.match(thrown.err, /^The preflight stopped: the planted open failure\n$/); + // every close rejects, after closing: the exit code is the checks', 0 and then 1 + const closeRejects = (settings: GraphSettings): Graph => { + const real = openGraph(settings); + return { + read: (query: string, params?: Record) => real.read(query, params), + close: async () => { + await real.close(); + throw new Error('the planted close failure'); + }, + }; + }; + const clean = await call(closeRejects, folder()); + assert.equal(clean.code, 0); + assert.equal(clean.out, ''); + assert.equal( + clean.err, + 'The graph did not close: the planted close failure\n'.repeat(2), + "both graphs, the start's and the checks'" + ); + const faulty = await call(closeRejects, scratch()); + assert.equal(faulty.code, 1); + assert.equal(faulty.out, 'fault incomplete-folder\n'); + }); + test('pdf-export preflight: exit 0 with no fault, 1 with each fault a line, 2 on bad arguments or no graph', async () => { const env: Record = { NEO4J_URI: process.env['NEO4J_URI'] }; const call = async (args: string[], environment = env) => { diff --git a/test/print/parts.test.ts b/test/print/parts.test.ts index 7d544e3..e356342 100644 --- a/test/print/parts.test.ts +++ b/test/print/parts.test.ts @@ -66,7 +66,8 @@ const THRESHOLD = 1; describe('parts', { skip: skipWithoutGraph }, () => { let built: Built; let started: Printer | undefined; - let options: LayoutOptions; + // the fixture's figures as a map, which a test copies and changes + let options: LayoutOptions & { figures: ReadonlyMap }; /** Each part printed, by its HTML. */ const printed = new Map(); /** Each PDF joined, saved, by its title. */ diff --git a/test/print/stamp.test.ts b/test/print/stamp.test.ts index 433772a..e9761f6 100644 --- a/test/print/stamp.test.ts +++ b/test/print/stamp.test.ts @@ -671,6 +671,9 @@ describe('pageStamps', () => { ['a\u2028b', 'a b'], ['a\u1680b', 'a b'], ['a\u3000 b', 'a b'], + // and not trimmed at either end: CSS strips only collapsible white space at a line's + // ends, so Chromium's box keeps it there, a blank before the name + ['\u3000a', ' a'], ['a\uFE0Fb', 'ab'], ['a \uFE0F b', 'a b'], ['a\u200Db', 'ab'],