diff --git a/ci/fixture/README.md b/ci/fixture/README.md index c22e1f2..6d8b700 100644 --- a/ci/fixture/README.md +++ b/ci/fixture/README.md @@ -14,6 +14,7 @@ release 98's (every relationship has `order` and `stoichiometry`). | Case | Where | For | | ---------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | --------------------------------------------- | | Release 98 | `DBInfo.releaseNumber` | the introduction, the base of ids | +| A checksum beyond what a JavaScript number holds whole (2^53 + 1) | `DBInfo.checksum` | the figures' folder's fingerprint (T024) | | Three human top-level pathways, ordered differently by id, by case and ignoring case | `beta pathway`, `Alpha pathway`, `Gamma pathway` | FR-002's order | | A non-human top-level pathway | `Mouse pathway` | `--species` | | A subpathway with no diagram, under one with a diagram | `Alpha sub one` | the ancestor's diagram with a selection | @@ -32,6 +33,7 @@ release 98's (every relationship has `order` and `stoichiometry`). | A disease, and one named exactly "disease" | `Disease reaction` | the "disease" exclusion | | A node that is not an event, with a stable id | `Placeholder complex` | 400 `not-an-event` | | Participants of each kind, in a complex, a set, a candidate set and a polymer, with identifiers (one an isoform) | `Reaction one` | a reaction's own identifiers, for an analysis | +| A compartment only a participant holds, named in no event's section | `placeholder membrane`, of Reaction one's output | FR-017 in the figures' names (T024) | ## The release's files diff --git a/ci/fixture/fixture.cypher b/ci/fixture/fixture.cypher index f27fb0f..a788c48 100644 --- a/ci/fixture/fixture.cypher +++ b/ci/fixture/fixture.cypher @@ -4,12 +4,13 @@ // lists the case each item is here for. Load with: cypher-shell -d graph.db -f fixture.cypher CREATE INDEX fixture_dbId IF NOT EXISTS FOR (n:DatabaseObject) ON (n.dbId); CREATE INDEX fixture_stId IF NOT EXISTS FOR (n:DatabaseObject) ON (n.stId); -CREATE (db:DatabaseObject:MetaDatabaseObject:DBInfo:Release {dbId: 90000001, displayName: '98 on 2026-09-18', name: 'reactome', releaseNumber: 98, releaseDate: '2026-09-18', schemaClass: 'DBInfo'}) +CREATE (db:DatabaseObject:MetaDatabaseObject:DBInfo:Release {dbId: 90000001, displayName: '98 on 2026-09-18', name: 'reactome', releaseNumber: 98, checksum: 9007199254740993, releaseDate: '2026-09-18', schemaClass: 'DBInfo'}) CREATE (hsa:DatabaseObject:Taxon:Species {dbId: 90000002, displayName: 'Homo sapiens', name: ['Homo sapiens'], taxId: '9606', abbreviation: 'HSA', schemaClass: 'Species'}) CREATE (mmu:DatabaseObject:Taxon:Species {dbId: 90000003, displayName: 'Mus musculus', name: ['Mus musculus'], taxId: '10090', abbreviation: 'MMU', schemaClass: 'Species'}) CREATE (rno:DatabaseObject:Taxon:Species {dbId: 90000004, displayName: 'Rattus norvegicus', name: ['Rattus norvegicus'], taxId: '10116', abbreviation: 'RNO', schemaClass: 'Species'}) CREATE (cyto:DatabaseObject:Compartment:GO_CellularComponent:GO_Term {dbId: 90000010, displayName: 'cytosol', name: 'cytosol', accession: '0000010', databaseName: 'GO', schemaClass: 'Compartment'}) CREATE (nucl:DatabaseObject:Compartment:GO_CellularComponent:GO_Term {dbId: 90000011, displayName: 'nucleoplasm', name: 'nucleoplasm', accession: '0000011', databaseName: 'GO', schemaClass: 'Compartment'}) +CREATE (memb:DatabaseObject:Compartment:GO_CellularComponent:GO_Term {dbId: 90000014, displayName: 'placeholder membrane', name: 'placeholder membrane', accession: '0000014', databaseName: 'GO', schemaClass: 'Compartment'}) CREATE (dis1:DatabaseObject:ExternalOntology:Disease {dbId: 90000012, displayName: 'placeholder infectious disease', name: ['placeholder infectious disease'], identifier: '0000001', databaseName: 'DOID', schemaClass: 'Disease'}) CREATE (dis2:DatabaseObject:ExternalOntology:Disease {dbId: 90000013, displayName: 'disease', name: ['disease'], identifier: '4', databaseName: 'DOID', schemaClass: 'Disease'}) CREATE (beta:DatabaseObject:Event:Pathway:Trackable:Deletable:TopLevelPathway {dbId: 90000100, stId: 'R-HSA-90000100', stIdVersion: 'R-HSA-90000100.1', displayName: 'beta pathway', name: ['beta pathway'], schemaClass: 'TopLevelPathway', speciesName: 'Homo sapiens', hasDiagram: true, hasEHLD: false, isInferred: false, isInDisease: false, releaseDate: '2026-09-18'}) @@ -144,6 +145,7 @@ CREATE (ewas5)-[:referenceEntity {order: 0, stoichiometry: 1}]->(ref5) CREATE (r1)-[:input {order: 2, stoichiometry: 1}]->(poly1) CREATE (poly1)-[:repeatedUnit {order: 0, stoichiometry: 1}]->(ewas6) CREATE (ewas6)-[:referenceEntity {order: 0, stoichiometry: 1}]->(ref6) +CREATE (ewas1)-[:compartment {order: 0, stoichiometry: 1}]->(memb) CREATE (r1)-[:compartment {order: 0, stoichiometry: 1}]->(nucl) CREATE (r1)-[:compartment {order: 1, stoichiometry: 1}]->(cyto) CREATE (r3)-[:compartment {order: 0, stoichiometry: 1}]->(cyto) diff --git a/specs/001-pathway-pdfs/contracts/cli.md b/specs/001-pathway-pdfs/contracts/cli.md index 85e3f78..759c75d 100644 --- a/specs/001-pathway-pdfs/contracts/cli.md +++ b/specs/001-pathway-pdfs/contracts/cli.md @@ -1,6 +1,6 @@ # Contract: the command line -One executable, `pdf-export`, with five commands. Settings come from flags or from the +One executable, `pdf-export`, with six commands. Settings come from flags or from the environment variables named beside them. A password never goes on the command line. ## `pdf-export figures` @@ -76,7 +76,9 @@ the figures already drawn by `figures`. - **before any figure is read** (and by `figures`, before any is drawn), in the graph's texts the layout writes, in the names the figures label (every physical entity a reaction's figure draws: its inputs, outputs, catalysts and regulators; and their compartments, from the graph), and in - the labels of the diagram files the Figure rules choose to draw (`--files`; today an + the labels of the diagram files the Figure rules choose to draw (the `displayName`s + of their nodes, compartments, notes, shadows and links, and the `label`s of their + nodes' attachments, a modification's lettering) (`--files`; today an illustrated pathway's file is not among them, its figure being its EHLD, data-model Figure); a fault in a text names the event whose section holds it, one in a figure's names the first event in document order whose figure holds it, and one in @@ -95,6 +97,29 @@ the figures already drawn by `figures`. - **Exit 2:** bad arguments; an `--out` that is not absent or empty, as for `figures` (an old book is never replaced in place); or the graph cannot be reached at the start. +## `pdf-export preflight` + +`book`'s preflight alone (T024), with `book`'s `--graph`, `--database`, `--files`, +`--figures` (both required) and `--species`, so a release's inputs and figures' folder can +be checked before the book is made. It prints one line per fault, `fault `, in the +forms above (`missing-file `, `incomplete-folder`, `no-checksum`, +`stale-folder / /`, `stale-summary`, +`stale-figure `, `bad-file-name `, `uncovered-character `, +and `bad-file ` for a diagram file that cannot be read as JSON, whose labels are then +not checked), +each naming a stable identifier or a file, never a text or a person. A release file is named +as in `--files` (`diagram/R-HSA-….json`), a figure's as in `--figures` (`R-HSA-….svg`). + +- **Exit 0:** no fault. **Exit 1:** at least one. **Exit 2:** bad arguments; the graph + cannot be reached at the start (said within seconds: that first read's retrying is + bounded, the rest keep the driver's own); or any other error during the checks (a graph + that fails mid-run), after the faults found before it are printed, the error on stderr. + Exit 1 always means faults, never an error. +- A folder that is not whole, or not this graph's release or summary's, is refused as a + whole: its figures are then not checked one by one. +- Without the EHLD summary the Figure rules cannot be followed: the one fault is + `missing-file ehld/svgsummary.txt`. + ## `pdf-export check-book ` FR-011's check, alone. Also run by `book`. diff --git a/specs/001-pathway-pdfs/contracts/figures.md b/specs/001-pathway-pdfs/contracts/figures.md index f7baff7..b2edd09 100644 --- a/specs/001-pathway-pdfs/contracts/figures.md +++ b/specs/001-pathway-pdfs/contracts/figures.md @@ -56,6 +56,13 @@ What `pdf-export figures` writes and `pdf-export book` reads (SC-010, research R - **Its last line, when every figure has its line:** `complete`, whose fingerprint fields equal the first line's. +- **The lines' form** (T024, `src/preflight/folder.ts`, which T046 writes with): each a JSON + object with a `type`. The fingerprint and `complete`: `release` (a number), `checksum` + (the `DBInfo` checksum's decimal digits, a string, as a long can exceed what a JSON + reader's number holds whole), `summary` (`sha256:` and the hex digest of the summary + file's bytes), `content` (a number) and `build` (a string). A figure: `stId`, `kind` + (`diagram`, `ancestor`, `ehld` or `reaction`), `source` and `highlight` (sorted; empty + but for `ancestor`). - **A deployment changed, a service gone, or the breaker tripped, during a run** stops it at once with exit 1 and no `complete` line. Figures still being drawn when it stops get no line. The report is written either way, diff --git a/specs/001-pathway-pdfs/tasks.md b/specs/001-pathway-pdfs/tasks.md index e7fb471..29516b8 100644 --- a/specs/001-pathway-pdfs/tasks.md +++ b/specs/001-pathway-pdfs/tasks.md @@ -351,6 +351,7 @@ has the Background's parts in order, and bookmarks. Each error case gives its an - [ ] T047 [US2] The figures' folder in the service, after T024 (contracts/service.md, spec SC-005). - With `--figures`, the folder is used for a PDF without a token while it passes T024's folder checks against the graph's current release: the full checks at start and whenever the folder's fingerprint or the graph's release and checksum change (those read again at most once a minute); otherwise every figure is drawn on request and `/health` says why (`stale`, `incomplete`, `none`). - A figure the folder lacks (another species' event) is drawn on request and logged. + - T024's `checkFolder` reports a folder's faults, not which state `/health` names: T047 compares the fingerprint (release, checksum, summary hash) itself, and tells `none` (no folder, or no manifest) apart from `incomplete` (a manifest that is not whole) (#149's review). Tests: a PDF without a token asks the render stub nothing; one with a token asks for each figure with the token; a folder of another release is not used and `/health` reports it; a figure the folder lacks is drawn and logged; the full checks run once at start and again only when the fingerprint or the release changes (the graph stub counts its reads). @@ -380,10 +381,10 @@ passes. - **Done (#127):** `buildBook` returns the book's PDFs, one `'book'` document per top-level pathway in FR-002's order. The tree is read once for them all (`readTree`, a level at a time, as before), and so are the events' facts, literature, edits and diagram choices (`readEvents`, over every PDF's events at once); each PDF is then walked on its own (`walk`, with its own set of events seen), so an event under two top-level pathways is in full in each, and what is computed over a PDF is computed over that one (`assemble`): the events it holds, so a reaction's events before and after in another PDF are left out; each event's first place; and the cover's names, from its own events' edits. Each cover has its pathway's name, the release, no excerpt note, and its pathway's own figure, the figure of its first section, so the cover's figure is always among the figures the sections draw (Gamma's is its illustration, not a diagram). `buildSingle` is unchanged in what it gives, through the same three steps. `Cover`'s and `Document`'s comments say what a book's PDF holds; data-model's Book drops its "until T048" note. The layout is unchanged. Tests: `test/model/build.test.ts` (12, from 9) has the PDFs' order and trees, the shared reaction and beta's subtree through "beta sub with diagram" in full in beta's PDF with their paths there, each cover's names, no link (a short entry, an event before or after, a contents entry) to a section the PDF does not hold, and each PDF's cover, figure and counts; `test/layout/layout.test.ts` (15, from 14) checks each PDF's placeholders and probes, its sections' order and short entries, Reaction one with Polymerisation four and Inferred reaction left out, and that each cover holds its pathway's figure, from the figures the sections draw; `test/layout/fixture.ts` builds every PDF of the book (`bookPdf` finds one by name), and `test/layout/print.test.ts` and `test/print/print.test.ts` print Alpha's. 178 tests before, 182 after. Each of these mutants made one fail: only the first PDF returned (the order test); one set of events seen across the PDFs, the second's shared events short entries (the in-full-in-each test, the no-link test); the book's names on every cover (the names test); the events held counted across the book (the no-link test, and the layout's events before and after); no cover figure, and a cover figure always of kind `diagram`, Gamma's then missing from the figures (the layout's cover test, and the model's); the names ordered with case (the order test). -- [ ] T024 [US1] The preflight, in `src/cli.ts` `book --files --figures ` (contracts/cli.md). +- [x] T024 [US1] The preflight, in `src/cli.ts` `book --files --figures ` (contracts/cli.md). - **Figures:** each figure the Figure rules choose must be in `--figures` (contracts/figures.md); one missing is a fault `missing-file {stId}.svg`, and the folder checks contracts/figures.md lists under "What `book` checks" (`incomplete-folder`, `no-checksum`, `stale-folder`, `stale-summary`, `stale-figure {stId}`); T024 adds a `checksum` to the fixture's `DBInfo` and to `src/graph/queries.ts`'s release reader. The preflight's checks live in `src/preflight/`, for T046 to run too. Tests, with a figures' folder the test writes (T046 comes after): a planted missing figure file, a manifest entry with another source, a figure the rules choose with no line, a line for an event the rules give no figure, a folder of another checksum and one of another release number (`stale-folder`), one of another summary hash (`stale-summary`), a folder with no `complete` line, one with a figure line after `complete`, one with a torn last line, one with a middle line that is not JSON, one with two lines for one figure, and one whose `complete` differs from its first line (each `incomplete-folder`), and a graph with no checksum (`no-checksum`) are each caught before anything is printed. - **Files:** the files the figures need come from the graph's events. The summary only says which of them are illustrations, and its entries for events the graph does not hold are ignored (research R7). Each missing file is a fault `missing-file `. - - **Characters (FR-017):** first FR-017's substitutions, in T022's `src/layout/characters.ts` (with T017's detector), which T024 extends with the book's fault mode, applied to every text the layout writes and to the text of every inlined figure's labels (their text content only, never an element, id or attribute): U+2011 (non-breaking hyphen) by U+002D, which every face draws and a reader's search finds; U+FB01 (the fi ligature) by "fi"; U+02D9 (spacing dot above, in release 98 a radical's dot, "O2˙−") by U+2022; and U+05F3 (geresh, in release 98 a prime, "2׳-O methylation") by U+2032. Measured on release 98, every text the PDFs print but people's names: summations and event names (U+2011 on 197 lines, U+FB01 on 5, U+02D9 on 1), publications' titles, journals and chapters (U+2011 on 6, U+05F3 on 1), and the names of compartments, diseases, species and physical entities (none); and the 115,376 `displayName`s of the nodes, compartments, notes, shadows and links in the release's 1,390 human diagram files (`{stId}.json` only, not `.graph.json`; the 220 illustrated pathways' files among them, a superset of what is drawn) (none). People's names were not measured (the organisation's rule on personal information); the preflight checks them, as everything, when the book is made. Then each character no font covers is a fault `uncovered-character ` (T017's detector), checked twice (contracts/cli.md): before drawing, over the texts, the names the figures label (every physical entity a reaction's figure draws, inputs, outputs, catalysts and regulators, and their compartments, from the graph) and the labels of the diagram files the Figure rules choose to draw, read in `src/preflight/labels.ts`; and after drawing, before each part is printed, over each SVG's label text, in T022's `src/figures/labels.ts`, extended here with the book's fault mode, which also makes the substitutions there after T014's id prefixing. Only the substitutions made in what is printed are counted for the run report, not the preflight's own. + - **Characters (FR-017):** first FR-017's substitutions, in T022's `src/layout/characters.ts` (with T017's detector), which T024 extends with the book's fault mode, applied to every text the layout writes and to the text of every inlined figure's labels (their text content only, never an element, id or attribute): U+2011 (non-breaking hyphen) by U+002D, which every face draws and a reader's search finds; U+FB01 (the fi ligature) by "fi"; U+02D9 (spacing dot above, in release 98 a radical's dot, "O2˙−") by U+2022; and U+05F3 (geresh, in release 98 a prime, "2׳-O methylation") by U+2032. Measured on release 98, every text the PDFs print but people's names: summations and event names (U+2011 on 197 lines, U+FB01 on 5, U+02D9 on 1), publications' titles, journals and chapters (U+2011 on 6, U+05F3 on 1), and the names of compartments, diseases, species and physical entities (none); and the 115,376 `displayName`s of the nodes, compartments, notes, shadows and links in the release's 1,390 human diagram files (`{stId}.json` only, not `.graph.json`; the 220 illustrated pathways' files among them, a superset of what is drawn) (none). People's names were not measured (the organisation's rule on personal information); the preflight checks them, as everything, when the book is made. Then each character no font covers is a fault `uncovered-character ` (T017's detector), checked twice (contracts/cli.md): before drawing, over the texts, the names the figures label (every physical entity a reaction's figure draws, inputs, outputs, catalysts and regulators, and their compartments, from the graph) and the labels of the diagram files the Figure rules choose to draw (the `displayName`s of their nodes, compartments, notes, shadows and links, and the `label`s of their nodes' attachments, which the website's diagram draws as a modification's lettering), read in `src/preflight/labels.ts`; and after drawing, before each part is printed, over each SVG's label text, in T022's `src/figures/labels.ts`, extended here with the book's fault mode, which also makes the substitutions there after T014's id prefixing. Only the substitutions made in what is printed are counted for the run report, not the preflight's own. - **File names:** a top-level pathway's name holding `/` or NUL, or over 255 bytes with `.pdf`, or two names equal ignoring case, is a fault `bad-file-name ` (data-model Book); tested with a planted name of each kind. - Any fault means exit 1, with no book written (Story 5, scenarios 3 and 4). @@ -398,6 +399,9 @@ passes. - a substitution in an inlined figure leaves its elements, ids and attributes as they were; - the fixture summary's entry for an absent event is not a fault. + - **Done (#150, replacing #149 after its review):** the preflight is `src/preflight/`, its checks exported one by one for T029 and T046, and `pdf-export preflight` (`src/cli.ts`, `src/preflight/command.ts`) runs it alone with `book`'s flags, one `fault ` line each, exit 0, 1 or 2 (contracts/cli.md gains the command). `preflight.ts` reads the EHLD summary (its absence the one fault `missing-file ehld/svgsummary.txt`, as the Figure rules cannot be followed without it), builds the book (T048's `buildBook`) and runs, in order: the file names (`names.ts`: `/`, NUL, over 255 bytes with `.pdf`, counted in bytes, and every name of a group equal ignoring case); the release's files (`files.ts`: `diagram/{stId}.json` and `.graph.json` for a figure drawn from a diagram, its own or an ancestor's, `ehld/{stId}.svg` for an illustration, none for a reaction, from the Figure rules over the graph's events, so the summary's `R-HSA-99999999` is never looked for); the folder (`folder.ts`); and the characters before drawing (`characters.ts`). **The folder:** `folder.ts` holds the manifest's line formats for T046 (`manifestLine`, each a JSON object with a `type`: `fingerprint`, `figure`, `complete`; contracts/figures.md now gives them), the expected figures (`expectedFigures`, one per event with a figure, by its stable id, from the built book) and `checkFolder`: `no-checksum`; then `incomplete-folder` for no manifest, a torn last line, a line not a JSON object of its type, no fingerprint first, a fingerprint or `complete` anywhere else, no `complete` last, a `complete` differing from the first line, or two lines for one figure; `stale-folder` and `stale-summary` (the summary's hash is `sha256:` and the hex digest of its bytes); and only on a whole, current folder, each figure's line (`stale-figure`: none, or another kind, source or selection; or a line for an event with no figure) and file (`missing-file {stId}.svg`). A folder refused as a whole is not checked figure by figure, so a stale folder is a line, not thousands. **The checksum:** `releaseInfo` (`src/graph/queries.ts`, beside `release`, which is unchanged) reads `toString(d.checksum)`, as a long can pass 2^53, where the driver's numbers lose digits; the fixture's is 2^53 + 1, and read as a number it fails the reader's test. **Characters:** `src/layout/characters.ts` gains `BookCharacters`, the book's mode (appended, nothing before it changed): each substitution made and counted by code point (`replaced`, for T029's report: the preflight's own instance is discarded), and any other character no font draws a fault `uncovered-character `, kept as it is; `text`, `markupText` (numeric references too), `markup` and `strings` (a summation as markup), as the on-request mode has. `src/figures/labels.ts` gains `labelsInBook` (an SVG's text alone, after T014's prefixing, its fault naming the figure's event), for T029 and T046 after drawing. `src/preflight/characters.ts` checks before drawing: every string of each section (naming its event), each cover (its PDF's top-level pathway) and each short entry (its event); the names the reactions' figures label, from a new reader query, `figureNames` (each reaction's inputs, outputs, catalysts' and regulators' entities, by display name and names, and their compartments), each blamed on the first event in document order whose own figure is its reaction's; and the labels of the diagram files the Figure rules draw (`src/preflight/labels.ts`: the `displayName`s of `nodes`, `compartments`, `notes`, `shadows` and `links`), naming the diagram's pathway, an illustrated pathway's file and a non-human one's never read. A character in several places is a fault for each stable id it is blamed on (a top-level pathway's name in its sections' locations, too). A diagram file that is not JSON is a fault, `bad-file ` (from #149's review, below). `openGraph` gains `retryTimeMs` (the driver's `maxTransactionRetryTime`), 0 for the command's first read, which otherwise took 65 s to say a graph on a closed port could not be reached (its other reads keep the driver's default, from #149's review). **The fixture** gains `DBInfo.checksum` and `placeholder membrane`, a compartment only Reaction one's output holds (81 nodes, 128 relationships; reloaded once, as ci/fixture/README.md says, which lists both). **Tests:** `test/preflight/preflight.test.ts` (13, on the fixture, with a figures' folder each test writes and characters planted through a graph that rewrites its rows, so the shared fixture is untouched), `test/preflight/units.test.ts` (10, no graph), `test/graph/reader.test.ts` (2 more). 304 tests before, 329 after. Each of these mutants made one fail: a torn last line read; two lines for one figure allowed; `complete` not compared with the first line; other lines in the middle skipped (a second fingerprint, a `complete` in the middle); the checksum, the release or the summary not compared; `no-checksum` not raised; the selection, or the kind, not compared; a line for no figure allowed; figure files not looked for; the checksum read as a number; every summary entry looked for (the fixture's own run then fails); `.graph.json`, or illustrations, not looked for; participants read without their compartments (the membrane's test, and the reader's); the figures' names not checked; the last event blamed for a name; an illustration's diagram file read; diagram files not read; sections not checked; covers not checked (an undated edit's author, planted in the edits alone, is printed only on the cover, as a section's history leaves undated edits out); the substitutions not made, or not counted; numeric references not read; labels read as plain text (attributes then changed); a `/`, a NUL, characters counted for bytes, 256 bytes allowed, case not folded, and only the second of two equal names; the file names not checked by the preflight; faults exit 0; no retry bound (the 15 s bound); an unreachable graph not exit 2. Two survive as equivalent on any book: an ancestor's diagram file not read (an ancestor with a diagram draws its own figure from the same file, and is in the book above the pathway it draws), and the short entries' strings not checked (a short entry's name and location are printed again in its event's full section, at its first place in the same PDF). Left for T029 and T046: wiring `labelsInBook` after drawing, and the run report's counts. + - **From #149's review:** (1) an error during the preflight was exit 1 with no fault line, the faults already found lost (a diagram file not JSON, a graph failing after its retries, `buildBook`'s "not read from the graph"); exit 1 is to mean faults. A diagram file that cannot be read as JSON is now a fault, `bad-file ` (contracts/cli.md), its labels unchecked and the run going on; any other error ends the command with exit 2, its error on stderr, after the faults found before it are printed (`preflight` adds each check's faults to a list its caller holds). Both seen failing first: a corrupt `diagram/R-HSA-90000130.json` with an empty `--figures` gave exit 1 and nothing printed, and a graph failing at the reactions' names exit 1 with `incomplete-folder` lost. (2) `retryTimeMs: 0` still makes one retry, and it applied to every read: the start's read now has a driver of its own with it (said within seconds), and the checks' reads the driver's default; the comment and contract say "bounds the retrying". (3) The website's diagram draws a node's attachments' `label`s (a modification's lettering); `src/preflight/labels.ts` now reads them. `figureNames` does not read a reaction figure's equivalent (its entities' modified residues' PSI-MOD labels), recorded, not built: the fixture has no modification to test it on. Recorded too: a normal pathway's diagram drawn beneath a disease diagram (`forNormalDraw` false) is not read apart from its file. (4) Tests: a stale folder that also has a figure with no line and a missing file gives the one fault; a selection that is the right one's beginning is `stale-figure`; a figure line whose selection is not all strings is not whole. (5) Recorded, not changed: the fingerprint's `content` is not compared with the graph's release by `book`, defence in depth left to T046's check before drawing; "ignoring case" is `toLowerCase`, not full case folding or Unicode normalisation (final sigma, ß, NFC against NFD); the text check reads strings that are never printed (web addresses, and whatever the model holds beside what the layout writes), so a fault can be false; T047 needs the fingerprint compared and `none` told from `incomplete` (now in T047's text); a truncated `.graph.json` is not found, only its presence checked. 347 tests before (on main with T027), 350 after. Each of these mutants made one fail: the refusal as a whole removed (the stale folder's test); the selection's length not compared (the beginning's case); the selection's ids not checked as strings (the unit case); attachments not read; a file not JSON thrown again (the bad-file test); the error not caught (the exit 2 test); the faults found not printed on an error (the same); the start's read without its bound (the 15 s bound). Of the earlier mutants, 26 were run again on this code and each still fails a test; the other 12 (among them the file-name rule's, the substitutions' and the unreachable graph's) were not re-run; the two equivalents still survive. + - [ ] T046 [US1] `pdf-export figures`, `src/figures/step.ts` and `src/cli.ts` (contracts/cli.md, contracts/figures.md, SC-010). - Lists the book's figures by the Figure rules (T013's `diagramChoices` with `nearestAncestor` from `src/model/build.ts`, and the EHLD summary from `--files`: the same code the book uses), and draws each through T014's client into `--out`, `{stId}.svg`, written beside its name and moved into place. - After T024: runs its release-file and pre-drawing character checks first (`src/preflight/`), and the label check after each figure is drawn. diff --git a/src/cli.ts b/src/cli.ts index 0ca0531..8840813 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node // pdf-export's command line (contracts/cli.md). Its commands come with spec 001's tasks: -// serve (T022) runs the single-event service; book, check-book and compare are to come. +// serve (T022) runs the single-event service; preflight (T024) runs the book's preflight +// alone; book, check-book and compare are to come. export const USAGE = `Usage: pdf-export [options] @@ -8,6 +9,7 @@ Commands: book write the release's book (spec 001, User Story 1) check-book check a book for FR-011's faults compare compare with event-pdf's output, by meaning (FR-012) + preflight check the book's inputs, as book does before printing serve the website's single-event PDF service (User Story 2) `; @@ -48,6 +50,12 @@ if (import.meta.main) { const args = process.argv.slice(2); if (args[0] === 'serve') { process.exitCode = await serve(args.slice(1)); + } else if (args[0] === 'preflight') { + const { preflightCommand } = await import('./preflight/command.ts'); + process.exitCode = await preflightCommand(args.slice(1), process.env, { + out: (text) => process.stdout.write(text), + err: (text) => process.stderr.write(text), + }); } else { const { code, out } = run(args); (code === 0 ? process.stdout : process.stderr).write(out); diff --git a/src/figures/labels.ts b/src/figures/labels.ts index c3ff2e9..090500d 100644 --- a/src/figures/labels.ts +++ b/src/figures/labels.ts @@ -2,8 +2,14 @@ // in the text of its labels, and in a PDF made on request any other character no font draws // replaced by U+FFFD and logged in the PDF's one log, shared with its texts. Only the text // content changes, never an element, an id or an attribute, so the drawing is unchanged. -// T024 adds the book's mode, in which such a character is a fault, after T014's id prefixing. -import { markupText, onRequestMarkupText, type CharacterLog } from '../layout/characters.ts'; +// In the book's mode (T024) such a character is a fault naming the figure's event; it is +// applied to the SVG after T014's id prefixing, as the figure is inlined. +import { + markupText, + onRequestMarkupText, + type BookCharacters, + type CharacterLog, +} from '../layout/characters.ts'; import { coveredCodePoints } from '../layout/fonts.ts'; /** A figure's SVG with its labels as a PDF made on request prints them. */ @@ -11,3 +17,12 @@ export function labelsOnRequest(svg: string, log: CharacterLog): string { const covered = coveredCodePoints(); return markupText(svg, (text) => onRequestMarkupText(text, log, covered)); } + +/** + * A figure's SVG with its labels as the book prints them: the substitutions made and counted, + * and any other character no font draws a fault in `characters`, naming `stId`, the event + * whose section shows the figure. The caller refuses the figure while it has a fault. + */ +export function labelsInBook(svg: string, stId: string, characters: BookCharacters): string { + return characters.markup(svg, stId); +} diff --git a/src/graph/connection.ts b/src/graph/connection.ts index e7e055a..a7bc1ad 100644 --- a/src/graph/connection.ts +++ b/src/graph/connection.ts @@ -45,6 +45,12 @@ export interface GraphOptions { * hint): a caller that must not wait on a silent one bounds its own wait. */ connectionTimeoutMs?: number; + /** + * How long the driver goes on retrying a read that failed for want of a connection, from + * its first failure (the driver's `maxTransactionRetryTime`); the driver's own default + * (30 s) without it. Even 0 makes one retry: it bounds the retrying, it does not end it. + */ + retryTimeMs?: number; } export function openGraph(settings: GraphSettings, options: GraphOptions = {}): Graph { @@ -60,6 +66,9 @@ export function openGraph(settings: GraphSettings, options: GraphOptions = {}): connectionTimeout: options.connectionTimeoutMs, connectionAcquisitionTimeout: options.connectionTimeoutMs, }), + ...(options.retryTimeMs === undefined + ? {} + : { maxTransactionRetryTime: options.retryTimeMs }), } ); return { diff --git a/src/graph/queries.ts b/src/graph/queries.ts index b3275e6..a4c2bd0 100644 --- a/src/graph/queries.ts +++ b/src/graph/queries.ts @@ -93,6 +93,25 @@ export async function release(graph: Graph): Promise { return row.release; } +/** The release's number and checksum, from its `DBInfo` (spec 001 T024). */ +export interface ReleaseInfo { + release: number; + /** + * The graph's `DBInfo.checksum`, as its decimal digits: a long, which a JavaScript number + * cannot hold whole past 2^53. Null where the graph has none. + */ + checksum: string | null; +} + +/** The release's number and checksum, for the figures' folder's fingerprint (T024, T046). */ +export async function releaseInfo(graph: Graph): Promise { + const [row] = await graph.read<{ release: number | null; checksum: string | null }>( + 'MATCH (d:DBInfo) RETURN d.releaseNumber AS release, toString(d.checksum) AS checksum LIMIT 1' + ); + if (row?.release == null) throw new Error('The graph has no DBInfo.releaseNumber'); + return { release: row.release, checksum: row.checksum ?? null }; +} + /** A species' top-level pathways, by database id: the book orders them by name (FR-002). */ export async function topLevelPathways(graph: Graph, species: string): Promise { return graph.read( @@ -425,3 +444,43 @@ export async function parentPathways(graph: Graph, id: number): Promise (a < b ? -1 : a > b ? 1 : 0); + +/** + * The names each reaction's figure labels (spec 001 T024, FR-017): its participants, the + * inputs, outputs, catalysts and regulators it draws, each by its display name and its + * names, and their compartments. Sorted, each once. The participants' compartments are read + * here, as no event's section names them. + */ +export async function figureNames( + graph: Graph, + ids: readonly number[] +): Promise> { + const rows = await graph.read<{ id: number; names: (string | null)[] }>( + `UNWIND $ids AS id + MATCH (r:DatabaseObject {dbId: id}) + WHERE r:ReactionLikeEvent + CALL { + WITH r + MATCH (r)-[:input|output]->(pe:PhysicalEntity) RETURN pe + UNION + WITH r + MATCH (r)-[:catalystActivity]->(:CatalystActivity)-[:physicalEntity]->(pe:PhysicalEntity) + RETURN pe + UNION + WITH r + MATCH (r)-[:regulatedBy]->(:Regulation)-[:regulator]->(pe:PhysicalEntity) RETURN pe + } + OPTIONAL MATCH (pe)-[:compartment]->(c) + WITH id, pe, collect(c.displayName) AS compartments + WITH id, [pe.displayName] + coalesce(pe.name, []) + compartments AS names + UNWIND names AS name + RETURN id, collect(DISTINCT name) AS names`, + { ids: unique(ids) } + ); + return new Map( + rows.map(({ id, names }) => [ + id, + names.filter((name): name is string => typeof name === 'string').sort(byCodeUnit), + ]) + ); +} diff --git a/src/layout/characters.ts b/src/layout/characters.ts index 78ac1cd..67c9a16 100644 --- a/src/layout/characters.ts +++ b/src/layout/characters.ts @@ -192,3 +192,92 @@ export function onRequestStrings(value: T, log: CharacterLog): T { }; return visit(value, false) as T; } + +/** + * FR-017 in the book (spec 001 T024): the substitutions made, each counted by code point for + * the run report, and any other character no font draws a fault, + * `uncovered-character `, naming the subject given with its text, never the + * text. Such a character is left as it is: the book is not written while there is a fault. + * One instance per purpose: the preflight's own counts are not the run report's. + */ +export class BookCharacters { + readonly #faults = new Set(); + readonly #replaced = new Map(); + readonly #covered: ReadonlySet; + + constructor(covered: ReadonlySet = coveredCodePoints()) { + this.#covered = covered; + } + + /** The faults met so far, each once, in the order met. */ + get faults(): string[] { + return [...this.#faults]; + } + + /** How often each listed character was replaced. */ + get replaced(): ReadonlyMap { + return this.#replaced; + } + + /** A character as the book prints it; `null` when it is printed as it is. */ + #char(char: string, subject: string): string | null { + const point = char.codePointAt(0) ?? 0; + const substitute = SUBSTITUTIONS.get(point); + if (substitute !== undefined) { + this.#replaced.set(point, (this.#replaced.get(point) ?? 0) + 1); + return substitute; + } + if (!this.#covered.has(point) && !NEVER_DRAWN.test(char)) { + this.#faults.add(`uncovered-character ${codePointName(point)} ${subject}`); + } + return null; + } + + /** A plain text as the book prints it. */ + text(text: string, subject: string): string { + if (PLAIN.test(text)) return text; + let out = ''; + for (const char of text) out += this.#char(char, subject) ?? char; + return out; + } + + /** A run of markup's text as the book prints it, its numeric references included. */ + markupText(text: string, subject: string): string { + const referenced = text.includes('&#') + ? text.replace(REFERENCE, (whole, hex: string | undefined, dec: string | undefined) => { + const point = hex === undefined ? Number(dec) : parseInt(hex, 16); + if (point > 0x10ffff) return whole; + return this.#char(String.fromCodePoint(point), subject) ?? whole; + }) + : text; + return this.text(referenced, subject); + } + + /** Markup (a summation, a figure's SVG) as the book prints it: its text alone changed. */ + markup(markup: string, subject: string): string { + return markupText(markup, (text) => this.markupText(text, subject)); + } + + /** + * Every string in a value as the book prints it: a summation's as markup, every other as + * plain text. Objects, lists and maps are copied. + */ + strings(value: T, subject: string): T { + const visit = (item: unknown, markup: boolean): unknown => { + if (typeof item === 'string') { + return markup ? this.markup(item, subject) : this.text(item, subject); + } + if (Array.isArray(item)) return item.map((element: unknown) => visit(element, markup)); + if (item instanceof Map) { + return new Map([...item].map(([key, element]) => [key, visit(element, false)])); + } + if (item !== null && typeof item === 'object') { + return Object.fromEntries( + Object.entries(item).map(([key, element]) => [key, visit(element, key === 'summations')]) + ); + } + return item; + }; + return visit(value, false) as T; + } +} diff --git a/src/preflight/characters.ts b/src/preflight/characters.ts new file mode 100644 index 0000000..b0e803d --- /dev/null +++ b/src/preflight/characters.ts @@ -0,0 +1,107 @@ +// FR-017 before anything is drawn or printed (spec 001 T024, contracts/cli.md): after the +// list's substitutions, a character no font draws is a fault +// `uncovered-character `, sought in three sources: +// - the texts the layout writes, each section's naming its event, a cover's naming its PDF's +// top-level pathway, a short entry's naming its event; +// - the names the reactions' figures label (src/graph/queries.ts `figureNames`: their +// participants and the participants' compartments), naming the first event in document +// order whose figure holds the name; +// - the labels of the diagram files the Figure rules choose to draw, naming the pathway whose +// diagram it is. An illustration's own diagram file is not drawn, so not read. A file that +// cannot be read as JSON is a fault `bad-file `, after the characters' faults. +// The second half of the check, over each drawn figure's labels, is src/figures/labels.ts's. +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +import type { Graph } from '../graph/connection.ts'; +import { figureNames } from '../graph/queries.ts'; +import { BookCharacters } from '../layout/characters.ts'; +import type { Document } from '../model/types.ts'; +import type { FigureRecord } from './folder.ts'; +import { readDiagramLabels } from './labels.ts'; + +/** How many reactions' names are asked for in one read. */ +const CHUNK = 2000; + +/** The texts' faults, each PDF in order: its cover, then its places. */ +export function textFaults(docs: readonly Document[], characters: BookCharacters): void { + for (const doc of docs) { + const [first] = doc.places; + const root = first ? (first.full ? first.section.event.stId : first.event.stId) : null; + if (root) characters.strings(doc.cover, root); + for (const place of doc.places) { + if (place.full) characters.strings(place.section, place.section.event.stId); + else characters.strings({ event: place.event, path: place.path }, place.event.stId); + } + } +} + +/** + * The names the reactions' figures label, each with the first event in document order whose + * figure holds it: only a reaction's own figure is drawn from these names. + */ +export function namesInOrder( + docs: readonly Document[], + names: ReadonlyMap +): Map { + const first = new Map(); + for (const doc of docs) { + for (const place of doc.places) { + if (!place.full || place.section.figure?.kind !== 'reaction') continue; + const { dbId, stId } = place.section.event; + for (const name of names.get(dbId) ?? []) if (!first.has(name)) first.set(name, stId); + } + } + return first; +} + +/** The reactions drawn from the graph: those whose own figure is their reaction's. */ +function drawnReactions(docs: readonly Document[]): number[] { + const ids = new Set(); + for (const doc of docs) { + for (const place of doc.places) { + if (place.full && place.section.figure?.kind === 'reaction') { + ids.add(place.section.event.dbId); + } + } + } + return [...ids]; +} + +/** + * Every fault FR-017 finds before drawing, each once, in the order found: the texts', the + * figures' names', then the diagram files', then the files unread (a file missing is the + * files' check's fault). + */ +export async function checkCharacters( + graph: Graph, + docs: readonly Document[], + { files, expected }: { files: string; expected: Iterable } +): Promise { + const characters = new BookCharacters(); + textFaults(docs, characters); + + const reactions = drawnReactions(docs); + const names = new Map(); + for (let i = 0; i < reactions.length; i += CHUNK) { + for (const [id, list] of await figureNames(graph, reactions.slice(i, i + CHUNK))) { + names.set(id, list); + } + } + for (const [name, stId] of namesInOrder(docs, names)) characters.text(name, stId); + + const unreadable: string[] = []; + const diagrams = new Set(); + for (const { kind, source } of expected) { + if (kind === 'diagram' || kind === 'ancestor') diagrams.add(source); + } + for (const stId of diagrams) { + const name = `diagram/${stId}.json`; + const path = join(files, name); + if (!existsSync(path)) continue; + const labels = readDiagramLabels(path); + if (labels === null) unreadable.push(`bad-file ${name}`); + for (const label of labels ?? []) characters.text(label, stId); + } + return [...characters.faults, ...unreadable]; +} diff --git a/src/preflight/command.ts b/src/preflight/command.ts new file mode 100644 index 0000000..a70336d --- /dev/null +++ b/src/preflight/command.ts @@ -0,0 +1,111 @@ +// `pdf-export preflight` (spec 001 T024, contracts/cli.md): the book's preflight alone, with +// the book's flags, so a release's inputs can be checked before the book is made. `book` +// (T029) runs the same checks first. Exit 0 with no fault, 1 with any (one line each, +// `fault `), 2 on bad arguments, a graph that cannot be reached at the start, or any +// other error during the checks, after printing the faults found before it. +import { + graphSettings, + openGraph, + type Graph, + type GraphOptions, + type GraphSettings, +} from '../graph/connection.ts'; +import { releaseInfo, type ReleaseInfo } from '../graph/queries.ts'; +import { preflight, type PreflightSettings } from './preflight.ts'; + +export class SettingsError extends Error { + override name = 'SettingsError'; +} + +const FLAGS = ['--graph', '--database', '--files', '--figures', '--species']; + +/** The settings from the flags and the environment; the credentials from the environment. */ +export function preflightSettings( + args: readonly string[], + env: Readonly> +): PreflightSettings & { graph: GraphSettings } { + const flags = new Map(); + for (let i = 0; i < args.length; i++) { + const [name = '', inline] = (args[i] ?? '').split(/=(.*)/s, 2); + if (!FLAGS.includes(name)) throw new SettingsError(`Unknown argument: ${name}`); + const value = inline ?? args[++i]; + if (value === undefined || value === '') throw new SettingsError(`${name} needs a value`); + if (flags.has(name)) throw new SettingsError(`${name} is given twice`); + flags.set(name, value); + } + const required = (flag: string): string => { + const value = flags.get(flag); + if (value === undefined) throw new SettingsError(`${flag} is needed`); + return value; + }; + const graphEnv = { ...env }; + const uri = flags.get('--graph'); + if (uri !== undefined) graphEnv['NEO4J_URI'] = uri; + const database = flags.get('--database'); + if (database !== undefined) graphEnv['NEO4J_DATABASE'] = database; + let graph: GraphSettings; + try { + graph = graphSettings(graphEnv); + } catch { + throw new SettingsError('--graph (or NEO4J_URI) is needed'); + } + return { + graph, + files: required('--files'), + figures: required('--figures'), + species: flags.get('--species') ?? 'Homo sapiens', + }; +} + +/** `pdf-export preflight`: its exit code; faults to `out`, errors to `err`. */ +export async function preflightCommand( + args: readonly string[], + env: Readonly>, + { + out, + err, + open = openGraph, + }: { + out: (text: string) => void; + err: (text: string) => void; + /** How the graph is opened (a test gives one that fails). */ + open?: (settings: GraphSettings, options?: GraphOptions) => Graph; + } +): Promise { + let settings; + try { + settings = preflightSettings(args, env); + } catch (error) { + if (!(error instanceof SettingsError)) throw error; + err(`${error.message}\n`); + return 2; + } + // the start's check, its own driver retrying a failed read briefly, so a graph that cannot + // be reached is said within seconds, not after the driver's default half minute + const start = open(settings.graph, { connectionTimeoutMs: 5000, retryTimeMs: 0 }); + let release: ReleaseInfo; + try { + release = await releaseInfo(start); + } catch (error) { + err(`The graph cannot be read: ${message(error)}\n`); + return 2; + } finally { + await start.close(); + } + // the checks' reads with the driver's own retries + const graph = open(settings.graph); + const faults: string[] = []; + try { + await preflight(graph, release, settings, faults); + return faults.length > 0 ? 1 : 0; + } catch (error) { + err(`The preflight stopped: ${message(error)}\n`); + return 2; + } finally { + for (const fault of faults) out(`fault ${fault}\n`); + await graph.close(); + } +} + +const message = (error: unknown): string => + error instanceof Error ? error.message : String(error); diff --git a/src/preflight/files.ts b/src/preflight/files.ts new file mode 100644 index 0000000..47b96c6 --- /dev/null +++ b/src/preflight/files.ts @@ -0,0 +1,60 @@ +// The release's files the figures need (spec 001 T024, research R7): which events need which +// file comes from the graph's events, through the Figure rules; the EHLD summary only says +// which of them are illustrations, so its entry for an event the graph does not hold is +// never looked for. A missing file is a fault `missing-file `, named as in `--files`. +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import type { FigureRecord } from './folder.ts'; + +/** The EHLD summary, in the release's folder. */ +export const SUMMARY = 'ehld/svgsummary.txt'; + +/** The EHLD summary: the pathways it names, and its hash for the folder's fingerprint. */ +export interface Summary { + illustrated: Set; + /** `sha256:` and the hex digest of the file's bytes. */ + hash: string; +} + +/** The release's EHLD summary, or null where the folder has none. */ +export function readSummary(files: string): Summary | null { + const path = join(files, SUMMARY); + if (!existsSync(path)) return null; + const bytes = readFileSync(path); + return { + illustrated: new Set( + bytes + .toString('utf8') + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith('R-')) + ), + hash: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, + }; +} + +/** + * The release's files the figures draw from, each once, sorted: a diagram's `{stId}.json` + * and `{stId}.graph.json` for a figure drawn from a diagram (its own or an ancestor's), an + * illustration's `{stId}.svg` for one drawn from an illustration. A reaction is drawn from + * the graph alone. + */ +export function neededFiles(figures: Iterable): string[] { + const names = new Set(); + for (const { kind, source } of figures) { + if (kind === 'diagram' || kind === 'ancestor') { + names.add(`diagram/${source}.json`); + names.add(`diagram/${source}.graph.json`); + } else if (kind === 'ehld') { + names.add(`ehld/${source}.svg`); + } + } + return [...names].sort(); +} + +/** Each needed file missing from the release's folder, as a fault. */ +export function checkFiles(files: string, names: readonly string[]): string[] { + return names.filter((name) => !existsSync(join(files, name))).map((n) => `missing-file ${n}`); +} diff --git a/src/preflight/folder.ts b/src/preflight/folder.ts new file mode 100644 index 0000000..ff7b212 --- /dev/null +++ b/src/preflight/folder.ts @@ -0,0 +1,182 @@ +// The figures' folder (spec 001 T024, contracts/figures.md): what `figures` (T046) writes and +// what `book` checks before anything is printed. The manifest, `figures.jsonl`, is a line of +// JSON per record: the fingerprint first, then a line per figure, then `complete`, whose +// fingerprint fields equal the first line's. The line formats are here, for T046 to write. +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import type { ReleaseInfo } from '../graph/queries.ts'; +import type { Document, Figure } from '../model/types.ts'; + +export const MANIFEST = 'figures.jsonl'; + +/** What a folder was drawn from (contracts/figures.md). */ +export interface Fingerprint { + /** The graph's `DBInfo.releaseNumber`. */ + release: number; + /** The graph's `DBInfo.checksum`, as its decimal digits. */ + checksum: string; + /** The EHLD summary's hash, `summaryHash` (src/preflight/files.ts). */ + summary: string; + /** The release the content service answered. */ + content: number; + /** The render service's `build`. */ + build: string; +} + +/** What one event's figure draws: its file is `{stId}.svg`. */ +export interface FigureRecord { + /** The event whose section shows it. */ + stId: string; + kind: Figure['kind']; + /** The stable identifier drawn. */ + source: string; + /** The ids selected, sorted; none but for `ancestor`. */ + highlight: string[]; +} + +export type ManifestLine = + | ({ type: 'fingerprint' } & Fingerprint) + | ({ type: 'figure' } & FigureRecord) + | ({ type: 'complete' } & Fingerprint); + +/** A manifest line as `figures` appends it, its newline included. */ +export function manifestLine(line: ManifestLine): string { + return `${JSON.stringify(line)}\n`; +} + +/** An event's figure as the folder records it. */ +export function figureRecord(stId: string, figure: Figure): FigureRecord { + return { + stId, + kind: figure.kind, + source: figure.stId, + highlight: figure.kind === 'ancestor' ? [...figure.highlight].sort() : [], + }; +} + +/** + * The figures the Figure rules choose for the book: one per event with a figure, by the + * event's stable identifier, in document order. An event in two of the book's PDFs has the + * same figure in each, as its figure depends on the event alone. + */ +export function expectedFigures(docs: readonly Document[]): Map { + const expected = new Map(); + for (const doc of docs) { + for (const place of doc.places) { + if (!place.full || !place.section.figure) continue; + const { stId } = place.section.event; + if (!expected.has(stId)) expected.set(stId, figureRecord(stId, place.section.figure)); + } + } + return expected; +} + +const FINGERPRINT_FIELDS = ['release', 'checksum', 'summary', 'content', 'build'] as const; +const KINDS: readonly string[] = ['diagram', 'ancestor', 'ehld', 'reaction']; + +const isFingerprint = (o: Record): boolean => + Number.isInteger(o['release']) && + typeof o['checksum'] === 'string' && + typeof o['summary'] === 'string' && + Number.isInteger(o['content']) && + typeof o['build'] === 'string'; + +const isFigure = (o: Record): boolean => + typeof o['stId'] === 'string' && + typeof o['kind'] === 'string' && + KINDS.includes(o['kind']) && + typeof o['source'] === 'string' && + Array.isArray(o['highlight']) && + o['highlight'].every((id) => typeof id === 'string'); + +/** + * The manifest's lines, or null when it is not whole (`incomplete-folder`): a line that is + * not a JSON object of its type, a last line torn (no newline), no fingerprint first, a + * fingerprint or `complete` anywhere else, no `complete` last, a `complete` whose fingerprint + * differs from the first line's, or two lines for one stable identifier. + */ +export function readManifest( + text: string +): { fingerprint: Fingerprint; figures: Map } | null { + if (!text.endsWith('\n')) return null; + const lines: Record[] = []; + for (const raw of text.slice(0, -1).split('\n')) { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + lines.push(parsed as Record); + } + const first = lines[0]; + const last = lines.at(-1); + if (lines.length < 2 || !first || !last) return null; + if (first['type'] !== 'fingerprint' || !isFingerprint(first)) return null; + if (last['type'] !== 'complete' || !isFingerprint(last)) return null; + if (FINGERPRINT_FIELDS.some((field) => first[field] !== last[field])) return null; + const figures = new Map(); + for (const line of lines.slice(1, -1)) { + if (line['type'] !== 'figure' || !isFigure(line)) return null; + const record = line as unknown as FigureRecord; + if (figures.has(record.stId)) return null; + figures.set(record.stId, { + stId: record.stId, + kind: record.kind, + source: record.source, + highlight: record.highlight, + }); + } + const fingerprint = Object.fromEntries( + FINGERPRINT_FIELDS.map((field) => [field, first[field]]) + ) as unknown as Fingerprint; + return { fingerprint, figures }; +} + +const sameFigure = (a: FigureRecord, b: FigureRecord): boolean => + a.kind === b.kind && + a.source === b.source && + a.highlight.length === b.highlight.length && + a.highlight.every((id, i) => id === b.highlight[i]); + +/** + * The folder's faults against the book's graph and EHLD summary (contracts/figures.md, + * "What `book` checks"), each a fault line. A folder that is not whole, or not this + * release's or summary's, is refused as a whole: its figures are not checked one by one. + */ +export function checkFolder( + folder: string, + { + release, + summary, + expected, + }: { release: ReleaseInfo; summary: string; expected: ReadonlyMap } +): string[] { + const faults: string[] = []; + if (release.checksum === null) faults.push('no-checksum'); + const path = join(folder, MANIFEST); + const manifest = existsSync(path) ? readManifest(readFileSync(path, 'utf8')) : null; + if (!manifest) return [...faults, 'incomplete-folder']; + const { fingerprint, figures } = manifest; + if ( + release.checksum !== null && + (fingerprint.release !== release.release || fingerprint.checksum !== release.checksum) + ) { + faults.push( + `stale-folder ${String(fingerprint.release)}/${fingerprint.checksum} ${String(release.release)}/${release.checksum}` + ); + } + if (fingerprint.summary !== summary) faults.push('stale-summary'); + if (faults.length > 0) return faults; + for (const [stId, wanted] of expected) { + const line = figures.get(stId); + if (!line || !sameFigure(line, wanted)) faults.push(`stale-figure ${stId}`); + if (!existsSync(join(folder, `${stId}.svg`))) faults.push(`missing-file ${stId}.svg`); + } + for (const stId of figures.keys()) { + if (!expected.has(stId)) faults.push(`stale-figure ${stId}`); + } + return faults; +} diff --git a/src/preflight/labels.ts b/src/preflight/labels.ts new file mode 100644 index 0000000..1d5a799 --- /dev/null +++ b/src/preflight/labels.ts @@ -0,0 +1,46 @@ +// The labels of a release's diagram file (spec 001 T024, FR-017): the text a diagram drawn +// from `diagram/{stId}.json` can print, read before anything is drawn: the `displayName`s of +// its nodes, compartments, notes, shadows and links (measured for T024 on release 98), and +// the `label` of each node's attachments, which the website's diagram draws as a +// modification's lettering; `{stId}.graph.json` holds no label. +import { readFileSync } from 'node:fs'; + +/** The lists of a diagram file whose items carry a label. */ +const LABELLED = ['nodes', 'compartments', 'notes', 'shadows', 'links'] as const; + +/** Each label in a diagram file's JSON, in the file's order. */ +export function diagramLabels(json: unknown): string[] { + if (json === null || typeof json !== 'object') return []; + const labels: string[] = []; + for (const list of LABELLED) { + const items: unknown = (json as Record)[list]; + if (!Array.isArray(items)) continue; + for (const item of items as unknown[]) { + if (item === null || typeof item !== 'object') continue; + const name: unknown = (item as Record)['displayName']; + if (typeof name === 'string') labels.push(name); + const attachments: unknown = (item as Record)['nodeAttachments']; + if (!Array.isArray(attachments)) continue; + for (const attachment of attachments as unknown[]) { + if (attachment === null || typeof attachment !== 'object') continue; + const label: unknown = (attachment as Record)['label']; + if (typeof label === 'string') labels.push(label); + } + } + } + return labels; +} + +/** + * A diagram file's labels, or null when it cannot be read or is not JSON: the caller's fault + * `bad-file `. + */ +export function readDiagramLabels(path: string): string[] | null { + let json: unknown; + try { + json = JSON.parse(readFileSync(path, 'utf8')); + } catch { + return null; + } + return diagramLabels(json); +} diff --git a/src/preflight/names.ts b/src/preflight/names.ts new file mode 100644 index 0000000..79eb8cb --- /dev/null +++ b/src/preflight/names.ts @@ -0,0 +1,32 @@ +// The book's file names (spec 001 T024, data-model Book): each PDF is named as today's, its +// top-level pathway's name and `.pdf`. A name a file system cannot hold (one holding `/` or +// NUL, or over 255 bytes with `.pdf`), or two names equal ignoring case, which a file system +// that ignores case would write as one file, is a fault `bad-file-name `. + +/** The longest file name, in bytes, that common file systems hold. */ +export const MAX_NAME_BYTES = 255; + +/** A top-level pathway, as its PDF is named. */ +export interface Named { + stId: string; + name: string; +} + +/** A PDF's file name: its pathway's name and `.pdf`. */ +export const fileName = (name: string): string => `${name}.pdf`; + +/** The faults in the book's file names, in the pathways' order; each stable id once. */ +export function checkFileNames(pathways: readonly Named[]): string[] { + const bad = new Set(); + const byFolded = new Map(); + for (const { stId, name } of pathways) { + const file = fileName(name); + if (file.includes('/') || file.includes('\0') || Buffer.byteLength(file) > MAX_NAME_BYTES) { + bad.add(stId); + } + const folded = file.toLowerCase(); + byFolded.set(folded, [...(byFolded.get(folded) ?? []), stId]); + } + for (const stIds of byFolded.values()) if (stIds.length > 1) for (const s of stIds) bad.add(s); + return pathways.filter((p) => bad.has(p.stId)).map((p) => `bad-file-name ${p.stId}`); +} diff --git a/src/preflight/preflight.ts b/src/preflight/preflight.ts new file mode 100644 index 0000000..110fa8a --- /dev/null +++ b/src/preflight/preflight.ts @@ -0,0 +1,65 @@ +// The book's preflight (spec 001 T024, contracts/cli.md): every check made before anything is +// printed, each fault a line naming a stable identifier or a file, never a text or a person. +// Any fault means the book is not written (exit 1). Its checks are exported one by one for +// `figures` (T046), which runs the release files' and the characters' before drawing. +import type { Graph } from '../graph/connection.ts'; +import type { ReleaseInfo } from '../graph/queries.ts'; +import { buildBook } from '../model/build.ts'; +import type { Document } from '../model/types.ts'; +import { checkCharacters } from './characters.ts'; +import { checkFiles, neededFiles, readSummary, SUMMARY } from './files.ts'; +import { checkFolder, expectedFigures } from './folder.ts'; +import { checkFileNames } from './names.ts'; + +export interface PreflightSettings { + species: string; + /** The release's folder: `diagram/`, `ehld/` and the EHLD summary. */ + files: string; + /** The figures' folder `figures` wrote. */ + figures: string; +} + +export interface Preflight { + /** Each fault, in the order the checks run: names, files, folder, characters. */ + faults: string[]; + /** The book's PDFs, as built for the checks; none when the summary is missing. */ + docs: Document[] | null; +} + +/** The top-level pathways the book's PDFs are named by: each PDF's first place. */ +export function bookPathways(docs: readonly Document[]): { stId: string; name: string }[] { + return docs.flatMap((doc) => { + const [first] = doc.places; + if (!first) return []; + return [first.full ? first.section.event : first.event].map(({ stId, name }) => ({ + stId, + name, + })); + }); +} + +/** + * The book's preflight, against its graph (whose release was read at the start). Each check's + * faults are added to `faults` as it ends, so a caller met by an error (a graph that fails + * mid-run) still has those found before it. + */ +export async function preflight( + graph: Graph, + release: ReleaseInfo, + { species, files, figures }: PreflightSettings, + faults: string[] = [] +): Promise { + const summary = readSummary(files); + // without the summary the Figure rules cannot be followed, so nothing more is checked + if (!summary) { + faults.push(`missing-file ${SUMMARY}`); + return { faults, docs: null }; + } + const docs = await buildBook(graph, { species, illustrated: summary.illustrated }); + const expected = expectedFigures(docs); + faults.push(...checkFileNames(bookPathways(docs))); + faults.push(...checkFiles(files, neededFiles(expected.values()))); + faults.push(...checkFolder(figures, { release, summary: summary.hash, expected })); + faults.push(...(await checkCharacters(graph, docs, { files, expected: expected.values() }))); + return { faults, docs }; +} diff --git a/test/graph/reader.test.ts b/test/graph/reader.test.ts index 718c69f..71e9e5b 100644 --- a/test/graph/reader.test.ts +++ b/test/graph/reader.test.ts @@ -8,10 +8,12 @@ import { edits, eventByIdentifier, eventFacts, + figureNames, literature, parentPathways, reactionIdentifiers, release, + releaseInfo, topLevelPathways, } from '../../src/graph/queries.ts'; @@ -32,6 +34,32 @@ describe('the graph reader', { skip }, () => { assert.equal(await release(graph), 98); }); + test("reads the release's checksum whole, as its digits, beyond what a number holds", async () => { + // the fixture's checksum is 2^53 + 1, which a JavaScript number reads as 2^53 + assert.deepEqual(await releaseInfo(graph), { release: 98, checksum: '9007199254740993' }); + }); + + test("reads the names a reaction's figure labels: its participants and their compartments", async () => { + const names = await figureNames(graph, [id(211), id(211), id(220)]); + assert.deepEqual(names.get(id(211)), [ + 'Placeholder candidate set', + 'Placeholder candidate set [cytosol]', + 'Placeholder complex', + 'Placeholder complex [cytosol]', + 'Placeholder polymer', + 'Placeholder polymer [cytosol]', + 'Placeholder protein 1', + 'Placeholder protein 1 [cytosol]', + 'Placeholder protein 2', + 'Placeholder protein 2 [cytosol]', + 'Placeholder set', + 'Placeholder set [cytosol]', + 'placeholder membrane', + ]); + // a reaction with no participants has no names + assert.equal(names.get(id(220)), undefined); + }); + test("reads a species' top-level pathways, and only its own", async () => { const human = await topLevelPathways(graph, 'Homo sapiens'); assert.deepEqual( diff --git a/test/preflight/preflight.test.ts b/test/preflight/preflight.test.ts new file mode 100644 index 0000000..02896da --- /dev/null +++ b/test/preflight/preflight.test.ts @@ -0,0 +1,432 @@ +// The book's preflight (spec 001 T024) on the fixture graph and files, with a figures' folder +// each test writes itself (T046 writes the real one): each fault planted is caught before +// anything is printed, and the fixture's own inputs pass. Characters are planted in what the +// graph answers, through a graph that rewrites its rows, so the shared fixture is unchanged. +import assert from 'node:assert/strict'; +import { cpSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { after, before, describe, test } from 'node:test'; + +import { + graphSettings, + openGraph, + type Graph, + type GraphSettings, +} from '../../src/graph/connection.ts'; +import { releaseInfo, type ReleaseInfo } from '../../src/graph/queries.ts'; +import { buildBook } from '../../src/model/build.ts'; +import { preflightCommand } from '../../src/preflight/command.ts'; +import { neededFiles, readSummary, type Summary } from '../../src/preflight/files.ts'; +import { + checkFolder, + expectedFigures, + manifestLine, + MANIFEST, + type FigureRecord, + type Fingerprint, +} from '../../src/preflight/folder.ts'; +import { preflight } from '../../src/preflight/preflight.ts'; + +const skip = process.env['NEO4J_URI'] ? false : 'no fixture graph: set NEO4J_URI'; +const FILES = fileURLToPath(new URL('../../ci/fixture/files/', import.meta.url)); + +/** The graph, its rows' strings and rows rewritten: a character planted, a checksum taken. */ +function rewriting( + graph: Graph, + text: (value: string, query: string) => string, + row: (value: Record) => Record = (r) => r +): Graph { + return { + async read(query: string, params?: Record): Promise { + const visit = (item: unknown): unknown => { + if (typeof item === 'string') return text(item, query); + if (Array.isArray(item)) return item.map(visit); + if (item !== null && typeof item === 'object') { + return Object.fromEntries(Object.entries(item).map(([k, v]) => [k, visit(v)])); + } + return item; + }; + const rows = await graph.read>(query, params); + return rows.map((r) => row(visit(r) as Record)) as T[]; + }, + close: () => Promise.resolve(), + }; +} + +/** A graph whose `from` is `to` wherever a string holds it. */ +const planted = (graph: Graph, from: string, to: string): Graph => + rewriting(graph, (value) => value.split(from).join(to)); + +describe("the book's preflight", { skip }, () => { + let graph: Graph; + let release: ReleaseInfo; + let summary: Summary; + let expected: Map; + let print: Fingerprint; + const dirs: string[] = []; + const scratch = (): string => { + const dir = mkdtempSync(join(tmpdir(), 'pdfx-preflight-')); + dirs.push(dir); + return dir; + }; + + /** A copy of the fixture's release files, to plant faults in. */ + const files = (): string => { + const dir = scratch(); + cpSync(FILES, dir, { recursive: true }); + return dir; + }; + + /** + * A figures' folder as `figures` would leave it for the fixture's book: a file per figure, + * and the manifest, its lines given to `edit` first. + */ + const folder = ( + edit: (lines: string[]) => string[] = (lines) => lines, + fingerprint: Fingerprint = print + ): string => { + const dir = scratch(); + const lines = [manifestLine({ type: 'fingerprint', ...fingerprint })]; + for (const record of expected.values()) { + writeFileSync(join(dir, `${record.stId}.svg`), ''); + lines.push(manifestLine({ type: 'figure', ...record })); + } + lines.push(manifestLine({ type: 'complete', ...fingerprint })); + writeFileSync(join(dir, MANIFEST), edit(lines).join('')); + return dir; + }; + + const run = async ( + on: Graph = graph, + settings: { files?: string; figures?: string } = {} + ): Promise => + ( + await preflight(on, await releaseInfo(on), { + species: 'Homo sapiens', + files: settings.files ?? FILES, + figures: settings.figures ?? folder(), + }) + ).faults; + + const folderFaults = (dir: string, against: ReleaseInfo = release): string[] => + checkFolder(dir, { release: against, summary: summary.hash, expected }); + + before(async () => { + graph = openGraph(graphSettings(process.env)); + release = await releaseInfo(graph); + const read = readSummary(FILES); + if (!read) throw new Error('the fixture has no EHLD summary'); + summary = read; + expected = expectedFigures(await buildBook(graph, { illustrated: summary.illustrated })); + print = { + release: release.release, + checksum: release.checksum ?? '', + summary: summary.hash, + content: release.release, + build: 'test-build', + }; + }); + after(async () => { + await graph.close(); + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); + }); + + test("the fixture's inputs pass, the summary's entry for an absent event no fault", async () => { + assert.deepEqual(await run(), []); + // every kind of figure is among those checked + assert.deepEqual( + new Set([...expected.values()].map((r) => r.kind)), + new Set(['diagram', 'ancestor', 'ehld', 'reaction']) + ); + assert.ok(summary.illustrated.has('R-HSA-99999999')); + assert.ok(!neededFiles(expected.values()).some((name) => name.includes('99999999'))); + }); + + test("a figure's missing file is missing-file", () => { + const dir = folder(); + unlinkSync(join(dir, 'R-HSA-90000211.svg')); + assert.deepEqual(folderFaults(dir), ['missing-file R-HSA-90000211.svg']); + }); + + test('a line with another source or selection, a figure with no line, and a line for no figure are each stale-figure', () => { + const swap = (stId: string, change: (record: FigureRecord) => FigureRecord) => + folder((lines) => + lines.map((line) => { + const record = JSON.parse(line) as FigureRecord & { type: string }; + return record.type === 'figure' && record.stId === stId + ? manifestLine({ ...change(record), type: 'figure' }) + : line; + }) + ); + // Alpha sub one is drawn in Alpha's diagram, its reactions selected + assert.equal(expected.get('R-HSA-90000210')?.kind, 'ancestor'); + assert.deepEqual( + folderFaults(swap('R-HSA-90000210', (r) => ({ ...r, source: 'R-HSA-90000230' }))), + ['stale-figure R-HSA-90000210'] + ); + assert.deepEqual( + folderFaults(swap('R-HSA-90000210', (r) => ({ ...r, highlight: r.highlight.slice(1) }))), + ['stale-figure R-HSA-90000210'] + ); + // a selection that is the right one's beginning + assert.deepEqual( + folderFaults(swap('R-HSA-90000210', (r) => ({ ...r, highlight: r.highlight.slice(0, -1) }))), + ['stale-figure R-HSA-90000210'] + ); + assert.deepEqual(folderFaults(swap('R-HSA-90000300', (r) => ({ ...r, kind: 'diagram' }))), [ + 'stale-figure R-HSA-90000300', + ]); + const noLine = folder((lines) => + lines.filter((line) => (JSON.parse(line) as { stId?: string }).stId !== 'R-HSA-90000132') + ); + assert.deepEqual(folderFaults(noLine), ['stale-figure R-HSA-90000132']); + const extra = folder((lines) => [ + ...lines.slice(0, -1), + manifestLine({ + type: 'figure', + stId: 'R-MMU-90000910', + kind: 'reaction', + source: 'R-MMU-90000910', + highlight: [], + }), + ...lines.slice(-1), + ]); + assert.deepEqual(folderFaults(extra), ['stale-figure R-MMU-90000910']); + }); + + test('a folder of another checksum or release is stale-folder, of another summary stale-summary', () => { + assert.deepEqual(folderFaults(folder(undefined, { ...print, checksum: '1' })), [ + 'stale-folder 98/1 98/9007199254740993', + ]); + assert.deepEqual(folderFaults(folder(undefined, { ...print, release: 97 })), [ + 'stale-folder 97/9007199254740993 98/9007199254740993', + ]); + assert.deepEqual(folderFaults(folder(undefined, { ...print, summary: 'sha256:0' })), [ + 'stale-summary', + ]); + }); + + test('a stale folder is refused as a whole, its figures not checked one by one', () => { + const dir = folder( + (lines) => + lines.filter((line) => (JSON.parse(line) as { stId?: string }).stId !== 'R-HSA-90000132'), + { ...print, checksum: '1' } + ); + unlinkSync(join(dir, 'R-HSA-90000211.svg')); + assert.deepEqual(folderFaults(dir), ['stale-folder 98/1 98/9007199254740993']); + }); + + test('a manifest that is not whole is incomplete-folder, each way', () => { + const cases: Record string[]> = { + 'no complete line': (lines) => lines.slice(0, -1), + 'a figure line after complete': (lines) => [...lines, lines[1] ?? ''], + 'a torn last line': (lines) => [...lines.slice(0, -1), (lines.at(-1) ?? '').slice(0, 20)], + 'a middle line that is not JSON': (lines) => [ + ...lines.slice(0, 2), + '{"type": "figure"\n', + ...lines.slice(2), + ], + 'two lines for one figure': (lines) => [...lines.slice(0, 2), ...lines.slice(1)], + 'complete differing from the first line': (lines) => [ + ...lines.slice(0, -1), + manifestLine({ type: 'complete', ...print, build: 'another-build' }), + ], + 'no manifest': () => [], + }; + for (const [name, edit] of Object.entries(cases)) { + const dir = folder(edit); + if (name === 'no manifest') unlinkSync(join(dir, MANIFEST)); + assert.deepEqual(folderFaults(dir), ['incomplete-folder'], name); + } + }); + + test('a graph with no checksum is no-checksum', async () => { + const without = rewriting( + graph, + (value) => value, + (row) => ('checksum' in row ? { ...row, checksum: null } : row) + ); + assert.equal((await releaseInfo(without)).checksum, null); + assert.deepEqual(await run(without), ['no-checksum']); + }); + + test("a release's missing diagram, illustration or summary is missing-file", async () => { + const dir = files(); + unlinkSync(join(dir, 'diagram/R-HSA-90000130.json')); + unlinkSync(join(dir, 'diagram/R-HSA-90000200.graph.json')); + unlinkSync(join(dir, 'ehld/R-HSA-90000300.svg')); + assert.deepEqual(await run(graph, { files: dir }), [ + 'missing-file diagram/R-HSA-90000130.json', + 'missing-file diagram/R-HSA-90000200.graph.json', + 'missing-file ehld/R-HSA-90000300.svg', + ]); + unlinkSync(join(dir, 'ehld/svgsummary.txt')); + assert.deepEqual(await run(graph, { files: dir }), ['missing-file ehld/svgsummary.txt']); + }); + + test('a character off the list in a text is a fault naming its event; the listed ones are none', async () => { + assert.deepEqual( + await run(planted(graph, 'A third placeholder', 'A third \u2030 placeholder')), + ['uncovered-character U+2030 R-HSA-90000211'] + ); + // in a top-level pathway's name: its cover and section, and each section whose location + // prints it, each a place of its own + assert.deepEqual(await run(planted(graph, 'Gamma pathway', 'Gamma \u2103 pathway')), [ + 'uncovered-character U+2103 R-HSA-90000300', + 'uncovered-character U+2103 R-HSA-90000310', + 'uncovered-character U+2103 R-HSA-90000313', + ]); + assert.deepEqual( + await run( + planted(graph, 'A third placeholder', 'A third \u2011 \uFB01 \u02D9 \u05F3 placeholder') + ), + [] + ); + }); + + test("a character off the list in a name only the cover prints is a fault naming its PDF's pathway", async () => { + // an undated edit's author is on the cover, which names every edit's authors, and in no + // section's edit history, which leaves undated edits out; planted in the edits alone, + // not in the references that also list this placeholder + const coverOnly = rewriting(graph, (value, query) => + query.includes('InstanceEdit') + ? value.replace('Placeholder06', 'Placeholder06 \u2030') + : value + ); + assert.deepEqual(await run(coverOnly), ['uncovered-character U+2030 R-HSA-90000200']); + }); + + test("a character off the list in a participant's name, or in a compartment only a participant holds, is a fault naming the reaction drawing it", async () => { + assert.deepEqual( + await run(planted(graph, 'Placeholder protein 1', 'Placeholder \u2030 protein 1')), + ['uncovered-character U+2030 R-HSA-90000211'] + ); + // the membrane is no event's compartment, so no section prints it: only the figure does + assert.deepEqual( + await run(planted(graph, 'placeholder membrane', 'placeholder \u2103 membrane')), + ['uncovered-character U+2103 R-HSA-90000211'] + ); + assert.deepEqual( + await run(planted(graph, 'placeholder membrane', 'placeholder \u2011\uFB01 membrane')), + [] + ); + }); + + test("a character off the list in a drawn diagram file's label is a fault naming its pathway; in a file not drawn, none", async () => { + const label = (dir: string, stId: string, text: string): void => { + writeFileSync( + join(dir, `diagram/${stId}.json`), + JSON.stringify({ stableId: stId, nodes: [{ displayName: text }], edges: [] }) + ); + }; + const dir = files(); + label(dir, 'R-HSA-90000130', 'node \u2030'); + // Alpha's diagram is drawn for Alpha and, its reactions selected, for Alpha sub one + label(dir, 'R-HSA-90000200', 'node \u2103'); + assert.deepEqual(await run(graph, { files: dir }), [ + 'uncovered-character U+2103 R-HSA-90000200', + 'uncovered-character U+2030 R-HSA-90000130', + ]); + const notDrawn = files(); + // Gamma's figure is its illustration; the mouse pathway is not in the human book + label(notDrawn, 'R-HSA-90000300', 'node \u2030'); + label(notDrawn, 'R-MMU-90000900', 'node \u2030'); + label(notDrawn, 'R-HSA-90000100', 'node \u2011 \uFB01'); + assert.deepEqual(await run(graph, { files: notDrawn }), []); + }); + + test("a top-level pathway's name a file cannot hold, or two equal ignoring case, is bad-file-name", async () => { + assert.deepEqual(await run(planted(graph, 'Gamma pathway', 'Gamma/pathway')), [ + 'bad-file-name R-HSA-90000300', + ]); + assert.deepEqual(await run(planted(graph, 'Gamma pathway', 'ALPHA PATHWAY')), [ + 'bad-file-name R-HSA-90000200', + 'bad-file-name R-HSA-90000300', + ]); + }); + + test('a diagram file that cannot be read as JSON is bad-file, and the run reports every fault', async () => { + const dir = files(); + writeFileSync(join(dir, 'diagram/R-HSA-90000130.json'), '{"nodes": ['); + const out: string[] = []; + const code = await preflightCommand( + ['--files', dir, '--figures', scratch()], + { NEO4J_URI: process.env['NEO4J_URI'] }, + { out: (text) => out.push(text), err: () => undefined } + ); + assert.equal(code, 1); + assert.equal( + out.join(''), + 'fault incomplete-folder\nfault bad-file diagram/R-HSA-90000130.json\n' + ); + }); + + test('an error during the preflight is exit 2, its error on stderr, the faults found so far printed', async () => { + const out: string[] = []; + const err: string[] = []; + // the graph fails at the reactions' names, after the folder's check has found its fault + const failing = (settings: GraphSettings): Graph => { + const real = openGraph(settings); + return { + read: (query: string, params?: Record): Promise => + query.includes('-[:compartment]->(c)') + ? Promise.reject(new Error('the planted failure')) + : real.read(query, params), + close: () => real.close(), + }; + }; + const code = await preflightCommand( + ['--files', FILES, '--figures', scratch()], + { NEO4J_URI: process.env['NEO4J_URI'] }, + { out: (text) => out.push(text), err: (text) => err.push(text), open: failing } + ); + assert.equal(code, 2); + assert.equal(out.join(''), 'fault incomplete-folder\n'); + assert.match(err.join(''), /the planted failure/); + }); + + test('pdf-export preflight: exit 0 with no fault, 1 with each fault a line, 2 on bad arguments or no graph', async () => { + const env: Record = { NEO4J_URI: process.env['NEO4J_URI'] }; + const call = async (args: string[], environment = env) => { + const out: string[] = []; + const err: string[] = []; + const code = await preflightCommand(args, environment, { + out: (text) => out.push(text), + err: (text) => err.push(text), + }); + return { code, out: out.join(''), err: err.join('') }; + }; + assert.deepEqual(await call(['--files', FILES, '--figures', folder()]), { + code: 0, + out: '', + err: '', + }); + const dir = folder(); + unlinkSync(join(dir, 'R-HSA-90000211.svg')); + unlinkSync(join(dir, 'R-HSA-90000132.svg')); + const faulty = await call([`--files=${FILES}`, '--figures', dir]); + assert.equal(faulty.code, 1); + assert.equal( + faulty.out, + 'fault missing-file R-HSA-90000211.svg\nfault missing-file R-HSA-90000132.svg\n' + ); + assert.equal((await call(['--files', FILES])).code, 2); + assert.equal((await call(['--files', FILES, '--figures', dir, '--bake', 'x'])).code, 2); + assert.equal((await call(['--files', FILES, '--figures', dir], {})).code, 2); + // said at once, not after the driver's half a minute of retries + const started = Date.now(); + const unreachable = await call([ + '--graph', + 'bolt://127.0.0.1:1', + '--files', + FILES, + '--figures', + dir, + ]); + assert.equal(unreachable.code, 2); + assert.equal(unreachable.out, ''); + assert.ok(Date.now() - started < 15_000, `${String(Date.now() - started)} ms`); + }); +}); diff --git a/test/preflight/units.test.ts b/test/preflight/units.test.ts new file mode 100644 index 0000000..b5cc9ca --- /dev/null +++ b/test/preflight/units.test.ts @@ -0,0 +1,201 @@ +// The preflight's checks that need no graph (spec 001 T024): the manifest's reading, the file +// names, the order the figures' names are blamed in, and the book's mode of FR-017 in texts +// and in a drawn figure's labels. +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { labelsInBook } from '../../src/figures/labels.ts'; +import { prefixIds } from '../../src/figures/prefix.ts'; +import { BookCharacters, SUBSTITUTIONS } from '../../src/layout/characters.ts'; +import type { Document, Section } from '../../src/model/types.ts'; +import { namesInOrder } from '../../src/preflight/characters.ts'; +import { diagramLabels } from '../../src/preflight/labels.ts'; +import { manifestLine, readManifest, type Fingerprint } from '../../src/preflight/folder.ts'; +import { checkFileNames } from '../../src/preflight/names.ts'; + +const PRINT: Fingerprint = { + release: 98, + checksum: '9007199254740993', + summary: 'sha256:00', + content: 98, + build: 'b1', +}; +const fig = (stId: string) => + manifestLine({ type: 'figure', stId, kind: 'reaction', source: stId, highlight: [] }); +const whole = (...figures: string[]) => + manifestLine({ type: 'fingerprint', ...PRINT }) + + figures.join('') + + manifestLine({ type: 'complete', ...PRINT }); + +test('a whole manifest is read: its fingerprint and a line per figure', () => { + const read = readManifest(whole(fig('R-HSA-1'), fig('R-HSA-2'))); + assert.ok(read); + assert.deepEqual(read.fingerprint, PRINT); + assert.deepEqual([...read.figures.keys()], ['R-HSA-1', 'R-HSA-2']); +}); + +test('a manifest that is not whole is refused, each way', () => { + const first = manifestLine({ type: 'fingerprint', ...PRINT }); + const complete = manifestLine({ type: 'complete', ...PRINT }); + const cases: Record = { + 'no complete line': first + fig('R-HSA-1'), + 'a figure line after complete': first + complete + fig('R-HSA-1'), + 'a complete line in the middle': first + complete + fig('R-HSA-1') + complete, + 'a second fingerprint': first + first + fig('R-HSA-1') + complete, + 'a torn last line': whole(fig('R-HSA-1')).slice(0, -5), + 'a last line with no newline': whole(fig('R-HSA-1')).slice(0, -1), + 'a middle line that is not JSON': first + '{"type":"figure",\n' + complete, + 'a middle line that is JSON but not an object': first + '[1]\n' + complete, + 'two lines for one figure': whole(fig('R-HSA-1'), fig('R-HSA-1')), + 'complete differing from the first line': + first + manifestLine({ type: 'complete', ...PRINT, build: 'b2' }), + 'no fingerprint first': fig('R-HSA-1') + complete, + 'a figure line whose selection is not all strings': + first + fig('R-HSA-1').replace('"highlight":[]', '"highlight":[1]') + complete, + 'a figure line of no known kind': + first + fig('R-HSA-1').replace('reaction', 'photo') + complete, + 'a fingerprint whose checksum is a number': + first.replace('"9007199254740993"', '9007199254740993') + + complete.replace('"9007199254740993"', '9007199254740993'), + empty: '', + }; + for (const [name, text] of Object.entries(cases)) assert.equal(readManifest(text), null, name); +}); + +test('a name a file cannot hold, or two equal ignoring case, is bad-file-name', () => { + const ok = { stId: 'R-HSA-1', name: 'Plain pathway' }; + assert.deepEqual(checkFileNames([ok]), []); + assert.deepEqual( + checkFileNames([ + ok, + { stId: 'R-HSA-2', name: 'A/B pathway' }, + { stId: 'R-HSA-3', name: 'Nul\0pathway' }, + // 251 bytes and `.pdf` is 255, the most a name may hold; 252 is one over + { stId: 'R-HSA-4', name: 'x'.repeat(251) }, + { stId: 'R-HSA-5', name: 'x'.repeat(252) }, + // bytes, not characters: 126 two-byte characters are 252 bytes + { stId: 'R-HSA-6', name: '\u00E9'.repeat(126) }, + { stId: 'R-HSA-7', name: 'Case pathway' }, + { stId: 'R-HSA-8', name: 'CASE Pathway' }, + ]), + [ + 'bad-file-name R-HSA-2', + 'bad-file-name R-HSA-3', + 'bad-file-name R-HSA-5', + 'bad-file-name R-HSA-6', + 'bad-file-name R-HSA-7', + 'bad-file-name R-HSA-8', + ] + ); +}); + +test("the labels of a diagram file are its nodes', compartments', notes', shadows' and links' names", () => { + const item = (displayName: string) => ({ displayName, id: 1 }); + assert.deepEqual( + diagramLabels({ + displayName: 'not a label', + nodes: [ + { + ...item('n'), + nodeAttachments: [{ label: 'P' }, { label: 7 }, { displayName: 'not drawn' }], + }, + item('n2'), + ], + compartments: [item('c')], + notes: [item('o')], + shadows: [item('s')], + links: [item('l'), { id: 2 }], + edges: [item('not drawn as a label')], + }), + ['n', 'P', 'n2', 'c', 'o', 's', 'l'] + ); +}); + +test('a name the figures label is blamed on the first event in document order drawing it', () => { + const section = (dbId: number, stId: string, kind: 'reaction' | 'diagram') => + ({ + event: { dbId, stId }, + figure: { kind, stId }, + }) as unknown as Section; + const doc = (...sections: Section[]) => + ({ places: sections.map((s) => ({ full: true, section: s })) }) as unknown as Document; + const names = new Map([ + [1, ['shared', 'one']], + [2, ['shared', 'two']], + [3, ['three']], + ]); + const first = namesInOrder( + [doc(section(2, 'R-HSA-2', 'reaction')), doc(section(1, 'R-HSA-1', 'reaction'))], + names + ); + assert.equal(first.get('shared'), 'R-HSA-2'); + assert.equal(first.get('one'), 'R-HSA-1'); + // a pathway's figure is its diagram, drawn from its file, not from these names + assert.equal(namesInOrder([doc(section(3, 'R-HSA-3', 'diagram'))], names).size, 0); +}); + +test("the book's mode: each substitution made and counted, none a fault", () => { + const characters = new BookCharacters(); + assert.equal( + characters.text('non\u2011breaking, \uFB01ne, O2\u02D9\u2212, 2\u05F3-O \u2011', 'R-HSA-1'), + 'non-breaking, fine, O2\u2022\u2212, 2\u2032-O -' + ); + assert.deepEqual(characters.faults, []); + assert.deepEqual( + [...characters.replaced], + [ + [0x2011, 2], + [0xfb01, 1], + [0x02d9, 1], + [0x05f3, 1], + ] + ); + assert.equal(SUBSTITUTIONS.size, 4); +}); + +test("the book's mode: any other character no font draws is a fault naming its subject, kept", () => { + const characters = new BookCharacters(); + assert.equal(characters.text('a\u2030b\u2030', 'R-HSA-1'), 'a\u2030b\u2030'); + characters.text('\u2103', 'R-HSA-2'); + characters.text('\u2030', 'R-HSA-2'); + characters.text('\u2030 again', 'R-HSA-1'); + assert.deepEqual(characters.faults, [ + 'uncovered-character U+2030 R-HSA-1', + 'uncovered-character U+2103 R-HSA-2', + 'uncovered-character U+2030 R-HSA-2', + ]); +}); + +test("the book's mode in markup: text and references only, a summation's among a value's strings", () => { + const characters = new BookCharacters(); + const value = { + name: 'x\u2011y', + summations: ['

a‑b ‑

'], + }; + assert.deepEqual(characters.strings(value, 'R-HSA-1'), { + name: 'x-y', + summations: ['

a-b -

'], + }); + assert.deepEqual(characters.faults, []); + characters.markup('

‰

', 'R-HSA-3'); + assert.deepEqual(characters.faults, ['uncovered-character U+2030 R-HSA-3']); +}); + +test("a drawn figure's label off the list is a fault naming the figure's event", () => { + const svg = 'Label \u2030'; + const characters = new BookCharacters(); + labelsInBook(svg, 'R-HSA-90000211', characters); + assert.deepEqual(characters.faults, ['uncovered-character U+2030 R-HSA-90000211']); +}); + +test('a substitution in an inlined figure changes its labels and leaves its elements, ids and attributes', () => { + const svg = + '' + + 'a\u2011b \uFB01 ˙' + + ''; + const prefixed = prefixIds(svg, 's-R-HSA-1'); + const characters = new BookCharacters(); + const out = labelsInBook(prefixed, 'R-HSA-1', characters); + assert.deepEqual(characters.faults, []); + assert.equal(out, prefixed.replace('a\u2011b \uFB01 ˙', 'a-b fi \u2022')); +});