From 6ad9a29a7ec979b7daa5e2061e6e1f7add475a0e Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 29 Oct 2026 18:15:43 +0000 Subject: [PATCH 1/3] Spec 015 T069: BioPAX in the library service The service gains BioPAX3Exporter's library, built from main at 64c716a (PINS.md, pins.spec.ts, whose ARG pattern now allows a digit): GET /biopax/{stId} answers one event's BioPAX Level 3 with the release's per-event xml:base, kept per release on disk as SBML's are. SbmlFiles and SbmlController become ExportFiles and ExportController, one per export, and the conversions' cap of two is shared by both, so the heap holds at most two whatever node sends. The pom holds Kotlin at the library's 2.4.21 (Boot 2.7 would bring 1.6.21); the Dockerfile checks the jar's Kotlin and Paxtools and compares the library's bytes. check.sh adds a BioPAX stage over BioPAX3Exporter's fixture graph: five events of each kind and both species, each contained in its species' expected file by that commit's contained.py, with the per-event base; and 404 and 400. Co-Authored-By: Claude Opus 5.5 --- deploy/java-library/Dockerfile | 33 +++-- deploy/java-library/check.sh | 119 ++++++++++++----- deploy/java-library/pins.spec.ts | 3 +- deploy/java-library/pom.xml | 13 +- ...lController.java => ExportController.java} | 36 ++++-- .../{SbmlFiles.java => ExportFiles.java} | 51 +++++--- .../server/library/HttpDiagramSource.java | 16 +-- .../server/library/LibraryApplication.java | 46 +++++-- .../server/library/ExportControllerTest.java | 117 +++++++++++++++++ ...bmlFilesTest.java => ExportFilesTest.java} | 120 ++++++++++++++---- .../server/library/HttpDiagramSourceTest.java | 12 +- .../server/library/SbmlControllerTest.java | 81 ------------ deploy/java/PINS.md | 5 +- docker-compose.yml | 11 +- 14 files changed, 457 insertions(+), 206 deletions(-) rename deploy/java-library/src/main/java/org/reactome/server/library/{SbmlController.java => ExportController.java} (69%) rename deploy/java-library/src/main/java/org/reactome/server/library/{SbmlFiles.java => ExportFiles.java} (77%) create mode 100644 deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java rename deploy/java-library/src/test/java/org/reactome/server/library/{SbmlFilesTest.java => ExportFilesTest.java} (51%) delete mode 100644 deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java diff --git a/deploy/java-library/Dockerfile b/deploy/java-library/Dockerfile index e3e8d6d05..82de8a375 100644 --- a/deploy/java-library/Dockerfile +++ b/deploy/java-library/Dockerfile @@ -1,11 +1,11 @@ -# Spec 010's library service, started here with SBML as its only operation -# (spec 015 T063): sbml-exporter's library behind one loopback endpoint, which -# node calls. +# Spec 010's library service, started here with SBML (spec 015 T063) and +# BioPAX Level 3 (T069): sbml-exporter's and BioPAX3Exporter's libraries behind +# loopback endpoints, which node calls. # # Every Reactome library is built from the commit deploy/java/PINS.md records, # never taken from a published snapshot: a snapshot can be republished under the -# same version, and the shared parent POM already was once. When the SBML library -# is published at a fixed version, its build stage goes and the pom names it. +# same version, and the shared parent POM already was once. When a library is +# published at a fixed version, its build goes and the pom names it. # # Built only on the beta host and in CI, and never pushed to a registry: the # libraries have no licence yet (spec 015 research R10). @@ -16,6 +16,7 @@ FROM maven:3.9.6-eclipse-temurin-17 AS libraries ARG REACTOME_PARENT_COMMIT=d93ad5a417724647f04810b476f6c2705a0e3aec ARG GRAPH_CORE_COMMIT=62aef35d7a89403b8a8800dc004d8de62f847aee ARG SBML_EXPORTER_COMMIT=f7b10342258d1aacd4f77f44bd928a29b516025e +ARG BIOPAX3_EXPORTER_COMMIT=64c716a7f37081b42c1cf7b30bdebbf55ff580e2 WORKDIR /build @@ -28,7 +29,8 @@ RUN set -eu; \ }; \ fetch reactome-parent "$REACTOME_PARENT_COMMIT"; \ fetch graph-core "$GRAPH_CORE_COMMIT"; \ - fetch sbml-exporter "$SBML_EXPORTER_COMMIT" + fetch sbml-exporter "$SBML_EXPORTER_COMMIT"; \ + fetch BioPAX3Exporter "$BIOPAX3_EXPORTER_COMMIT" # -nsu: a snapshot installed here is never replaced by a repository's. RUN set -eu; \ @@ -36,29 +38,38 @@ RUN set -eu; \ cd /build/reactome-parent/boms/reactome-bom && mvn -B -q -nsu install; \ cd /build/reactome-parent/parents/reactome-parent && mvn -B -q -nsu install; \ cd /build/graph-core && mvn -B -q -nsu install -DskipTests; \ - cd /build/sbml-exporter && mvn -B -q -nsu install -pl lib -am -DskipTests + cd /build/sbml-exporter && mvn -B -q -nsu install -pl lib -am -DskipTests; \ + cd /build/BioPAX3Exporter && mvn -B -q -nsu install -pl lib -am -DskipTests FROM libraries AS service ARG REACTOME_PARENT_COMMIT=d93ad5a417724647f04810b476f6c2705a0e3aec ARG GRAPH_CORE_COMMIT=62aef35d7a89403b8a8800dc004d8de62f847aee ARG SBML_EXPORTER_COMMIT=f7b10342258d1aacd4f77f44bd928a29b516025e +ARG BIOPAX3_EXPORTER_COMMIT=64c716a7f37081b42c1cf7b30bdebbf55ff580e2 WORKDIR /service COPY deploy/java-library/pom.xml ./ COPY deploy/java-library/src ./src # The commits the health endpoint reports. -RUN printf 'reactome-parent=%s\ngraph-core=%s\nsbml-exporter=%s\n' \ - "$REACTOME_PARENT_COMMIT" "$GRAPH_CORE_COMMIT" "$SBML_EXPORTER_COMMIT" \ +RUN printf 'reactome-parent=%s\ngraph-core=%s\nsbml-exporter=%s\nBioPAX3Exporter=%s\n' \ + "$REACTOME_PARENT_COMMIT" "$GRAPH_CORE_COMMIT" "$SBML_EXPORTER_COMMIT" "$BIOPAX3_EXPORTER_COMMIT" \ > src/main/resources/pins.properties RUN mvn -B -q -nsu package # The jar must carry the libraries built above, not a repository's copies of -# their versions: compare their bytes. +# their versions: compare their bytes. And the BioPAX library's Kotlin and +# Paxtools at its own versions, alone: Spring Boot's dependency management +# would give Kotlin's standard library as 1.6.21 (the pom holds it). RUN set -eu; \ mkdir /check && cd /check && jar -xf /service/target/java-library.jar BOOT-INF/lib; \ cmp BOOT-INF/lib/graph-core-3.0.0-SNAPSHOT.jar /build/graph-core/target/graph-core.jar; \ - cmp BOOT-INF/lib/sbml-exporter-lib-3.0.0-SNAPSHOT.jar /build/sbml-exporter/lib/target/sbml-exporter-lib-3.0.0-SNAPSHOT.jar + cmp BOOT-INF/lib/sbml-exporter-lib-3.0.0-SNAPSHOT.jar /build/sbml-exporter/lib/target/sbml-exporter-lib-3.0.0-SNAPSHOT.jar; \ + cmp BOOT-INF/lib/biopax3-exporter-lib-0.0.1-SNAPSHOT.jar /build/BioPAX3Exporter/lib/target/biopax3-exporter-lib-0.0.1-SNAPSHOT.jar; \ + kotlin=$(sed -n 's|.*\(.*\).*|\1|p' /build/BioPAX3Exporter/pom.xml); \ + paxtools=$(sed -n 's|.*\(.*\).*|\1|p' /build/BioPAX3Exporter/pom.xml); \ + [ "$(ls BOOT-INF/lib | grep '^kotlin-stdlib-[0-9]')" = "kotlin-stdlib-$kotlin.jar" ]; \ + [ "$(ls BOOT-INF/lib | grep '^paxtools-core-')" = "paxtools-core-$paxtools.jar" ] FROM eclipse-temurin:17-jre diff --git a/deploy/java-library/check.sh b/deploy/java-library/check.sh index 0be405f26..e8c139c75 100755 --- a/deploy/java-library/check.sh +++ b/deploy/java-library/check.sh @@ -1,8 +1,13 @@ #!/usr/bin/env bash -# The library service, checked end to end over sbml-exporter's fixture graph -# (spec 015 T063): the image this repository builds, its answers for the -# fixture's three events compared with the expected files of the sbml-exporter -# commit it is pinned to, by that commit's own ci/compare.py. +# The library service, checked end to end (spec 015 T063, T069): the image this +# repository builds, over each exporter's own fixture graph, its answers +# compared with the expected files of the commit it is pinned to, by that +# commit's own instruments: +# - SBML over sbml-exporter's fixture: the three events' files, by its +# ci/compare.py; +# - BioPAX over BioPAX3Exporter's fixture: events of each kind and both +# species, each contained in its species' expected file, by its +# ci/contained.py, with the release's per-event xml:base. # # Usage: deploy/java-library/check.sh [image] # GRAPH_PORT (7687) and FILES_PORT (8097) move the fixture graph and the files @@ -30,9 +35,11 @@ work=$(mktemp -d) # Named for this run, so a run that stops early removes only its own. neo=java-library-check-graph-$$ service=java-library-check-$$ +neo2=java-library-check-graph2-$$ +service2=java-library-check2-$$ # shellcheck disable=SC2317 # called by the trap cleanup() { - docker rm -f -v "$neo" "$service" >/dev/null 2>&1 || true + docker rm -f -v "$neo" "$service" "$neo2" "$service2" >/dev/null 2>&1 || true if [ -n "${server:-}" ]; then kill "$server" 2>/dev/null || true; fi rm -rf "$work" } @@ -51,15 +58,36 @@ curl -fsSL "https://codeload.github.com/reactome/sbml-exporter/tar.gz/$commit" \ | tar -xz -C "$work/sbml-exporter" --strip-components=1 --wildcards '*/ci/*' ci="$work/sbml-exporter/ci" -# The fixture graph, on loopback. -docker run -d --name "$neo" -p "127.0.0.1:$graph_port:7687" \ - -e NEO4J_AUTH=neo4j/fixturepass -e NEO4J_dbms_default__database=graph.db \ - -v "$ci/fixture/fixture.cypher:/fixture.cypher:ro" neo4j:4.4.48-community >/dev/null -for _ in $(seq 90); do - docker exec "$neo" cypher-shell -u neo4j -p fixturepass -d graph.db 'RETURN 1' >/dev/null 2>&1 && break - sleep 2 -done -docker exec "$neo" cypher-shell -u neo4j -p fixturepass -d graph.db -f /fixture.cypher >/dev/null +# A fixture graph, on loopback: start_graph +start_graph() { + docker run -d --name "$1" -p "127.0.0.1:$graph_port:7687" \ + -e NEO4J_AUTH=neo4j/fixturepass -e NEO4J_dbms_default__database=graph.db \ + -v "$2:/fixture.cypher:ro" neo4j:4.4.48-community >/dev/null + for _ in $(seq 90); do + docker exec "$1" cypher-shell -u neo4j -p fixturepass -d graph.db 'RETURN 1' >/dev/null 2>&1 && break + sleep 2 + done + docker exec "$1" cypher-shell -u neo4j -p fixturepass -d graph.db -f /fixture.cypher >/dev/null +} + +# The service over it, answering its health check: start_service +start_service() { + docker run -d --name "$1" --network host \ + -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \ + -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache "$image" >/dev/null + for _ in $(seq 90); do + curl -fs http://127.0.0.1:4360/health >/dev/null && break + sleep 2 + done + if ! curl -fsS http://127.0.0.1:4360/health; then + echo "The service did not answer its health check. Its log:" >&2 + docker logs "$1" 2>&1 | tail -60 >&2 + exit 1 + fi + echo +} + +start_graph "$neo" "$ci/fixture/fixture.cypher" # The release's download directory, as far as the fixture needs it. mkdir -p "$work/www/97/diagram" @@ -67,19 +95,7 @@ cp "$ci"/fixture/R-*.json "$work/www/97/diagram/" python3 -m http.server "$files_port" --bind 127.0.0.1 --directory "$work/www" >/dev/null 2>&1 & server=$! -docker run -d --name "$service" --network host \ - -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \ - -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache "$image" >/dev/null -for _ in $(seq 90); do - curl -fs http://127.0.0.1:4360/health >/dev/null && break - sleep 2 -done -if ! curl -fsS http://127.0.0.1:4360/health; then - echo "The service did not answer its health check. Its log:" >&2 - docker logs "$service" 2>&1 | tail -60 >&2 - exit 1 -fi -echo +start_service "$service" status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:4360/sbml/$1"; } fail=0 @@ -93,6 +109,51 @@ code=$(status R-HSA-1 /dev/null) code=$(status R-HSA-9613354.1 /dev/null) [ "$code" = 400 ] || { echo "R-HSA-9613354.1 (not a bare stable id): $code, expected 400" >&2; fail=1; } -python3 "$ci/validate.py" "$work/out" -python3 "$ci/compare.py" "$ci/fixture/expected" "$work/out" +python3 "$ci/validate.py" "$work/out" || fail=1 +python3 "$ci/compare.py" "$ci/fixture/expected" "$work/out" || fail=1 + +# BioPAX, over BioPAX3Exporter's fixture graph (release 98), in a graph and a +# service of their own. +docker rm -f -v "$neo" "$service" >/dev/null +bcommit=$(sed -n 's/^ARG BIOPAX3_EXPORTER_COMMIT=\([0-9a-f]\{40\}\)$/\1/p' "$root/deploy/java-library/Dockerfile" | sort -u) +[ "$(printf '%s\n' "$bcommit" | wc -l)" -eq 1 ] || { echo "The Dockerfile names more than one BioPAX3Exporter commit" >&2; exit 1; } +mkdir "$work/BioPAX3Exporter" +curl -fsSL "https://codeload.github.com/reactome/BioPAX3Exporter/tar.gz/$bcommit" \ + | tar -xz -C "$work/BioPAX3Exporter" --strip-components=1 --wildcards '*/ci/*' +bci="$work/BioPAX3Exporter/ci" +start_graph "$neo2" "$bci/fixture/fixture.cypher" +start_service "$service2" + +# Events of each kind and both species, chosen by the graph: the first of each by +# database id. Each line: stId, dbId, species. +events=$(docker exec "$neo2" cypher-shell -u neo4j -p fixturepass -d graph.db --format plain ' + CALL { + MATCH (e:TopLevelPathway {speciesName: "Homo sapiens"}) RETURN e ORDER BY e.dbId LIMIT 1 + UNION ALL MATCH (e:Reaction {speciesName: "Homo sapiens"}) RETURN e ORDER BY e.dbId LIMIT 1 + UNION ALL MATCH (e:BlackBoxEvent {speciesName: "Homo sapiens"}) RETURN e ORDER BY e.dbId LIMIT 1 + UNION ALL MATCH (e:ReactionLikeEvent {dbId: 174731}) RETURN e + UNION ALL MATCH (e:Pathway {speciesName: "Plasmodium falciparum"}) RETURN e ORDER BY e.dbId LIMIT 1 + } + RETURN e.stId + " " + e.dbId + " " + replace(e.speciesName, " ", "_")' | tail -n +2 | tr -d '"') +[ "$(printf '%s\n' "$events" | wc -l)" -eq 5 ] || { echo "Expected five events in the fixture, got: $events" >&2; exit 1; } + +bstatus() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:4360/biopax/$1"; } +mkdir -p "$work/owl/Homo_sapiens" "$work/owl/Plasmodium_falciparum" +while read -r id dbid species; do + file="$work/owl/$species/$id.owl" + code=$(bstatus "$id" "$file") + [ "$code" = 200 ] || { echo "$id: $code, expected 200" >&2; fail=1; continue; } + # The release's per-event base: Reactome's BioPAX namespace, the release, the event's database id + grep -q "xml:base=\"http://www.reactome.org/biopax/98/$dbid#\"" "$file" \ + || { echo "$id: not the release's per-event xml:base" >&2; fail=1; } +done <<<"$events" +code=$(bstatus R-HSA-1 /dev/null) +[ "$code" = 404 ] || { echo "R-HSA-1 (not in the graph): $code, expected 404" >&2; fail=1; } +code=$(bstatus R-HSA-174731.1 /dev/null) +[ "$code" = 400 ] || { echo "R-HSA-174731.1 (not a bare stable id): $code, expected 400" >&2; fail=1; } + +# Each event's file holds nothing its species' file lacks, and is whole +for species in Homo_sapiens Plasmodium_falciparum; do + python3 -I "$bci/contained.py" "$bci/fixture/expected/$species.owl" "$work/owl/$species"/*.owl || fail=1 +done exit $fail diff --git a/deploy/java-library/pins.spec.ts b/deploy/java-library/pins.spec.ts index 565d43f38..1f863849a 100644 --- a/deploy/java-library/pins.spec.ts +++ b/deploy/java-library/pins.spec.ts @@ -15,7 +15,7 @@ const pins = readFileSync(join(__dirname, '../java/PINS.md'), 'utf8'); /** The Dockerfile's ARG _COMMIT= lines, by name, each stage's. */ function commitArgs(text: string) { const found = new Map>(); - for (const [, name = '', sha = ''] of text.matchAll(/^ARG ([A-Z_]+)_COMMIT=([0-9a-f]{40})$/gm)) + for (const [, name = '', sha = ''] of text.matchAll(/^ARG ([A-Z0-9_]+)_COMMIT=([0-9a-f]{40})$/gm)) found.set(name, (found.get(name) ?? new Set()).add(sha)); return found; } @@ -30,6 +30,7 @@ const REPOSITORIES: Record = { REACTOME_PARENT: 'reactome-parent', GRAPH_CORE: 'graph-core', SBML_EXPORTER: 'sbml-exporter', + BIOPAX3_EXPORTER: 'BioPAX3Exporter', }; describe('the library service pins', () => { diff --git a/deploy/java-library/pom.xml b/deploy/java-library/pom.xml index 9abca68ec..5c9aa9f9b 100644 --- a/deploy/java-library/pom.xml +++ b/deploy/java-library/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - + org.reactome.server java-library 0.1.0 @@ -26,6 +26,12 @@ 3.0.0-SNAPSHOT + + 0.0.1-SNAPSHOT + + 2.4.21 @@ -34,6 +40,11 @@ sbml-exporter-lib ${sbml-exporter-lib.version} + + org.reactome.server.tools + biopax3-exporter-lib + ${biopax3-exporter-lib.version} + org.springframework.boot spring-boot-starter-web diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java b/deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java similarity index 69% rename from deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java rename to deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java index ba89c59e1..ad46a550f 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpStatus; @@ -20,21 +21,25 @@ /** * The service's answers to node, over loopback. Node writes the public answer: its headers, its errors in the - * service's envelope and its cache labels (spec 015 T065). This writes only the body, or a status and a reason. + * service's envelope and its cache labels (spec 015 T065, T070). This writes only the body, or a status and a reason. */ @RestController -public class SbmlController { +public class ExportController { - private static final Logger logger = LoggerFactory.getLogger(SbmlController.class); + private static final Logger logger = LoggerFactory.getLogger(ExportController.class); private static final Pattern ST_ID = Pattern.compile("R-[A-Z]{3}-\\d+"); static final MediaType SBML = MediaType.parseMediaType("application/sbml+xml;charset=UTF-8"); + static final MediaType BIOPAX = MediaType.parseMediaType("application/rdf+xml;charset=UTF-8"); - private final SbmlFiles files; - private final SbmlFiles.Graph graph; + private final ExportFiles sbml; + private final ExportFiles biopax; + private final ExportFiles.Graph graph; private final Properties pins; - public SbmlController(SbmlFiles files, SbmlFiles.Graph graph, Properties pins) { - this.files = files; + public ExportController(@Qualifier("sbmlFiles") ExportFiles sbml, @Qualifier("biopaxFiles") ExportFiles biopax, + ExportFiles.Graph graph, Properties pins) { + this.sbml = sbml; + this.biopax = biopax; this.graph = graph; this.pins = pins; } @@ -42,13 +47,26 @@ public SbmlController(SbmlFiles files, SbmlFiles.Graph graph, Properties pins) { /** An event's SBML, by its stable id without a version: node resolves any other form first. */ @GetMapping("/sbml/{stId}") public ResponseEntity sbml(@PathVariable String stId) throws InterruptedException { + return answer(sbml, SBML, stId); + } + + /** + * An event's BioPAX Level 3, by its stable id without a version. Node decides 404 and 400 first (T070); the + * service's own 404 is a backstop. + */ + @GetMapping("/biopax/{stId}") + public ResponseEntity biopax(@PathVariable String stId) throws InterruptedException { + return answer(biopax, BIOPAX, stId); + } + + private static ResponseEntity answer(ExportFiles files, MediaType type, String stId) throws InterruptedException { if (!ST_ID.matcher(stId).matches()) return reason(HttpStatus.BAD_REQUEST, "Not a stable id: " + stId); try { Optional file = files.get(stId); if (file.isEmpty()) return reason(HttpStatus.NOT_FOUND, "No event " + stId + " in the graph"); Resource body = new FileSystemResource(file.get()); - return ResponseEntity.ok().contentType(SBML).body(body); - } catch (SbmlFiles.DiagramUnavailable e) { + return ResponseEntity.ok().contentType(type).body(body); + } catch (ExportFiles.DiagramUnavailable e) { logger.warn("{}: {}", stId, e.getMessage()); return reason(HttpStatus.SERVICE_UNAVAILABLE, e.getMessage()); } catch (RuntimeException e) { diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlFiles.java b/deploy/java-library/src/main/java/org/reactome/server/library/ExportFiles.java similarity index 77% rename from deploy/java-library/src/main/java/org/reactome/server/library/SbmlFiles.java rename to deploy/java-library/src/main/java/org/reactome/server/library/ExportFiles.java index 83a0a89e7..97f8b063e 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlFiles.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/ExportFiles.java @@ -20,17 +20,18 @@ import org.slf4j.LoggerFactory; /** - * The SBML files, kept on disk per release as ContentService keeps its exports. + * One export's files (SBML, BioPAX), kept on disk per release as ContentService keeps its exports. * *

The release is the graph's, read for every request, so a graph moved to a new release is never answered from the * old one's files, and the old release's files are removed when it is first seen. One id is converted once at a time: - * a second request for it waits for the first. Node's share of 2 is the limit on conversions (spec 015 T065); the cap - * here is a backstop. + * a second request for it waits for the first. Node gives each export a share of 2 (spec 015 T065, T070); the + * conversions' cap is shared by every export in the service, so the heap holds at most that many at once whatever + * node sends (T069). * *

Removing files is best effort: a release can change while a conversion of the old one is still writing, and that * must not fail the new release's request. What is left behind goes when the next release is first seen. */ -public class SbmlFiles { +public class ExportFiles { /** What the service needs from the graph. */ public interface Graph { @@ -42,7 +43,7 @@ public interface Graph { /** An event, as far as the diagram files are concerned. */ public static final class Event { - /** A pathway the graph says has a diagram of its own: its layout needs the diagram file. */ + /** A pathway the graph says has a diagram of its own: its SBML layout needs the diagram file. */ public final boolean hasDiagram; public Event(boolean hasDiagram) { @@ -50,7 +51,7 @@ public Event(boolean hasDiagram) { } } - /** sbml-exporter's library, writing {@code {stId}.sbml} into a folder. */ + /** An exporter's library, writing {@code {stId}{extension}} into a folder. */ public interface Converter { void convert(String stId, int release, Path diagrams, Path into) throws Exception; } @@ -72,34 +73,50 @@ public DiagramUnavailable(String message, Throwable cause) { } } - private static final Logger logger = LoggerFactory.getLogger(SbmlFiles.class); + private static final Logger logger = LoggerFactory.getLogger(ExportFiles.class); + private final String kind; + private final String extension; private final Graph graph; private final Converter converter; private final DiagramSource diagrams; private final Path cache; - private final Semaphore conversions = new Semaphore(2); + private final Semaphore conversions; private final Map> running = new ConcurrentHashMap<>(); private volatile int seen = -1; - public SbmlFiles(Graph graph, Converter converter, DiagramSource diagrams, Path cache) { + /** + * @param kind the export's folder in the cache, {@code sbml} or {@code biopax} + * @param extension what the converter names its file, {@code .sbml} or {@code .owl} + * @param diagrams where a pathway's diagram file comes from, or null for an export that needs none + * @param conversions the cap on conversions at once, shared by the service's exports + */ + public ExportFiles(String kind, String extension, Graph graph, Converter converter, DiagramSource diagrams, + Path cache, Semaphore conversions) { + this.kind = kind; + this.extension = extension; this.graph = graph; this.converter = converter; this.diagrams = diagrams; this.cache = cache; + this.conversions = conversions; // Work in progress when the service last stopped, which nothing will finish - delete(cache.resolve("tmp")); + delete(work()); + } + + private Path work() { + return cache.resolve("tmp").resolve(kind); } /** - * The event's SBML file, or empty when the graph has no such event. + * The event's file, or empty when the graph has no such event. * * @throws DiagramUnavailable when its diagram file cannot be had */ public Optional get(String stId) throws DiagramUnavailable, InterruptedException { int release = graph.release(); forgetOtherReleases(release); - Path file = cache.resolve("sbml").resolve(String.valueOf(release)).resolve(stId + ".sbml"); + Path file = cache.resolve(kind).resolve(String.valueOf(release)).resolve(stId + extension); if (Files.isRegularFile(file)) return Optional.of(file); Optional event = graph.event(stId); @@ -124,13 +141,13 @@ private Path convert(String stId, int release, Event event, Path file) throws Ex conversions.acquire(); try { Path diagramFolder = Files.createDirectories(cache.resolve("diagram").resolve(String.valueOf(release))); - if (event.hasDiagram && !Files.isRegularFile(diagramFolder.resolve(stId + ".json"))) { + if (diagrams != null && event.hasDiagram && !Files.isRegularFile(diagramFolder.resolve(stId + ".json"))) { diagrams.fetch(stId, release, diagramFolder); } - Path work = Files.createDirectories(cache.resolve("tmp").resolve(UUID.randomUUID().toString())); + Path work = Files.createDirectories(work().resolve(UUID.randomUUID().toString())); try { converter.convert(stId, release, diagramFolder, work); - Path written = work.resolve(stId + ".sbml"); + Path written = work.resolve(stId + extension); if (!Files.isRegularFile(written)) throw new IllegalStateException("The converter wrote no " + written); Files.createDirectories(file.getParent()); return Files.move(written, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); @@ -159,8 +176,8 @@ private void forgetOtherReleases(int release) { if (seen == release) return; synchronized (this) { if (seen == release) return; - for (String kind : new String[]{"sbml", "diagram"}) { - Path folder = cache.resolve(kind); + for (String folderName : new String[]{kind, "diagram"}) { + Path folder = cache.resolve(folderName); if (!Files.isDirectory(folder)) continue; try (DirectoryStream releases = Files.newDirectoryStream(folder)) { for (Path other : releases) { diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java b/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java index 6f61034f1..c31c02f4f 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java @@ -23,7 +23,7 @@ * per converted pathway that has a diagram; the caller keeps it per release, so a file is kept only once it is * whole and is the pathway's: the converter would refuse anything else on every request until the next release. */ -public class HttpDiagramSource implements SbmlFiles.DiagramSource { +public class HttpDiagramSource implements ExportFiles.DiagramSource { private static final ObjectMapper JSON = new ObjectMapper(); @@ -45,7 +45,7 @@ public HttpDiagramSource(URI base) { } @Override - public void fetch(String stId, int release, Path into) throws SbmlFiles.DiagramUnavailable { + public void fetch(String stId, int release, Path into) throws ExportFiles.DiagramUnavailable { URI uri = base.resolve(release + "/diagram/" + stId + ".json"); Path part = into.resolve(stId + ".json.part"); try { @@ -59,27 +59,27 @@ public void fetch(String stId, int release, Path into) throws SbmlFiles.DiagramU response = download.get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { download.cancel(true); - throw new SbmlFiles.DiagramUnavailable(uri + " did not arrive within " + timeout.toSeconds() + " s", e); + throw new ExportFiles.DiagramUnavailable(uri + " did not arrive within " + timeout.toSeconds() + " s", e); } catch (ExecutionException e) { - throw new SbmlFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getCause(), e.getCause()); + throw new ExportFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getCause(), e.getCause()); } catch (InterruptedException e) { // Waiting is interrupted, not the transfer: it is stopped here, as on a timeout download.cancel(true); throw e; } if (response.statusCode() != 200) { - throw new SbmlFiles.DiagramUnavailable(uri + " answered " + response.statusCode(), null); + throw new ExportFiles.DiagramUnavailable(uri + " answered " + response.statusCode(), null); } if (!isTheDiagramOf(stId, part)) { - throw new SbmlFiles.DiagramUnavailable(uri + " is not " + stId + "'s diagram", null); + throw new ExportFiles.DiagramUnavailable(uri + " is not " + stId + "'s diagram", null); } Files.move(part, into.resolve(stId + ".json"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - throw new SbmlFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getMessage(), e); + throw new ExportFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getMessage(), e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new SbmlFiles.DiagramUnavailable("Interrupted fetching " + uri, e); + throw new ExportFiles.DiagramUnavailable("Interrupted fetching " + uri, e); } finally { try { Files.deleteIfExists(part); diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java b/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java index 7384af2d3..f4995633d 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java @@ -1,9 +1,11 @@ package org.reactome.server.library; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Optional; import java.util.Properties; +import java.util.concurrent.Semaphore; import org.reactome.server.graph.config.GraphCoreNeo4jConfig; import org.reactome.server.graph.domain.model.Event; @@ -11,6 +13,7 @@ import org.reactome.server.graph.service.AdvancedDatabaseObjectService; import org.reactome.server.graph.service.DatabaseObjectService; import org.reactome.server.graph.service.GeneralService; +import org.reactome.server.tools.WriteBioPAX3; import org.reactome.server.tools.sbml.converter.SbmlConverter; import org.reactome.server.tools.sbml.layout.DiagramFiles; import org.springframework.boot.SpringApplication; @@ -19,8 +22,8 @@ import org.springframework.context.annotation.Import; /** - * Spec 010's library service, started in this repository with SBML as its only operation (spec 015 T063). Node - * answers the public URLs and calls this over loopback; this writes only the body. + * Spec 010's library service, started in this repository with SBML (spec 015 T063) and BioPAX Level 3 (T069) as its + * operations. Node answers the public URLs and calls this over loopback; this writes only the body. */ @SpringBootApplication @Import(GraphCoreNeo4jConfig.class) @@ -39,36 +42,53 @@ public static void main(String[] args) { } @Bean - public SbmlFiles.Graph graph(DatabaseObjectService ds, GeneralService general) { - return new SbmlFiles.Graph() { + public ExportFiles.Graph graph(DatabaseObjectService ds, GeneralService general) { + return new ExportFiles.Graph() { @Override public int release() { return general.getDBInfo().getVersion(); } @Override - public Optional event(String stId) { + public Optional event(String stId) { Object found = ds.findById(stId); if (!(found instanceof Event)) return Optional.empty(); boolean hasDiagram = found instanceof Pathway && Boolean.TRUE.equals(((Pathway) found).getHasDiagram()); - return Optional.of(new SbmlFiles.Event(hasDiagram)); + return Optional.of(new ExportFiles.Event(hasDiagram)); } }; } + /** The conversions at once, across every export: two, within the heap (spec 015 T069). */ @Bean - public SbmlFiles.Converter converter(DatabaseObjectService ds, AdvancedDatabaseObjectService ads) { - return (stId, release, diagrams, into) -> { + public Semaphore conversions() { + return new Semaphore(2); + } + + @Bean + public ExportFiles sbmlFiles(Settings settings, ExportFiles.Graph graph, DatabaseObjectService ds, + AdvancedDatabaseObjectService ads, Semaphore conversions) { + ExportFiles.Converter converter = (stId, release, diagrams, into) -> { Event event = ds.findById(stId); - SbmlConverter converter = new SbmlConverter(event, release, ds, ads, new DiagramFiles(diagrams, release, release)); - converter.convert(); - converter.writeToFile(into.toString()); + SbmlConverter sbml = new SbmlConverter(event, release, ds, ads, new DiagramFiles(diagrams, release, release)); + sbml.convert(); + sbml.writeToFile(into.toString()); }; + return new ExportFiles("sbml", ".sbml", graph, converter, new HttpDiagramSource(settings.diagramBase), + settings.cache, conversions); } + /** BioPAX3Exporter's library: one event with all it holds, with the release's per-event xml:base. */ @Bean - public SbmlFiles sbmlFiles(Settings settings, SbmlFiles.Graph graph, SbmlFiles.Converter converter) { - return new SbmlFiles(graph, converter, new HttpDiagramSource(settings.diagramBase), settings.cache); + public ExportFiles biopaxFiles(Settings settings, ExportFiles.Graph graph, DatabaseObjectService ds, + Semaphore conversions) { + ExportFiles.Converter converter = (stId, release, diagrams, into) -> { + Event event = ds.findById(stId); + WriteBioPAX3 biopax = new WriteBioPAX3(event, release); + biopax.createModel(); + biopax.toFile(new File(into.toFile(), stId + ".owl")); + }; + return new ExportFiles("biopax", ".owl", graph, converter, null, settings.cache, conversions); } /** The pinned commits the image was built from, written by the Dockerfile (deploy/java/PINS.md). */ diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java new file mode 100644 index 000000000..480eea7bf --- /dev/null +++ b/deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java @@ -0,0 +1,117 @@ +package org.reactome.server.library; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.Semaphore; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.http.ResponseEntity; + +class ExportControllerTest { + + @TempDir + Path cache; + + private final ExportFiles.Graph graph = new ExportFiles.Graph() { + @Override + public int release() { + return 97; + } + + @Override + public Optional event(String stId) { + return stId.equals("R-HSA-1") ? Optional.of(new ExportFiles.Event(true)) : Optional.empty(); + } + }; + + private final ExportFiles.Converter owl = (stId, rel, d, into) -> Files.writeString(into.resolve(stId + ".owl"), ""); + + private ExportController controller(ExportFiles.DiagramSource diagrams, ExportFiles.Converter converter) { + return controller(diagrams, converter, owl); + } + + private ExportController controller(ExportFiles.DiagramSource diagrams, ExportFiles.Converter converter, + ExportFiles.Converter biopax) { + Semaphore cap = new Semaphore(2); + return new ExportController(new ExportFiles("sbml", ".sbml", graph, converter, diagrams, cache, cap), + new ExportFiles("biopax", ".owl", graph, biopax, null, cache, cap), graph, new Properties()); + } + + private final ExportFiles.Converter writes = (stId, rel, d, into) -> Files.writeString(into.resolve(stId + ".sbml"), ""); + private final ExportFiles.DiagramSource fetches = (stId, rel, into) -> { + try { + Files.writeString(into.resolve(stId + ".json"), "{}"); + } catch (java.io.IOException e) { + throw new ExportFiles.DiagramUnavailable(e.getMessage(), e); + } + }; + + @Test + void answersTheFileAsSbml() throws Exception { + ResponseEntity answer = controller(fetches, writes).sbml("R-HSA-1"); + assertEquals(200, answer.getStatusCodeValue()); + assertEquals(ExportController.SBML, answer.getHeaders().getContentType()); + } + + @Test + void anIdNotInTheGraphIs404() throws Exception { + assertEquals(404, controller(fetches, writes).sbml("R-HSA-9").getStatusCodeValue()); + } + + @Test + void anythingButAStableIdIs400() throws Exception { + for (String id : new String[]{"R-HSA-1.2", "123", "../R-HSA-1", "r-hsa-1"}) { + assertEquals(400, controller(fetches, writes).sbml(id).getStatusCodeValue(), id); + } + } + + @Test + void aDiagramThatCannotBeHadIs503() throws Exception { + ExportFiles.DiagramSource failing = (stId, rel, into) -> { + throw new ExportFiles.DiagramUnavailable("answered 404", null); + }; + ResponseEntity answer = controller(failing, writes).sbml("R-HSA-1"); + assertEquals(503, answer.getStatusCodeValue()); + assertEquals(Map.of("error", "answered 404"), answer.getBody()); + } + + @Test + void aConversionThatFailsIs500() throws Exception { + ExportFiles.Converter failing = (stId, rel, d, into) -> { + throw new IllegalStateException("no reactions"); + }; + assertEquals(500, controller(fetches, failing).sbml("R-HSA-1").getStatusCodeValue()); + } + + @Test + void answersBiopaxAsRdf() throws Exception { + ResponseEntity answer = controller(fetches, writes).biopax("R-HSA-1"); + assertEquals(200, answer.getStatusCodeValue()); + assertEquals(ExportController.BIOPAX, answer.getHeaders().getContentType()); + assertEquals("R-HSA-1.owl", ((org.springframework.core.io.Resource) answer.getBody()).getFilename()); + } + + @Test + void biopaxNeedsNoDiagram() throws Exception { + ExportFiles.DiagramSource failing = (stId, rel, into) -> { + throw new ExportFiles.DiagramUnavailable("answered 404", null); + }; + assertEquals(200, controller(failing, writes).biopax("R-HSA-1").getStatusCodeValue()); + } + + @Test + void biopaxAnswersTheSameErrors() throws Exception { + assertEquals(404, controller(fetches, writes).biopax("R-HSA-9").getStatusCodeValue()); + assertEquals(400, controller(fetches, writes).biopax("R-HSA-1.2").getStatusCodeValue()); + ExportFiles.Converter failing = (stId, rel, d, into) -> { + throw new IllegalStateException("no model"); + }; + assertEquals(500, controller(fetches, writes, failing).biopax("R-HSA-1").getStatusCodeValue()); + } +} diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlFilesTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/ExportFilesTest.java similarity index 51% rename from deploy/java-library/src/test/java/org/reactome/server/library/SbmlFilesTest.java rename to deploy/java-library/src/test/java/org/reactome/server/library/ExportFilesTest.java index 37bf24688..9cecb3b9b 100644 --- a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlFilesTest.java +++ b/deploy/java-library/src/test/java/org/reactome/server/library/ExportFilesTest.java @@ -15,13 +15,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -class SbmlFilesTest { +class ExportFilesTest { @TempDir Path cache; @@ -31,45 +32,115 @@ class SbmlFilesTest { private final AtomicInteger lookups = new AtomicInteger(); /** A graph with a pathway that has a diagram, one that has none, and nothing else. */ - private final SbmlFiles.Graph graph = new SbmlFiles.Graph() { + private final ExportFiles.Graph graph = new ExportFiles.Graph() { @Override public int release() { return release; } @Override - public Optional event(String stId) { + public Optional event(String stId) { lookups.incrementAndGet(); - if (stId.equals("R-HSA-1")) return Optional.of(new SbmlFiles.Event(true)); - if (stId.equals("R-HSA-2")) return Optional.of(new SbmlFiles.Event(false)); + if (stId.equals("R-HSA-1")) return Optional.of(new ExportFiles.Event(true)); + if (stId.equals("R-HSA-2")) return Optional.of(new ExportFiles.Event(false)); return Optional.empty(); } }; - private final SbmlFiles.Converter converter = (stId, rel, diagrams, into) -> { + private final ExportFiles.Converter converter = (stId, rel, diagrams, into) -> { calls.add("convert " + stId + " " + rel + " diagram=" + Files.exists(diagrams.resolve(stId + ".json"))); Files.writeString(into.resolve(stId + ".sbml"), ""); }; - private final SbmlFiles.DiagramSource diagrams = (stId, rel, into) -> { + private final ExportFiles.DiagramSource diagrams = (stId, rel, into) -> { calls.add("fetch " + stId + " " + rel); try { Files.writeString(into.resolve(stId + ".json"), "{}"); } catch (java.io.IOException e) { - throw new SbmlFiles.DiagramUnavailable(e.getMessage(), e); + throw new ExportFiles.DiagramUnavailable(e.getMessage(), e); } }; + private final Semaphore cap = new Semaphore(2); + + private ExportFiles sbml(ExportFiles.Converter converter, ExportFiles.DiagramSource diagrams) { + return new ExportFiles("sbml", ".sbml", graph, converter, diagrams, cache, cap); + } + + /** BioPAX needs no diagram file: the export is made with none. */ + private final ExportFiles.Converter owl = (stId, rel, diagrams, into) -> { + calls.add("biopax " + stId + " " + rel); + Files.writeString(into.resolve(stId + ".owl"), ""); + }; + + private ExportFiles biopax(ExportFiles.Converter converter) { + return new ExportFiles("biopax", ".owl", graph, converter, null, cache, cap); + } + + @Test + void anExportWithNoDiagramSourceNeverFetchesOne() throws Exception { + ExportFiles files = biopax(owl); + Path file = files.get("R-HSA-1").orElseThrow(); + assertEquals(cache.resolve("biopax/97/R-HSA-1.owl"), file); + assertEquals(List.of("biopax R-HSA-1 97"), calls); + } + + @Test + void eachExportKeepsItsOwnFiles() throws Exception { + ExportFiles sbml = sbml(converter, diagrams); + ExportFiles biopax = biopax(owl); + assertEquals("", Files.readString(sbml.get("R-HSA-2").orElseThrow())); + assertEquals("", Files.readString(biopax.get("R-HSA-2").orElseThrow())); + // A new release seen by one export leaves the other's files to it, and each then converts afresh + release = 98; + biopax.get("R-HSA-2"); + assertTrue(Files.exists(cache.resolve("sbml/97/R-HSA-2.sbml"))); + assertFalse(Files.exists(cache.resolve("biopax/97"))); + } + + @Test + void theCapIsSharedByTheExports() throws Exception { + CountDownLatch inside = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger at = new AtomicInteger(); + AtomicInteger most = new AtomicInteger(); + ExportFiles.Converter held = (stId, rel, d, into) -> { + most.accumulateAndGet(at.incrementAndGet(), Math::max); + inside.countDown(); + assertTrue(release.await(10, TimeUnit.SECONDS)); + at.decrementAndGet(); + Files.writeString(into.resolve(stId + (into.toString().contains("biopax") ? ".owl" : ".sbml")), "x"); + }; + ExportFiles sbml = sbml(held, diagrams); + ExportFiles biopax = biopax(held); + ExecutorService pool = Executors.newFixedThreadPool(3); + try { + Future> a = pool.submit(() -> sbml.get("R-HSA-2")); + Future> b = pool.submit(() -> biopax.get("R-HSA-2")); + assertTrue(inside.await(10, TimeUnit.SECONDS)); + Future> c = pool.submit(() -> biopax.get("R-HSA-1")); + Thread.sleep(200); // c waits for a place: two conversions are running, one of each export + assertEquals(2, at.get()); + release.countDown(); + a.get(10, TimeUnit.SECONDS); + b.get(10, TimeUnit.SECONDS); + c.get(10, TimeUnit.SECONDS); + } finally { + pool.shutdownNow(); + } + assertEquals(2, most.get()); + } + @Test void anIdNotInTheGraphIsEmpty() throws Exception { - SbmlFiles files = new SbmlFiles(graph, converter, diagrams, cache); + ExportFiles files = sbml(converter, diagrams); assertTrue(files.get("R-HSA-3").isEmpty()); assertTrue(calls.isEmpty()); } @Test void convertsOnceAndKeepsTheFile() throws Exception { - SbmlFiles files = new SbmlFiles(graph, converter, diagrams, cache); + ExportFiles files = sbml(converter, diagrams); Path first = files.get("R-HSA-2").orElseThrow(); Path second = files.get("R-HSA-2").orElseThrow(); assertEquals(first, second); @@ -81,7 +152,7 @@ void convertsOnceAndKeepsTheFile() throws Exception { @Test void fetchesTheDiagramOfAPathwayThatHasOne() throws Exception { - SbmlFiles files = new SbmlFiles(graph, converter, diagrams, cache); + ExportFiles files = sbml(converter, diagrams); files.get("R-HSA-1"); files.get("R-HSA-2"); assertEquals(List.of("fetch R-HSA-1 97", "convert R-HSA-1 97 diagram=true", "convert R-HSA-2 97 diagram=false"), @@ -90,11 +161,11 @@ void fetchesTheDiagramOfAPathwayThatHasOne() throws Exception { @Test void aDiagramThatCannotBeHadIsNeverSbmlWithoutItsLayout() throws Exception { - SbmlFiles.DiagramSource failing = (stId, rel, into) -> { - throw new SbmlFiles.DiagramUnavailable("answered 503", null); + ExportFiles.DiagramSource failing = (stId, rel, into) -> { + throw new ExportFiles.DiagramUnavailable("answered 503", null); }; - SbmlFiles files = new SbmlFiles(graph, converter, failing, cache); - assertThrows(SbmlFiles.DiagramUnavailable.class, () -> files.get("R-HSA-1")); + ExportFiles files = sbml(converter, failing); + assertThrows(ExportFiles.DiagramUnavailable.class, () -> files.get("R-HSA-1")); assertTrue(calls.isEmpty()); assertFalse(Files.exists(cache.resolve("sbml/97/R-HSA-1.sbml"))); } @@ -102,11 +173,11 @@ void aDiagramThatCannotBeHadIsNeverSbmlWithoutItsLayout() throws Exception { @Test void aFailedConversionIsTriedAgain() throws Exception { AtomicInteger attempts = new AtomicInteger(); - SbmlFiles.Converter once = (stId, rel, d, into) -> { + ExportFiles.Converter once = (stId, rel, d, into) -> { if (attempts.incrementAndGet() == 1) throw new IllegalStateException("the first attempt fails"); Files.writeString(into.resolve(stId + ".sbml"), ""); }; - SbmlFiles files = new SbmlFiles(graph, once, diagrams, cache); + ExportFiles files = sbml(once, diagrams); assertThrows(IllegalStateException.class, () -> files.get("R-HSA-2")); assertEquals("", Files.readString(files.get("R-HSA-2").orElseThrow())); assertEquals(2, attempts.get()); @@ -114,15 +185,18 @@ void aFailedConversionIsTriedAgain() throws Exception { @Test void workLeftByAStoppedServiceIsRemoved() throws Exception { - Path stale = Files.createDirectories(cache.resolve("tmp/left-by-a-killed-run")); + Path stale = Files.createDirectories(cache.resolve("tmp/sbml/left-by-a-killed-run")); Files.writeString(stale.resolve("R-HSA-2.sbml"), " { + ExportFiles.Converter slow = (stId, rel, d, into) -> { conversions.incrementAndGet(); started.countDown(); assertTrue(release.await(10, TimeUnit.SECONDS)); Files.writeString(into.resolve(stId + ".sbml"), ""); }; - SbmlFiles files = new SbmlFiles(graph, slow, diagrams, cache); + ExportFiles files = sbml(slow, diagrams); ExecutorService pool = Executors.newFixedThreadPool(2); try { Future> a = pool.submit(() -> files.get("R-HSA-2")); diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java index 8b26e5793..ce88a0022 100644 --- a/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java +++ b/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java @@ -124,7 +124,7 @@ void writesTheReleasesFile() throws Exception { @Test void aMissingFileIsUnavailableAndLeavesNothing() throws Exception { - SbmlFiles.DiagramUnavailable e = assertThrows(SbmlFiles.DiagramUnavailable.class, + ExportFiles.DiagramUnavailable e = assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-2", 97, into)); assertTrue(e.getMessage().contains("answered 404"), e.getMessage()); try (var left = Files.list(into)) { @@ -134,7 +134,7 @@ void aMissingFileIsUnavailableAndLeavesNothing() throws Exception { @Test void aPageInPlaceOfTheFileIsUnavailableAndNotKept() throws Exception { - SbmlFiles.DiagramUnavailable e = assertThrows(SbmlFiles.DiagramUnavailable.class, + ExportFiles.DiagramUnavailable e = assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-3", 97, into)); assertTrue(e.getMessage().contains("not R-HSA-3's diagram"), e.getMessage()); try (var left = Files.list(into)) { @@ -144,7 +144,7 @@ void aPageInPlaceOfTheFileIsUnavailableAndNotKept() throws Exception { @Test void anotherPathwaysDiagramIsUnavailableAndNotKept() throws Exception { - SbmlFiles.DiagramUnavailable e = assertThrows(SbmlFiles.DiagramUnavailable.class, + ExportFiles.DiagramUnavailable e = assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-4", 97, into)); assertTrue(e.getMessage().contains("not R-HSA-4's diagram"), e.getMessage()); try (var left = Files.list(into)) { @@ -156,7 +156,7 @@ void anotherPathwaysDiagramIsUnavailableAndNotKept() throws Exception { void aBodyThatStallsIsUnavailableWithinTheTimeout() throws Exception { // The request's own timeout covers only the wait for the headers assertTimeoutPreemptively(Duration.ofSeconds(15), - () -> assertThrows(SbmlFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-5", 97, into))); + () -> assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-5", 97, into))); try (var left = Files.list(into)) { assertEquals(0, left.count()); } @@ -164,7 +164,7 @@ void aBodyThatStallsIsUnavailableWithinTheTimeout() throws Exception { @Test void aDownloadThatTimesOutIsAbandoned() throws Exception { - assertThrows(SbmlFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-6", 97, into)); + assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-6", 97, into)); assertTrue(abandoned.await(10, TimeUnit.SECONDS), "the server was still sending"); } @@ -187,7 +187,7 @@ void anInterruptedDownloadIsAbandoned() throws Exception { fetching.interrupt(); fetching.join(10_000); assertFalse(fetching.isAlive(), "the fetch did not return"); - assertTrue(outcome.get() instanceof SbmlFiles.DiagramUnavailable, String.valueOf(outcome.get())); + assertTrue(outcome.get() instanceof ExportFiles.DiagramUnavailable, String.valueOf(outcome.get())); assertTrue(stillInterrupted.get(), "the interrupt was swallowed"); assertTrue(abandoned.await(10, TimeUnit.SECONDS), "the server was still sending"); } diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java deleted file mode 100644 index f3ba26905..000000000 --- a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.reactome.server.library; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.springframework.http.ResponseEntity; - -class SbmlControllerTest { - - @TempDir - Path cache; - - private final SbmlFiles.Graph graph = new SbmlFiles.Graph() { - @Override - public int release() { - return 97; - } - - @Override - public Optional event(String stId) { - return stId.equals("R-HSA-1") ? Optional.of(new SbmlFiles.Event(true)) : Optional.empty(); - } - }; - - private SbmlController controller(SbmlFiles.DiagramSource diagrams, SbmlFiles.Converter converter) { - return new SbmlController(new SbmlFiles(graph, converter, diagrams, cache), graph, new Properties()); - } - - private final SbmlFiles.Converter writes = (stId, rel, d, into) -> Files.writeString(into.resolve(stId + ".sbml"), ""); - private final SbmlFiles.DiagramSource fetches = (stId, rel, into) -> { - try { - Files.writeString(into.resolve(stId + ".json"), "{}"); - } catch (java.io.IOException e) { - throw new SbmlFiles.DiagramUnavailable(e.getMessage(), e); - } - }; - - @Test - void answersTheFileAsSbml() throws Exception { - ResponseEntity answer = controller(fetches, writes).sbml("R-HSA-1"); - assertEquals(200, answer.getStatusCodeValue()); - assertEquals(SbmlController.SBML, answer.getHeaders().getContentType()); - } - - @Test - void anIdNotInTheGraphIs404() throws Exception { - assertEquals(404, controller(fetches, writes).sbml("R-HSA-9").getStatusCodeValue()); - } - - @Test - void anythingButAStableIdIs400() throws Exception { - for (String id : new String[]{"R-HSA-1.2", "123", "../R-HSA-1", "r-hsa-1"}) { - assertEquals(400, controller(fetches, writes).sbml(id).getStatusCodeValue(), id); - } - } - - @Test - void aDiagramThatCannotBeHadIs503() throws Exception { - SbmlFiles.DiagramSource failing = (stId, rel, into) -> { - throw new SbmlFiles.DiagramUnavailable("answered 404", null); - }; - ResponseEntity answer = controller(failing, writes).sbml("R-HSA-1"); - assertEquals(503, answer.getStatusCodeValue()); - assertEquals(Map.of("error", "answered 404"), answer.getBody()); - } - - @Test - void aConversionThatFailsIs500() throws Exception { - SbmlFiles.Converter failing = (stId, rel, d, into) -> { - throw new IllegalStateException("no reactions"); - }; - assertEquals(500, controller(fetches, failing).sbml("R-HSA-1").getStatusCodeValue()); - } -} diff --git a/deploy/java/PINS.md b/deploy/java/PINS.md index f982e5b60..30fbf42ef 100644 --- a/deploy/java/PINS.md +++ b/deploy/java/PINS.md @@ -17,6 +17,7 @@ The Reactome Java this deployment builds (spec 010), each pinned to the commit t | reactome-parent 1.0.5-SNAPSHOT (and its BOM) | `reactome/reactome-parent` | `d93ad5a` (branch `feature/new-pwb`) | the BOM equal to the published snapshot; the parent differs only in Batik and FOP's versions (published 1.18 and 2.10, here 1.16 and 2.8), which nothing built on it here uses: the published snapshot matches no pushed commit | 17 | | graph-core 3.0.0-SNAPSHOT | `reactome/graph-core` | `62aef35` | the deployed ContentService's jar is the published snapshot of 25 Sep 2025, byte for byte; rebuilt here, 172/377 classes identical, 203 identical by `javap`, 2 differ only in order (a method's place; which of three identical handlers each exception jumps to) | 17 | | sbml-exporter-lib 3.0.0-SNAPSHOT | `reactome/sbml-exporter` | `f7b1034` (branch `graph-only`) | not Java's relational answer, so no byte comparison: proven by `ci/compare.py` against today's files (spec 015 T037, T039) | 17 | +| biopax3-exporter-lib 0.0.1-SNAPSHOT | `reactome/BioPAX3Exporter` | `64c716a` (`main`) | not Java's relational answer, so no byte comparison: proven by its own `ci/compare.py` and `ci/contained.py` against release 98's published files (its spec 001 T020, T021), and here by spec 015 T049 against the legacy service | 17 | ## Frozen on purpose @@ -27,9 +28,9 @@ The Reactome Java this deployment builds (spec 010), each pinned to the commit t - **What uses it:** only diagram-exporter's PPTX path, which no Java-homed operation uses, and it runs unlicensed today (research R8). - **To use it again:** a licence, and an operation that needs it. -## The library service (spec 015 T063) +## The library service (spec 015 T063, T069) -`deploy/java-library/Dockerfile` builds the last three rows from these commits, and checks that its jar carries the libraries it built, byte for byte. `deploy/java-library/pins.spec.ts` checks that the Dockerfile's commits are this table's. Changing `sbml-exporter-lib`'s pin means rerunning spec 015 T067's comparison, not `tools/java-parity`: its body is not Java's. +`deploy/java-library/Dockerfile` builds the last four rows from these commits, and checks that its jar carries the libraries it built, byte for byte. `deploy/java-library/pins.spec.ts` checks that the Dockerfile's commits are this table's. Changing `sbml-exporter-lib`'s pin means rerunning spec 015 T067's comparison, not `tools/java-parity`: its body is not Java's. Changing `biopax3-exporter-lib`'s likewise means rerunning spec 015 T049's comparison and T072's check. ## Changing a pin diff --git a/docker-compose.yml b/docker-compose.yml index 2a557d7ba..a1b6ad1dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -317,10 +317,11 @@ services: # fails to build its lists, and rebuilds on the next request -- which is # the behaviour specs/006 D10 asks for -- rather than refusing to start. - # The library service (spec 015 T063, T064): the website's SBML, made on - # demand from the graph by sbml-exporter's library. Spec 010's Java library - # service, started here with SBML as its only operation. Node is its only - # client (T065); nginx never names it. + # The library service (spec 015 T063, T064, T069): the website's SBML and + # BioPAX Level 3, made on demand from the graph by sbml-exporter's and + # BioPAX3Exporter's libraries. Spec 010's Java library service, started here + # with those two operations. Node is its only client (T065, T070); nginx + # never names it. # # Built here, on this host and in CI, from the commits deploy/java/PINS.md # pins, and never pushed to a registry: no `image:` to pull or push @@ -404,7 +405,7 @@ volumes: # Survives a container replacement, which is the point: a cached figure is a # file read, and rebuilding the cache means paying for every render again. render-cache: - # The library service's SBML and diagram files, per release: kept across a + # The library service's SBML, BioPAX and diagram files, per release: kept across a # container replacement, as render's figures are. java-library-cache: graphdb-release98: From 0909fbb269b5e5c55fcaa132e5c5c887ccee8220 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 29 Oct 2026 18:41:35 +0000 Subject: [PATCH 2/3] Spec 015, from #754's review: BioPAX's 503s tested, the shares said as they are The service's conversions are capped across both exports (T069), so node's shares keep one export from taking the other's places in node, not at the service; the comment says so. BioPAX's own 503 messages, the service's 503 with Retry-After 30 and an unreachable service, are tested, and the slashed form joins the forms of the id; a changed message is caught. Co-Authored-By: Claude Opus 5.5 --- tools/content-node/biopax-export.spec.ts | 27 ++++++++++++++++++++--- tools/content-node/exporters-endpoints.ts | 11 ++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/tools/content-node/biopax-export.spec.ts b/tools/content-node/biopax-export.spec.ts index c492ce357..eaa9a83b3 100644 --- a/tools/content-node/biopax-export.spec.ts +++ b/tools/content-node/biopax-export.spec.ts @@ -105,15 +105,15 @@ describe('the file', () => { res.writeHead(200, { 'Content-Type': 'application/rdf+xml;charset=UTF-8' }); res.end(''); }; - for (const form of ['R-HSA-1', 'R-HSA-1.4', '1']) { - const answer = await answers(`event/${form}.biopax`); + for (const path of ['R-HSA-1.biopax', 'R-HSA-1.4.biopax', '1.biopax', 'R-HSA-1.biopax/']) { + const answer = await answers(`event/${path}`); expect(answer.status).toBe(200); expect(answer.type).toBe('application/rdf+xml;charset=UTF-8'); expect(answer.headers?.['Content-Disposition']).toBe('attachment; filename="R-HSA-1.biopax"'); expect(await text(answer)).toBe(''); } // The service is asked for BioPAX, by the bare stable id - expect(asked).toEqual(['/biopax/R-HSA-1', '/biopax/R-HSA-1', '/biopax/R-HSA-1']); + expect(asked).toEqual(Array(4).fill('/biopax/R-HSA-1')); }); it('the service’s 404 is a 404, and its failure a 500 without its own reason', async () => { @@ -134,6 +134,27 @@ describe('the file', () => { ]); }); + it('the service’s 503 is a 503 to try again, and a service not there is a 503, in BioPAX’s words', async () => { + withService(); + behave = (_path, res) => { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end('{"error":"/cache busy"}'); + }; + const busy = await answers('event/R-HSA-1.biopax'); + expect(envelope(busy)).toEqual([ + 503, + 'SERVICE_UNAVAILABLE', + ['The BioPAX file cannot be made just now; try again shortly'], + ]); + expect(busy.headers?.['Retry-After']).toBe('30'); + process.env['SBML_SERVICE'] = 'http://127.0.0.1:9'; + expect(envelope(await answers('event/R-HSA-1.biopax'))).toEqual([ + 503, + 'SERVICE_UNAVAILABLE', + ['The BioPAX service cannot be reached'], + ]); + }); + it('a service that sends no headers in time is a 503', async () => { withService(); behave = () => undefined; diff --git a/tools/content-node/exporters-endpoints.ts b/tools/content-node/exporters-endpoints.ts index 89f1e2f2f..1fa0afc9c 100644 --- a/tools/content-node/exporters-endpoints.ts +++ b/tools/content-node/exporters-endpoints.ts @@ -370,11 +370,12 @@ async function exportEvent(raw: string): Promise { } /** - * Each export's own share (research R6): Java made one file at a time, and the - * service two at once across its exports (its backstop, spec 015 T069); past - * a share, a 503 with Retry-After, as the deep pool answers. Separate from the - * deep pool and from each other, so a burst of one cannot starve the queries - * the pages need, or the other export. + * Each export's own share (research R6): Java made one file at a time; past a + * share, a 503 with Retry-After, as the deep pool answers. Separate from the + * deep pool, so a burst of exports cannot starve the queries the pages need, + * and from each other, so one export cannot take the other's places here. The + * service converts two at once across both (spec 015 T069), for its heap: a + * BioPAX file may wait there behind two SBML ones, within the headers' time. */ const SBML_SHARE = deepShare(2); const BIOPAX_SHARE = deepShare(2); From 53bea8d53fc801cdfb522ea31fd468c1441805e5 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 29 Oct 2026 19:10:10 +0000 Subject: [PATCH 3/3] Spec 015 T069: checked and measured; check.sh picks events its files hold check.sh chooses, for each kind, the first fixture event its species' expected file holds (the fixture has events beyond those files), and takes a SERVICE_PORT, so it runs beside a deployment's own service. Run here: both stages pass, and against beta's image, with no BioPAX, it fails. Measured on a private copy of release 98's graph: the largest pathways' BioPAX and SBML, four at once, peak at 1.10 GiB within T064's limits; the shared cap queues the second pair past node's 290 s, recorded in T069. Co-Authored-By: Claude Opus 5.5 --- deploy/java-library/check.sh | 51 +++++++++++++++++++------------- specs/015-graph-exports/tasks.md | 5 +++- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/deploy/java-library/check.sh b/deploy/java-library/check.sh index e8c139c75..38932c61f 100755 --- a/deploy/java-library/check.sh +++ b/deploy/java-library/check.sh @@ -10,22 +10,24 @@ # ci/contained.py, with the release's per-event xml:base. # # Usage: deploy/java-library/check.sh [image] -# GRAPH_PORT (7687) and FILES_PORT (8097) move the fixture graph and the files -# off ports a host already uses; the service keeps its own, 4360. +# GRAPH_PORT (7687), FILES_PORT (8097) and SERVICE_PORT (4360) move the fixture +# graph, the files and the service off ports a host already uses (a deployment's +# own service is on 4360). # Without an image, it builds one. Needs docker, curl and python3 with # python-libsbml 5.21.2 (pip install python-libsbml==5.21.2). set -euo pipefail graph_port=${GRAPH_PORT:-7687} files_port=${FILES_PORT:-8097} +service_port=${SERVICE_PORT:-4360} -# The check's own service takes 4360 on the host's network. One already there +# The check's own service takes its port on the host's network. One already there # (a deployment's) would answer the checks below in its place, over its graph. # Any answer refuses, a 503 too (a deployment whose graph is down), and it is # refused before the trap below, whose clean-up would remove another run's # containers. -if curl -s -m 5 -o /dev/null http://127.0.0.1:4360/health; then - echo "Something already answers on 127.0.0.1:4360; stop it, or run this elsewhere" >&2 +if curl -s -m 5 -o /dev/null http://127.0.0.1:$service_port/health; then + echo "Something already answers on 127.0.0.1:$service_port; stop it, or set SERVICE_PORT" >&2 exit 1 fi @@ -74,12 +76,13 @@ start_graph() { start_service() { docker run -d --name "$1" --network host \ -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \ - -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache "$image" >/dev/null + -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache \ + -e SERVER_PORT="$service_port" "$image" >/dev/null for _ in $(seq 90); do - curl -fs http://127.0.0.1:4360/health >/dev/null && break + curl -fs http://127.0.0.1:$service_port/health >/dev/null && break sleep 2 done - if ! curl -fsS http://127.0.0.1:4360/health; then + if ! curl -fsS http://127.0.0.1:$service_port/health; then echo "The service did not answer its health check. Its log:" >&2 docker logs "$1" 2>&1 | tail -60 >&2 exit 1 @@ -97,7 +100,7 @@ server=$! start_service "$service" -status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:4360/sbml/$1"; } +status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:$service_port/sbml/$1"; } fail=0 mkdir "$work/out" for id in $(find "$ci/fixture/expected" -name 'R-*.sbml' -exec basename {} .sbml \; | sort); do @@ -124,20 +127,26 @@ bci="$work/BioPAX3Exporter/ci" start_graph "$neo2" "$bci/fixture/fixture.cypher" start_service "$service2" -# Events of each kind and both species, chosen by the graph: the first of each by -# database id. Each line: stId, dbId, species. -events=$(docker exec "$neo2" cypher-shell -u neo4j -p fixturepass -d graph.db --format plain ' +# Events of each kind and both species: of the graph's, by database id, the +# first of each kind that its species' expected file holds (the fixture has +# events beyond those files, such as an inferred event's source). Each line: +# stId, dbId, species. +candidates=$(docker exec "$neo2" cypher-shell -u neo4j -p fixturepass -d graph.db --format plain ' CALL { - MATCH (e:TopLevelPathway {speciesName: "Homo sapiens"}) RETURN e ORDER BY e.dbId LIMIT 1 - UNION ALL MATCH (e:Reaction {speciesName: "Homo sapiens"}) RETURN e ORDER BY e.dbId LIMIT 1 - UNION ALL MATCH (e:BlackBoxEvent {speciesName: "Homo sapiens"}) RETURN e ORDER BY e.dbId LIMIT 1 - UNION ALL MATCH (e:ReactionLikeEvent {dbId: 174731}) RETURN e - UNION ALL MATCH (e:Pathway {speciesName: "Plasmodium falciparum"}) RETURN e ORDER BY e.dbId LIMIT 1 + MATCH (e:TopLevelPathway {speciesName: "Homo sapiens"}) RETURN "pathway" AS kind, e + UNION ALL MATCH (e:Reaction {speciesName: "Homo sapiens"}) RETURN "reaction" AS kind, e + UNION ALL MATCH (e:BlackBoxEvent {speciesName: "Homo sapiens"}) RETURN "blackbox" AS kind, e + UNION ALL MATCH (e:ReactionLikeEvent {dbId: 174731}) RETURN "degradation" AS kind, e + UNION ALL MATCH (e:Pathway {speciesName: "Plasmodium falciparum"}) RETURN "falciparum" AS kind, e } - RETURN e.stId + " " + e.dbId + " " + replace(e.speciesName, " ", "_")' | tail -n +2 | tr -d '"') -[ "$(printf '%s\n' "$events" | wc -l)" -eq 5 ] || { echo "Expected five events in the fixture, got: $events" >&2; exit 1; } - -bstatus() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:4360/biopax/$1"; } + RETURN kind + " " + e.stId + " " + e.dbId + " " + replace(e.speciesName, " ", "_") + ORDER BY kind, e.dbId' | tail -n +2 | tr -d '"') +events=$(while read -r kind id dbid species; do + grep -q ">$dbid" "$bci/fixture/expected/$species.owl" && echo "$kind $id $dbid $species" +done <<<"$candidates" | sort -u -k1,1 | cut -d' ' -f2-) +[ "$(printf '%s\n' "$events" | wc -l)" -eq 5 ] || { echo "Expected one event of each of five kinds in the fixture's files, got: $events" >&2; exit 1; } + +bstatus() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:$service_port/biopax/$1"; } mkdir -p "$work/owl/Homo_sapiens" "$work/owl/Plasmodium_falciparum" while read -r id dbid species; do file="$work/owl/$species/$id.owl" diff --git a/specs/015-graph-exports/tasks.md b/specs/015-graph-exports/tasks.md index c1a14540e..cffa18f28 100644 --- a/specs/015-graph-exports/tasks.md +++ b/specs/015-graph-exports/tasks.md @@ -265,7 +265,10 @@ Both exporters' adoption pull requests found that their own checks do not run ye - [ ] T050 [US2] (The species files are done: the validator gives no error a published file lacks, its T020, [BioPAX3Exporter#79](https://github.com/reactome/BioPAX3Exporter/pull/79).) What is left: validate the sample's per-event files from the library, and the legacy service's, and record any error the library's has that the legacy's lacks. - [x] T051 [US2] [ext] In `BioPAX3Exporter`'s CI: the validator over its output for a small fixture graph, failing on any error in the reports. — its T004 ([BioPAX3Exporter#15](https://github.com/reactome/BioPAX3Exporter/pull/15)) and T007 ([BioPAX3Exporter#27](https://github.com/reactome/BioPAX3Exporter/pull/27)). - [ ] T052 [US2] Serve the website's BioPAX through the library service in `deploy/java-library/`, as R10's amendment of 29 Oct 2026 decided: T069–T072, in that order. T071 waits for T049 and T050. -- [ ] T069 [US2] The BioPAX operation in the library service, on T063's terms: +- [x] T069 [US2] The BioPAX operation in the library service, on T063's terms: — #755. `SbmlFiles` and `SbmlController` became `ExportFiles` and `ExportController`, one `ExportFiles` per export, with the conversions' cap of two shared by both. BioPAX3Exporter `main` at `64c716a` is pinned (PINS.md; `pins.spec.ts`, whose ARG pattern now allows a digit). The Dockerfile checks the jar's `kotlin-stdlib` 2.4.21 and `paxtools-core` 6.0.0. `check.sh` gains a BioPAX stage and a `SERVICE_PORT`, so it runs beside a deployment's service. + - **Checked here:** the image built and `check.sh` passed over both fixtures: SBML's three files valid and equal to expected; five BioPAX events, one of each kind across both species, each contained and whole in its species' expected file, with the per-event base. Against beta's image, which has no BioPAX, it failed: five 404s. + - **Measured** on a private copy of release 98's graph, in a container held to 1.5 GiB with T064's 1 GiB heap: BioPAX and SBML of Signal Transduction and Metabolism, all four asked at once. Peak 1.10 GiB, no kill. The cap converted two at once: Metabolism's pair in 156–158 s, then Signal Transduction's pair, finished at 300–310 s. Alone, a conversion of either is about 150 s, under node's 290 s. In that worst case the queued pair's readers get node's 503 ("took too long"); the service finishes and keeps the files, so a second request is answered at once. Files: BioPAX 47 MB and 41 MB, SBML 23 MB and 19 MB. + - **Unit tests:** `ExportFilesTest` (11) and `ExportControllerTest` (8). The new ones: an export with no diagram source never fetches one; each export keeps its own files; the cap is shared, at most two at once across both; BioPAX answered as RDF, needing no diagram, with SBML's errors. From #754's review, in the same PR: node's comment on the shares, and BioPAX's 503s tested. - **The library from source:** the Dockerfile builds `biopax3-exporter-lib` from a pinned commit of `BioPAX3Exporter`'s `main`, with its row in `deploy/java/PINS.md` and `pins.spec.ts` following. Its graph-core must be the build already pinned, or the PR says why not. The service's pom sets `kotlin.version` to the library's: Spring Boot 2.7's dependency management would otherwise bring Kotlin's standard library down to 1.6.21 under a library compiled with 2.4. The Dockerfile's jar check `cmp`s `biopax3-exporter-lib` as it does `sbml-exporter-lib`, and requires the jar's `kotlin-stdlib` and `paxtools-core` to be the library's versions. - **Its operation:** `GET /biopax/{stId}` answers the library's one-event conversion, with the release's per-event `xml:base`, and 404 for an id that is not a pathway or reaction in the graph, a backstop: node decides 404 and 400 first (T070). - **The files** are kept on disk per release, written beside and moved into place. One id is never converted twice at the same time.