diff --git a/deploy/java-library/Dockerfile b/deploy/java-library/Dockerfile index e3e8d6d05..82de8a375 100644 --- a/deploy/java-library/Dockerfile +++ b/deploy/java-library/Dockerfile @@ -1,11 +1,11 @@ -# Spec 010's library service, started here with SBML as its only operation -# (spec 015 T063): sbml-exporter's library behind one loopback endpoint, which -# node calls. +# Spec 010's library service, started here with SBML (spec 015 T063) and +# BioPAX Level 3 (T069): sbml-exporter's and BioPAX3Exporter's libraries behind +# loopback endpoints, which node calls. # # Every Reactome library is built from the commit deploy/java/PINS.md records, # never taken from a published snapshot: a snapshot can be republished under the -# same version, and the shared parent POM already was once. When the SBML library -# is published at a fixed version, its build stage goes and the pom names it. +# same version, and the shared parent POM already was once. When a library is +# published at a fixed version, its build goes and the pom names it. # # Built only on the beta host and in CI, and never pushed to a registry: the # libraries have no licence yet (spec 015 research R10). @@ -16,6 +16,7 @@ FROM maven:3.9.6-eclipse-temurin-17 AS libraries ARG REACTOME_PARENT_COMMIT=d93ad5a417724647f04810b476f6c2705a0e3aec ARG GRAPH_CORE_COMMIT=62aef35d7a89403b8a8800dc004d8de62f847aee ARG SBML_EXPORTER_COMMIT=f7b10342258d1aacd4f77f44bd928a29b516025e +ARG BIOPAX3_EXPORTER_COMMIT=64c716a7f37081b42c1cf7b30bdebbf55ff580e2 WORKDIR /build @@ -28,7 +29,8 @@ RUN set -eu; \ }; \ fetch reactome-parent "$REACTOME_PARENT_COMMIT"; \ fetch graph-core "$GRAPH_CORE_COMMIT"; \ - fetch sbml-exporter "$SBML_EXPORTER_COMMIT" + fetch sbml-exporter "$SBML_EXPORTER_COMMIT"; \ + fetch BioPAX3Exporter "$BIOPAX3_EXPORTER_COMMIT" # -nsu: a snapshot installed here is never replaced by a repository's. RUN set -eu; \ @@ -36,29 +38,38 @@ RUN set -eu; \ cd /build/reactome-parent/boms/reactome-bom && mvn -B -q -nsu install; \ cd /build/reactome-parent/parents/reactome-parent && mvn -B -q -nsu install; \ cd /build/graph-core && mvn -B -q -nsu install -DskipTests; \ - cd /build/sbml-exporter && mvn -B -q -nsu install -pl lib -am -DskipTests + cd /build/sbml-exporter && mvn -B -q -nsu install -pl lib -am -DskipTests; \ + cd /build/BioPAX3Exporter && mvn -B -q -nsu install -pl lib -am -DskipTests FROM libraries AS service ARG REACTOME_PARENT_COMMIT=d93ad5a417724647f04810b476f6c2705a0e3aec ARG GRAPH_CORE_COMMIT=62aef35d7a89403b8a8800dc004d8de62f847aee ARG SBML_EXPORTER_COMMIT=f7b10342258d1aacd4f77f44bd928a29b516025e +ARG BIOPAX3_EXPORTER_COMMIT=64c716a7f37081b42c1cf7b30bdebbf55ff580e2 WORKDIR /service COPY deploy/java-library/pom.xml ./ COPY deploy/java-library/src ./src # The commits the health endpoint reports. -RUN printf 'reactome-parent=%s\ngraph-core=%s\nsbml-exporter=%s\n' \ - "$REACTOME_PARENT_COMMIT" "$GRAPH_CORE_COMMIT" "$SBML_EXPORTER_COMMIT" \ +RUN printf 'reactome-parent=%s\ngraph-core=%s\nsbml-exporter=%s\nBioPAX3Exporter=%s\n' \ + "$REACTOME_PARENT_COMMIT" "$GRAPH_CORE_COMMIT" "$SBML_EXPORTER_COMMIT" "$BIOPAX3_EXPORTER_COMMIT" \ > src/main/resources/pins.properties RUN mvn -B -q -nsu package # The jar must carry the libraries built above, not a repository's copies of -# their versions: compare their bytes. +# their versions: compare their bytes. And the BioPAX library's Kotlin and +# Paxtools at its own versions, alone: Spring Boot's dependency management +# would give Kotlin's standard library as 1.6.21 (the pom holds it). RUN set -eu; \ mkdir /check && cd /check && jar -xf /service/target/java-library.jar BOOT-INF/lib; \ cmp BOOT-INF/lib/graph-core-3.0.0-SNAPSHOT.jar /build/graph-core/target/graph-core.jar; \ - cmp BOOT-INF/lib/sbml-exporter-lib-3.0.0-SNAPSHOT.jar /build/sbml-exporter/lib/target/sbml-exporter-lib-3.0.0-SNAPSHOT.jar + cmp BOOT-INF/lib/sbml-exporter-lib-3.0.0-SNAPSHOT.jar /build/sbml-exporter/lib/target/sbml-exporter-lib-3.0.0-SNAPSHOT.jar; \ + cmp BOOT-INF/lib/biopax3-exporter-lib-0.0.1-SNAPSHOT.jar /build/BioPAX3Exporter/lib/target/biopax3-exporter-lib-0.0.1-SNAPSHOT.jar; \ + kotlin=$(sed -n 's|.*\(.*\).*|\1|p' /build/BioPAX3Exporter/pom.xml); \ + paxtools=$(sed -n 's|.*\(.*\).*|\1|p' /build/BioPAX3Exporter/pom.xml); \ + [ "$(ls BOOT-INF/lib | grep '^kotlin-stdlib-[0-9]')" = "kotlin-stdlib-$kotlin.jar" ]; \ + [ "$(ls BOOT-INF/lib | grep '^paxtools-core-')" = "paxtools-core-$paxtools.jar" ] FROM eclipse-temurin:17-jre diff --git a/deploy/java-library/check.sh b/deploy/java-library/check.sh index 0be405f26..38932c61f 100755 --- a/deploy/java-library/check.sh +++ b/deploy/java-library/check.sh @@ -1,26 +1,33 @@ #!/usr/bin/env bash -# The library service, checked end to end over sbml-exporter's fixture graph -# (spec 015 T063): the image this repository builds, its answers for the -# fixture's three events compared with the expected files of the sbml-exporter -# commit it is pinned to, by that commit's own ci/compare.py. +# The library service, checked end to end (spec 015 T063, T069): the image this +# repository builds, over each exporter's own fixture graph, its answers +# compared with the expected files of the commit it is pinned to, by that +# commit's own instruments: +# - SBML over sbml-exporter's fixture: the three events' files, by its +# ci/compare.py; +# - BioPAX over BioPAX3Exporter's fixture: events of each kind and both +# species, each contained in its species' expected file, by its +# ci/contained.py, with the release's per-event xml:base. # # Usage: deploy/java-library/check.sh [image] -# GRAPH_PORT (7687) and FILES_PORT (8097) move the fixture graph and the files -# off ports a host already uses; the service keeps its own, 4360. +# GRAPH_PORT (7687), FILES_PORT (8097) and SERVICE_PORT (4360) move the fixture +# graph, the files and the service off ports a host already uses (a deployment's +# own service is on 4360). # Without an image, it builds one. Needs docker, curl and python3 with # python-libsbml 5.21.2 (pip install python-libsbml==5.21.2). set -euo pipefail graph_port=${GRAPH_PORT:-7687} files_port=${FILES_PORT:-8097} +service_port=${SERVICE_PORT:-4360} -# The check's own service takes 4360 on the host's network. One already there +# The check's own service takes its port on the host's network. One already there # (a deployment's) would answer the checks below in its place, over its graph. # Any answer refuses, a 503 too (a deployment whose graph is down), and it is # refused before the trap below, whose clean-up would remove another run's # containers. -if curl -s -m 5 -o /dev/null http://127.0.0.1:4360/health; then - echo "Something already answers on 127.0.0.1:4360; stop it, or run this elsewhere" >&2 +if curl -s -m 5 -o /dev/null http://127.0.0.1:$service_port/health; then + echo "Something already answers on 127.0.0.1:$service_port; stop it, or set SERVICE_PORT" >&2 exit 1 fi @@ -30,9 +37,11 @@ work=$(mktemp -d) # Named for this run, so a run that stops early removes only its own. neo=java-library-check-graph-$$ service=java-library-check-$$ +neo2=java-library-check-graph2-$$ +service2=java-library-check2-$$ # shellcheck disable=SC2317 # called by the trap cleanup() { - docker rm -f -v "$neo" "$service" >/dev/null 2>&1 || true + docker rm -f -v "$neo" "$service" "$neo2" "$service2" >/dev/null 2>&1 || true if [ -n "${server:-}" ]; then kill "$server" 2>/dev/null || true; fi rm -rf "$work" } @@ -51,15 +60,37 @@ curl -fsSL "https://codeload.github.com/reactome/sbml-exporter/tar.gz/$commit" \ | tar -xz -C "$work/sbml-exporter" --strip-components=1 --wildcards '*/ci/*' ci="$work/sbml-exporter/ci" -# The fixture graph, on loopback. -docker run -d --name "$neo" -p "127.0.0.1:$graph_port:7687" \ - -e NEO4J_AUTH=neo4j/fixturepass -e NEO4J_dbms_default__database=graph.db \ - -v "$ci/fixture/fixture.cypher:/fixture.cypher:ro" neo4j:4.4.48-community >/dev/null -for _ in $(seq 90); do - docker exec "$neo" cypher-shell -u neo4j -p fixturepass -d graph.db 'RETURN 1' >/dev/null 2>&1 && break - sleep 2 -done -docker exec "$neo" cypher-shell -u neo4j -p fixturepass -d graph.db -f /fixture.cypher >/dev/null +# A fixture graph, on loopback: start_graph +start_graph() { + docker run -d --name "$1" -p "127.0.0.1:$graph_port:7687" \ + -e NEO4J_AUTH=neo4j/fixturepass -e NEO4J_dbms_default__database=graph.db \ + -v "$2:/fixture.cypher:ro" neo4j:4.4.48-community >/dev/null + for _ in $(seq 90); do + docker exec "$1" cypher-shell -u neo4j -p fixturepass -d graph.db 'RETURN 1' >/dev/null 2>&1 && break + sleep 2 + done + docker exec "$1" cypher-shell -u neo4j -p fixturepass -d graph.db -f /fixture.cypher >/dev/null +} + +# The service over it, answering its health check: start_service +start_service() { + docker run -d --name "$1" --network host \ + -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \ + -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache \ + -e SERVER_PORT="$service_port" "$image" >/dev/null + for _ in $(seq 90); do + curl -fs http://127.0.0.1:$service_port/health >/dev/null && break + sleep 2 + done + if ! curl -fsS http://127.0.0.1:$service_port/health; then + echo "The service did not answer its health check. Its log:" >&2 + docker logs "$1" 2>&1 | tail -60 >&2 + exit 1 + fi + echo +} + +start_graph "$neo" "$ci/fixture/fixture.cypher" # The release's download directory, as far as the fixture needs it. mkdir -p "$work/www/97/diagram" @@ -67,21 +98,9 @@ cp "$ci"/fixture/R-*.json "$work/www/97/diagram/" python3 -m http.server "$files_port" --bind 127.0.0.1 --directory "$work/www" >/dev/null 2>&1 & server=$! -docker run -d --name "$service" --network host \ - -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \ - -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache "$image" >/dev/null -for _ in $(seq 90); do - curl -fs http://127.0.0.1:4360/health >/dev/null && break - sleep 2 -done -if ! curl -fsS http://127.0.0.1:4360/health; then - echo "The service did not answer its health check. Its log:" >&2 - docker logs "$service" 2>&1 | tail -60 >&2 - exit 1 -fi -echo +start_service "$service" -status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:4360/sbml/$1"; } +status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:$service_port/sbml/$1"; } fail=0 mkdir "$work/out" for id in $(find "$ci/fixture/expected" -name 'R-*.sbml' -exec basename {} .sbml \; | sort); do @@ -93,6 +112,57 @@ code=$(status R-HSA-1 /dev/null) code=$(status R-HSA-9613354.1 /dev/null) [ "$code" = 400 ] || { echo "R-HSA-9613354.1 (not a bare stable id): $code, expected 400" >&2; fail=1; } -python3 "$ci/validate.py" "$work/out" -python3 "$ci/compare.py" "$ci/fixture/expected" "$work/out" +python3 "$ci/validate.py" "$work/out" || fail=1 +python3 "$ci/compare.py" "$ci/fixture/expected" "$work/out" || fail=1 + +# BioPAX, over BioPAX3Exporter's fixture graph (release 98), in a graph and a +# service of their own. +docker rm -f -v "$neo" "$service" >/dev/null +bcommit=$(sed -n 's/^ARG BIOPAX3_EXPORTER_COMMIT=\([0-9a-f]\{40\}\)$/\1/p' "$root/deploy/java-library/Dockerfile" | sort -u) +[ "$(printf '%s\n' "$bcommit" | wc -l)" -eq 1 ] || { echo "The Dockerfile names more than one BioPAX3Exporter commit" >&2; exit 1; } +mkdir "$work/BioPAX3Exporter" +curl -fsSL "https://codeload.github.com/reactome/BioPAX3Exporter/tar.gz/$bcommit" \ + | tar -xz -C "$work/BioPAX3Exporter" --strip-components=1 --wildcards '*/ci/*' +bci="$work/BioPAX3Exporter/ci" +start_graph "$neo2" "$bci/fixture/fixture.cypher" +start_service "$service2" + +# Events of each kind and both species: of the graph's, by database id, the +# first of each kind that its species' expected file holds (the fixture has +# events beyond those files, such as an inferred event's source). Each line: +# stId, dbId, species. +candidates=$(docker exec "$neo2" cypher-shell -u neo4j -p fixturepass -d graph.db --format plain ' + CALL { + MATCH (e:TopLevelPathway {speciesName: "Homo sapiens"}) RETURN "pathway" AS kind, e + UNION ALL MATCH (e:Reaction {speciesName: "Homo sapiens"}) RETURN "reaction" AS kind, e + UNION ALL MATCH (e:BlackBoxEvent {speciesName: "Homo sapiens"}) RETURN "blackbox" AS kind, e + UNION ALL MATCH (e:ReactionLikeEvent {dbId: 174731}) RETURN "degradation" AS kind, e + UNION ALL MATCH (e:Pathway {speciesName: "Plasmodium falciparum"}) RETURN "falciparum" AS kind, e + } + RETURN kind + " " + e.stId + " " + e.dbId + " " + replace(e.speciesName, " ", "_") + ORDER BY kind, e.dbId' | tail -n +2 | tr -d '"') +events=$(while read -r kind id dbid species; do + grep -q ">$dbid" "$bci/fixture/expected/$species.owl" && echo "$kind $id $dbid $species" +done <<<"$candidates" | sort -u -k1,1 | cut -d' ' -f2-) +[ "$(printf '%s\n' "$events" | wc -l)" -eq 5 ] || { echo "Expected one event of each of five kinds in the fixture's files, got: $events" >&2; exit 1; } + +bstatus() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:$service_port/biopax/$1"; } +mkdir -p "$work/owl/Homo_sapiens" "$work/owl/Plasmodium_falciparum" +while read -r id dbid species; do + file="$work/owl/$species/$id.owl" + code=$(bstatus "$id" "$file") + [ "$code" = 200 ] || { echo "$id: $code, expected 200" >&2; fail=1; continue; } + # The release's per-event base: Reactome's BioPAX namespace, the release, the event's database id + grep -q "xml:base=\"http://www.reactome.org/biopax/98/$dbid#\"" "$file" \ + || { echo "$id: not the release's per-event xml:base" >&2; fail=1; } +done <<<"$events" +code=$(bstatus R-HSA-1 /dev/null) +[ "$code" = 404 ] || { echo "R-HSA-1 (not in the graph): $code, expected 404" >&2; fail=1; } +code=$(bstatus R-HSA-174731.1 /dev/null) +[ "$code" = 400 ] || { echo "R-HSA-174731.1 (not a bare stable id): $code, expected 400" >&2; fail=1; } + +# Each event's file holds nothing its species' file lacks, and is whole +for species in Homo_sapiens Plasmodium_falciparum; do + python3 -I "$bci/contained.py" "$bci/fixture/expected/$species.owl" "$work/owl/$species"/*.owl || fail=1 +done exit $fail diff --git a/deploy/java-library/pins.spec.ts b/deploy/java-library/pins.spec.ts index 565d43f38..1f863849a 100644 --- a/deploy/java-library/pins.spec.ts +++ b/deploy/java-library/pins.spec.ts @@ -15,7 +15,7 @@ const pins = readFileSync(join(__dirname, '../java/PINS.md'), 'utf8'); /** The Dockerfile's ARG _COMMIT= lines, by name, each stage's. */ function commitArgs(text: string) { const found = new Map>(); - for (const [, name = '', sha = ''] of text.matchAll(/^ARG ([A-Z_]+)_COMMIT=([0-9a-f]{40})$/gm)) + for (const [, name = '', sha = ''] of text.matchAll(/^ARG ([A-Z0-9_]+)_COMMIT=([0-9a-f]{40})$/gm)) found.set(name, (found.get(name) ?? new Set()).add(sha)); return found; } @@ -30,6 +30,7 @@ const REPOSITORIES: Record = { REACTOME_PARENT: 'reactome-parent', GRAPH_CORE: 'graph-core', SBML_EXPORTER: 'sbml-exporter', + BIOPAX3_EXPORTER: 'BioPAX3Exporter', }; describe('the library service pins', () => { diff --git a/deploy/java-library/pom.xml b/deploy/java-library/pom.xml index 9abca68ec..5c9aa9f9b 100644 --- a/deploy/java-library/pom.xml +++ b/deploy/java-library/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - + org.reactome.server java-library 0.1.0 @@ -26,6 +26,12 @@ 3.0.0-SNAPSHOT + + 0.0.1-SNAPSHOT + + 2.4.21 @@ -34,6 +40,11 @@ sbml-exporter-lib ${sbml-exporter-lib.version} + + org.reactome.server.tools + biopax3-exporter-lib + ${biopax3-exporter-lib.version} + org.springframework.boot spring-boot-starter-web diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java b/deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java similarity index 69% rename from deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java rename to deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java index ba89c59e1..ad46a550f 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.FileSystemResource; import org.springframework.core.io.Resource; import org.springframework.http.HttpStatus; @@ -20,21 +21,25 @@ /** * The service's answers to node, over loopback. Node writes the public answer: its headers, its errors in the - * service's envelope and its cache labels (spec 015 T065). This writes only the body, or a status and a reason. + * service's envelope and its cache labels (spec 015 T065, T070). This writes only the body, or a status and a reason. */ @RestController -public class SbmlController { +public class ExportController { - private static final Logger logger = LoggerFactory.getLogger(SbmlController.class); + private static final Logger logger = LoggerFactory.getLogger(ExportController.class); private static final Pattern ST_ID = Pattern.compile("R-[A-Z]{3}-\\d+"); static final MediaType SBML = MediaType.parseMediaType("application/sbml+xml;charset=UTF-8"); + static final MediaType BIOPAX = MediaType.parseMediaType("application/rdf+xml;charset=UTF-8"); - private final SbmlFiles files; - private final SbmlFiles.Graph graph; + private final ExportFiles sbml; + private final ExportFiles biopax; + private final ExportFiles.Graph graph; private final Properties pins; - public SbmlController(SbmlFiles files, SbmlFiles.Graph graph, Properties pins) { - this.files = files; + public ExportController(@Qualifier("sbmlFiles") ExportFiles sbml, @Qualifier("biopaxFiles") ExportFiles biopax, + ExportFiles.Graph graph, Properties pins) { + this.sbml = sbml; + this.biopax = biopax; this.graph = graph; this.pins = pins; } @@ -42,13 +47,26 @@ public SbmlController(SbmlFiles files, SbmlFiles.Graph graph, Properties pins) { /** An event's SBML, by its stable id without a version: node resolves any other form first. */ @GetMapping("/sbml/{stId}") public ResponseEntity sbml(@PathVariable String stId) throws InterruptedException { + return answer(sbml, SBML, stId); + } + + /** + * An event's BioPAX Level 3, by its stable id without a version. Node decides 404 and 400 first (T070); the + * service's own 404 is a backstop. + */ + @GetMapping("/biopax/{stId}") + public ResponseEntity biopax(@PathVariable String stId) throws InterruptedException { + return answer(biopax, BIOPAX, stId); + } + + private static ResponseEntity answer(ExportFiles files, MediaType type, String stId) throws InterruptedException { if (!ST_ID.matcher(stId).matches()) return reason(HttpStatus.BAD_REQUEST, "Not a stable id: " + stId); try { Optional file = files.get(stId); if (file.isEmpty()) return reason(HttpStatus.NOT_FOUND, "No event " + stId + " in the graph"); Resource body = new FileSystemResource(file.get()); - return ResponseEntity.ok().contentType(SBML).body(body); - } catch (SbmlFiles.DiagramUnavailable e) { + return ResponseEntity.ok().contentType(type).body(body); + } catch (ExportFiles.DiagramUnavailable e) { logger.warn("{}: {}", stId, e.getMessage()); return reason(HttpStatus.SERVICE_UNAVAILABLE, e.getMessage()); } catch (RuntimeException e) { diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlFiles.java b/deploy/java-library/src/main/java/org/reactome/server/library/ExportFiles.java similarity index 77% rename from deploy/java-library/src/main/java/org/reactome/server/library/SbmlFiles.java rename to deploy/java-library/src/main/java/org/reactome/server/library/ExportFiles.java index 83a0a89e7..97f8b063e 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/SbmlFiles.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/ExportFiles.java @@ -20,17 +20,18 @@ import org.slf4j.LoggerFactory; /** - * The SBML files, kept on disk per release as ContentService keeps its exports. + * One export's files (SBML, BioPAX), kept on disk per release as ContentService keeps its exports. * *

The release is the graph's, read for every request, so a graph moved to a new release is never answered from the * old one's files, and the old release's files are removed when it is first seen. One id is converted once at a time: - * a second request for it waits for the first. Node's share of 2 is the limit on conversions (spec 015 T065); the cap - * here is a backstop. + * a second request for it waits for the first. Node gives each export a share of 2 (spec 015 T065, T070); the + * conversions' cap is shared by every export in the service, so the heap holds at most that many at once whatever + * node sends (T069). * *

Removing files is best effort: a release can change while a conversion of the old one is still writing, and that * must not fail the new release's request. What is left behind goes when the next release is first seen. */ -public class SbmlFiles { +public class ExportFiles { /** What the service needs from the graph. */ public interface Graph { @@ -42,7 +43,7 @@ public interface Graph { /** An event, as far as the diagram files are concerned. */ public static final class Event { - /** A pathway the graph says has a diagram of its own: its layout needs the diagram file. */ + /** A pathway the graph says has a diagram of its own: its SBML layout needs the diagram file. */ public final boolean hasDiagram; public Event(boolean hasDiagram) { @@ -50,7 +51,7 @@ public Event(boolean hasDiagram) { } } - /** sbml-exporter's library, writing {@code {stId}.sbml} into a folder. */ + /** An exporter's library, writing {@code {stId}{extension}} into a folder. */ public interface Converter { void convert(String stId, int release, Path diagrams, Path into) throws Exception; } @@ -72,34 +73,50 @@ public DiagramUnavailable(String message, Throwable cause) { } } - private static final Logger logger = LoggerFactory.getLogger(SbmlFiles.class); + private static final Logger logger = LoggerFactory.getLogger(ExportFiles.class); + private final String kind; + private final String extension; private final Graph graph; private final Converter converter; private final DiagramSource diagrams; private final Path cache; - private final Semaphore conversions = new Semaphore(2); + private final Semaphore conversions; private final Map> running = new ConcurrentHashMap<>(); private volatile int seen = -1; - public SbmlFiles(Graph graph, Converter converter, DiagramSource diagrams, Path cache) { + /** + * @param kind the export's folder in the cache, {@code sbml} or {@code biopax} + * @param extension what the converter names its file, {@code .sbml} or {@code .owl} + * @param diagrams where a pathway's diagram file comes from, or null for an export that needs none + * @param conversions the cap on conversions at once, shared by the service's exports + */ + public ExportFiles(String kind, String extension, Graph graph, Converter converter, DiagramSource diagrams, + Path cache, Semaphore conversions) { + this.kind = kind; + this.extension = extension; this.graph = graph; this.converter = converter; this.diagrams = diagrams; this.cache = cache; + this.conversions = conversions; // Work in progress when the service last stopped, which nothing will finish - delete(cache.resolve("tmp")); + delete(work()); + } + + private Path work() { + return cache.resolve("tmp").resolve(kind); } /** - * The event's SBML file, or empty when the graph has no such event. + * The event's file, or empty when the graph has no such event. * * @throws DiagramUnavailable when its diagram file cannot be had */ public Optional get(String stId) throws DiagramUnavailable, InterruptedException { int release = graph.release(); forgetOtherReleases(release); - Path file = cache.resolve("sbml").resolve(String.valueOf(release)).resolve(stId + ".sbml"); + Path file = cache.resolve(kind).resolve(String.valueOf(release)).resolve(stId + extension); if (Files.isRegularFile(file)) return Optional.of(file); Optional event = graph.event(stId); @@ -124,13 +141,13 @@ private Path convert(String stId, int release, Event event, Path file) throws Ex conversions.acquire(); try { Path diagramFolder = Files.createDirectories(cache.resolve("diagram").resolve(String.valueOf(release))); - if (event.hasDiagram && !Files.isRegularFile(diagramFolder.resolve(stId + ".json"))) { + if (diagrams != null && event.hasDiagram && !Files.isRegularFile(diagramFolder.resolve(stId + ".json"))) { diagrams.fetch(stId, release, diagramFolder); } - Path work = Files.createDirectories(cache.resolve("tmp").resolve(UUID.randomUUID().toString())); + Path work = Files.createDirectories(work().resolve(UUID.randomUUID().toString())); try { converter.convert(stId, release, diagramFolder, work); - Path written = work.resolve(stId + ".sbml"); + Path written = work.resolve(stId + extension); if (!Files.isRegularFile(written)) throw new IllegalStateException("The converter wrote no " + written); Files.createDirectories(file.getParent()); return Files.move(written, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); @@ -159,8 +176,8 @@ private void forgetOtherReleases(int release) { if (seen == release) return; synchronized (this) { if (seen == release) return; - for (String kind : new String[]{"sbml", "diagram"}) { - Path folder = cache.resolve(kind); + for (String folderName : new String[]{kind, "diagram"}) { + Path folder = cache.resolve(folderName); if (!Files.isDirectory(folder)) continue; try (DirectoryStream releases = Files.newDirectoryStream(folder)) { for (Path other : releases) { diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java b/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java index 6f61034f1..c31c02f4f 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/HttpDiagramSource.java @@ -23,7 +23,7 @@ * per converted pathway that has a diagram; the caller keeps it per release, so a file is kept only once it is * whole and is the pathway's: the converter would refuse anything else on every request until the next release. */ -public class HttpDiagramSource implements SbmlFiles.DiagramSource { +public class HttpDiagramSource implements ExportFiles.DiagramSource { private static final ObjectMapper JSON = new ObjectMapper(); @@ -45,7 +45,7 @@ public HttpDiagramSource(URI base) { } @Override - public void fetch(String stId, int release, Path into) throws SbmlFiles.DiagramUnavailable { + public void fetch(String stId, int release, Path into) throws ExportFiles.DiagramUnavailable { URI uri = base.resolve(release + "/diagram/" + stId + ".json"); Path part = into.resolve(stId + ".json.part"); try { @@ -59,27 +59,27 @@ public void fetch(String stId, int release, Path into) throws SbmlFiles.DiagramU response = download.get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { download.cancel(true); - throw new SbmlFiles.DiagramUnavailable(uri + " did not arrive within " + timeout.toSeconds() + " s", e); + throw new ExportFiles.DiagramUnavailable(uri + " did not arrive within " + timeout.toSeconds() + " s", e); } catch (ExecutionException e) { - throw new SbmlFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getCause(), e.getCause()); + throw new ExportFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getCause(), e.getCause()); } catch (InterruptedException e) { // Waiting is interrupted, not the transfer: it is stopped here, as on a timeout download.cancel(true); throw e; } if (response.statusCode() != 200) { - throw new SbmlFiles.DiagramUnavailable(uri + " answered " + response.statusCode(), null); + throw new ExportFiles.DiagramUnavailable(uri + " answered " + response.statusCode(), null); } if (!isTheDiagramOf(stId, part)) { - throw new SbmlFiles.DiagramUnavailable(uri + " is not " + stId + "'s diagram", null); + throw new ExportFiles.DiagramUnavailable(uri + " is not " + stId + "'s diagram", null); } Files.move(part, into.resolve(stId + ".json"), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - throw new SbmlFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getMessage(), e); + throw new ExportFiles.DiagramUnavailable("Cannot fetch " + uri + ": " + e.getMessage(), e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new SbmlFiles.DiagramUnavailable("Interrupted fetching " + uri, e); + throw new ExportFiles.DiagramUnavailable("Interrupted fetching " + uri, e); } finally { try { Files.deleteIfExists(part); diff --git a/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java b/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java index 7384af2d3..f4995633d 100644 --- a/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java +++ b/deploy/java-library/src/main/java/org/reactome/server/library/LibraryApplication.java @@ -1,9 +1,11 @@ package org.reactome.server.library; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Optional; import java.util.Properties; +import java.util.concurrent.Semaphore; import org.reactome.server.graph.config.GraphCoreNeo4jConfig; import org.reactome.server.graph.domain.model.Event; @@ -11,6 +13,7 @@ import org.reactome.server.graph.service.AdvancedDatabaseObjectService; import org.reactome.server.graph.service.DatabaseObjectService; import org.reactome.server.graph.service.GeneralService; +import org.reactome.server.tools.WriteBioPAX3; import org.reactome.server.tools.sbml.converter.SbmlConverter; import org.reactome.server.tools.sbml.layout.DiagramFiles; import org.springframework.boot.SpringApplication; @@ -19,8 +22,8 @@ import org.springframework.context.annotation.Import; /** - * Spec 010's library service, started in this repository with SBML as its only operation (spec 015 T063). Node - * answers the public URLs and calls this over loopback; this writes only the body. + * Spec 010's library service, started in this repository with SBML (spec 015 T063) and BioPAX Level 3 (T069) as its + * operations. Node answers the public URLs and calls this over loopback; this writes only the body. */ @SpringBootApplication @Import(GraphCoreNeo4jConfig.class) @@ -39,36 +42,53 @@ public static void main(String[] args) { } @Bean - public SbmlFiles.Graph graph(DatabaseObjectService ds, GeneralService general) { - return new SbmlFiles.Graph() { + public ExportFiles.Graph graph(DatabaseObjectService ds, GeneralService general) { + return new ExportFiles.Graph() { @Override public int release() { return general.getDBInfo().getVersion(); } @Override - public Optional event(String stId) { + public Optional event(String stId) { Object found = ds.findById(stId); if (!(found instanceof Event)) return Optional.empty(); boolean hasDiagram = found instanceof Pathway && Boolean.TRUE.equals(((Pathway) found).getHasDiagram()); - return Optional.of(new SbmlFiles.Event(hasDiagram)); + return Optional.of(new ExportFiles.Event(hasDiagram)); } }; } + /** The conversions at once, across every export: two, within the heap (spec 015 T069). */ @Bean - public SbmlFiles.Converter converter(DatabaseObjectService ds, AdvancedDatabaseObjectService ads) { - return (stId, release, diagrams, into) -> { + public Semaphore conversions() { + return new Semaphore(2); + } + + @Bean + public ExportFiles sbmlFiles(Settings settings, ExportFiles.Graph graph, DatabaseObjectService ds, + AdvancedDatabaseObjectService ads, Semaphore conversions) { + ExportFiles.Converter converter = (stId, release, diagrams, into) -> { Event event = ds.findById(stId); - SbmlConverter converter = new SbmlConverter(event, release, ds, ads, new DiagramFiles(diagrams, release, release)); - converter.convert(); - converter.writeToFile(into.toString()); + SbmlConverter sbml = new SbmlConverter(event, release, ds, ads, new DiagramFiles(diagrams, release, release)); + sbml.convert(); + sbml.writeToFile(into.toString()); }; + return new ExportFiles("sbml", ".sbml", graph, converter, new HttpDiagramSource(settings.diagramBase), + settings.cache, conversions); } + /** BioPAX3Exporter's library: one event with all it holds, with the release's per-event xml:base. */ @Bean - public SbmlFiles sbmlFiles(Settings settings, SbmlFiles.Graph graph, SbmlFiles.Converter converter) { - return new SbmlFiles(graph, converter, new HttpDiagramSource(settings.diagramBase), settings.cache); + public ExportFiles biopaxFiles(Settings settings, ExportFiles.Graph graph, DatabaseObjectService ds, + Semaphore conversions) { + ExportFiles.Converter converter = (stId, release, diagrams, into) -> { + Event event = ds.findById(stId); + WriteBioPAX3 biopax = new WriteBioPAX3(event, release); + biopax.createModel(); + biopax.toFile(new File(into.toFile(), stId + ".owl")); + }; + return new ExportFiles("biopax", ".owl", graph, converter, null, settings.cache, conversions); } /** The pinned commits the image was built from, written by the Dockerfile (deploy/java/PINS.md). */ diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java new file mode 100644 index 000000000..480eea7bf --- /dev/null +++ b/deploy/java-library/src/test/java/org/reactome/server/library/ExportControllerTest.java @@ -0,0 +1,117 @@ +package org.reactome.server.library; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.Semaphore; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.http.ResponseEntity; + +class ExportControllerTest { + + @TempDir + Path cache; + + private final ExportFiles.Graph graph = new ExportFiles.Graph() { + @Override + public int release() { + return 97; + } + + @Override + public Optional event(String stId) { + return stId.equals("R-HSA-1") ? Optional.of(new ExportFiles.Event(true)) : Optional.empty(); + } + }; + + private final ExportFiles.Converter owl = (stId, rel, d, into) -> Files.writeString(into.resolve(stId + ".owl"), ""); + + private ExportController controller(ExportFiles.DiagramSource diagrams, ExportFiles.Converter converter) { + return controller(diagrams, converter, owl); + } + + private ExportController controller(ExportFiles.DiagramSource diagrams, ExportFiles.Converter converter, + ExportFiles.Converter biopax) { + Semaphore cap = new Semaphore(2); + return new ExportController(new ExportFiles("sbml", ".sbml", graph, converter, diagrams, cache, cap), + new ExportFiles("biopax", ".owl", graph, biopax, null, cache, cap), graph, new Properties()); + } + + private final ExportFiles.Converter writes = (stId, rel, d, into) -> Files.writeString(into.resolve(stId + ".sbml"), ""); + private final ExportFiles.DiagramSource fetches = (stId, rel, into) -> { + try { + Files.writeString(into.resolve(stId + ".json"), "{}"); + } catch (java.io.IOException e) { + throw new ExportFiles.DiagramUnavailable(e.getMessage(), e); + } + }; + + @Test + void answersTheFileAsSbml() throws Exception { + ResponseEntity answer = controller(fetches, writes).sbml("R-HSA-1"); + assertEquals(200, answer.getStatusCodeValue()); + assertEquals(ExportController.SBML, answer.getHeaders().getContentType()); + } + + @Test + void anIdNotInTheGraphIs404() throws Exception { + assertEquals(404, controller(fetches, writes).sbml("R-HSA-9").getStatusCodeValue()); + } + + @Test + void anythingButAStableIdIs400() throws Exception { + for (String id : new String[]{"R-HSA-1.2", "123", "../R-HSA-1", "r-hsa-1"}) { + assertEquals(400, controller(fetches, writes).sbml(id).getStatusCodeValue(), id); + } + } + + @Test + void aDiagramThatCannotBeHadIs503() throws Exception { + ExportFiles.DiagramSource failing = (stId, rel, into) -> { + throw new ExportFiles.DiagramUnavailable("answered 404", null); + }; + ResponseEntity answer = controller(failing, writes).sbml("R-HSA-1"); + assertEquals(503, answer.getStatusCodeValue()); + assertEquals(Map.of("error", "answered 404"), answer.getBody()); + } + + @Test + void aConversionThatFailsIs500() throws Exception { + ExportFiles.Converter failing = (stId, rel, d, into) -> { + throw new IllegalStateException("no reactions"); + }; + assertEquals(500, controller(fetches, failing).sbml("R-HSA-1").getStatusCodeValue()); + } + + @Test + void answersBiopaxAsRdf() throws Exception { + ResponseEntity answer = controller(fetches, writes).biopax("R-HSA-1"); + assertEquals(200, answer.getStatusCodeValue()); + assertEquals(ExportController.BIOPAX, answer.getHeaders().getContentType()); + assertEquals("R-HSA-1.owl", ((org.springframework.core.io.Resource) answer.getBody()).getFilename()); + } + + @Test + void biopaxNeedsNoDiagram() throws Exception { + ExportFiles.DiagramSource failing = (stId, rel, into) -> { + throw new ExportFiles.DiagramUnavailable("answered 404", null); + }; + assertEquals(200, controller(failing, writes).biopax("R-HSA-1").getStatusCodeValue()); + } + + @Test + void biopaxAnswersTheSameErrors() throws Exception { + assertEquals(404, controller(fetches, writes).biopax("R-HSA-9").getStatusCodeValue()); + assertEquals(400, controller(fetches, writes).biopax("R-HSA-1.2").getStatusCodeValue()); + ExportFiles.Converter failing = (stId, rel, d, into) -> { + throw new IllegalStateException("no model"); + }; + assertEquals(500, controller(fetches, writes, failing).biopax("R-HSA-1").getStatusCodeValue()); + } +} diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlFilesTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/ExportFilesTest.java similarity index 51% rename from deploy/java-library/src/test/java/org/reactome/server/library/SbmlFilesTest.java rename to deploy/java-library/src/test/java/org/reactome/server/library/ExportFilesTest.java index 37bf24688..9cecb3b9b 100644 --- a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlFilesTest.java +++ b/deploy/java-library/src/test/java/org/reactome/server/library/ExportFilesTest.java @@ -15,13 +15,14 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -class SbmlFilesTest { +class ExportFilesTest { @TempDir Path cache; @@ -31,45 +32,115 @@ class SbmlFilesTest { private final AtomicInteger lookups = new AtomicInteger(); /** A graph with a pathway that has a diagram, one that has none, and nothing else. */ - private final SbmlFiles.Graph graph = new SbmlFiles.Graph() { + private final ExportFiles.Graph graph = new ExportFiles.Graph() { @Override public int release() { return release; } @Override - public Optional event(String stId) { + public Optional event(String stId) { lookups.incrementAndGet(); - if (stId.equals("R-HSA-1")) return Optional.of(new SbmlFiles.Event(true)); - if (stId.equals("R-HSA-2")) return Optional.of(new SbmlFiles.Event(false)); + if (stId.equals("R-HSA-1")) return Optional.of(new ExportFiles.Event(true)); + if (stId.equals("R-HSA-2")) return Optional.of(new ExportFiles.Event(false)); return Optional.empty(); } }; - private final SbmlFiles.Converter converter = (stId, rel, diagrams, into) -> { + private final ExportFiles.Converter converter = (stId, rel, diagrams, into) -> { calls.add("convert " + stId + " " + rel + " diagram=" + Files.exists(diagrams.resolve(stId + ".json"))); Files.writeString(into.resolve(stId + ".sbml"), ""); }; - private final SbmlFiles.DiagramSource diagrams = (stId, rel, into) -> { + private final ExportFiles.DiagramSource diagrams = (stId, rel, into) -> { calls.add("fetch " + stId + " " + rel); try { Files.writeString(into.resolve(stId + ".json"), "{}"); } catch (java.io.IOException e) { - throw new SbmlFiles.DiagramUnavailable(e.getMessage(), e); + throw new ExportFiles.DiagramUnavailable(e.getMessage(), e); } }; + private final Semaphore cap = new Semaphore(2); + + private ExportFiles sbml(ExportFiles.Converter converter, ExportFiles.DiagramSource diagrams) { + return new ExportFiles("sbml", ".sbml", graph, converter, diagrams, cache, cap); + } + + /** BioPAX needs no diagram file: the export is made with none. */ + private final ExportFiles.Converter owl = (stId, rel, diagrams, into) -> { + calls.add("biopax " + stId + " " + rel); + Files.writeString(into.resolve(stId + ".owl"), ""); + }; + + private ExportFiles biopax(ExportFiles.Converter converter) { + return new ExportFiles("biopax", ".owl", graph, converter, null, cache, cap); + } + + @Test + void anExportWithNoDiagramSourceNeverFetchesOne() throws Exception { + ExportFiles files = biopax(owl); + Path file = files.get("R-HSA-1").orElseThrow(); + assertEquals(cache.resolve("biopax/97/R-HSA-1.owl"), file); + assertEquals(List.of("biopax R-HSA-1 97"), calls); + } + + @Test + void eachExportKeepsItsOwnFiles() throws Exception { + ExportFiles sbml = sbml(converter, diagrams); + ExportFiles biopax = biopax(owl); + assertEquals("", Files.readString(sbml.get("R-HSA-2").orElseThrow())); + assertEquals("", Files.readString(biopax.get("R-HSA-2").orElseThrow())); + // A new release seen by one export leaves the other's files to it, and each then converts afresh + release = 98; + biopax.get("R-HSA-2"); + assertTrue(Files.exists(cache.resolve("sbml/97/R-HSA-2.sbml"))); + assertFalse(Files.exists(cache.resolve("biopax/97"))); + } + + @Test + void theCapIsSharedByTheExports() throws Exception { + CountDownLatch inside = new CountDownLatch(2); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger at = new AtomicInteger(); + AtomicInteger most = new AtomicInteger(); + ExportFiles.Converter held = (stId, rel, d, into) -> { + most.accumulateAndGet(at.incrementAndGet(), Math::max); + inside.countDown(); + assertTrue(release.await(10, TimeUnit.SECONDS)); + at.decrementAndGet(); + Files.writeString(into.resolve(stId + (into.toString().contains("biopax") ? ".owl" : ".sbml")), "x"); + }; + ExportFiles sbml = sbml(held, diagrams); + ExportFiles biopax = biopax(held); + ExecutorService pool = Executors.newFixedThreadPool(3); + try { + Future> a = pool.submit(() -> sbml.get("R-HSA-2")); + Future> b = pool.submit(() -> biopax.get("R-HSA-2")); + assertTrue(inside.await(10, TimeUnit.SECONDS)); + Future> c = pool.submit(() -> biopax.get("R-HSA-1")); + Thread.sleep(200); // c waits for a place: two conversions are running, one of each export + assertEquals(2, at.get()); + release.countDown(); + a.get(10, TimeUnit.SECONDS); + b.get(10, TimeUnit.SECONDS); + c.get(10, TimeUnit.SECONDS); + } finally { + pool.shutdownNow(); + } + assertEquals(2, most.get()); + } + @Test void anIdNotInTheGraphIsEmpty() throws Exception { - SbmlFiles files = new SbmlFiles(graph, converter, diagrams, cache); + ExportFiles files = sbml(converter, diagrams); assertTrue(files.get("R-HSA-3").isEmpty()); assertTrue(calls.isEmpty()); } @Test void convertsOnceAndKeepsTheFile() throws Exception { - SbmlFiles files = new SbmlFiles(graph, converter, diagrams, cache); + ExportFiles files = sbml(converter, diagrams); Path first = files.get("R-HSA-2").orElseThrow(); Path second = files.get("R-HSA-2").orElseThrow(); assertEquals(first, second); @@ -81,7 +152,7 @@ void convertsOnceAndKeepsTheFile() throws Exception { @Test void fetchesTheDiagramOfAPathwayThatHasOne() throws Exception { - SbmlFiles files = new SbmlFiles(graph, converter, diagrams, cache); + ExportFiles files = sbml(converter, diagrams); files.get("R-HSA-1"); files.get("R-HSA-2"); assertEquals(List.of("fetch R-HSA-1 97", "convert R-HSA-1 97 diagram=true", "convert R-HSA-2 97 diagram=false"), @@ -90,11 +161,11 @@ void fetchesTheDiagramOfAPathwayThatHasOne() throws Exception { @Test void aDiagramThatCannotBeHadIsNeverSbmlWithoutItsLayout() throws Exception { - SbmlFiles.DiagramSource failing = (stId, rel, into) -> { - throw new SbmlFiles.DiagramUnavailable("answered 503", null); + ExportFiles.DiagramSource failing = (stId, rel, into) -> { + throw new ExportFiles.DiagramUnavailable("answered 503", null); }; - SbmlFiles files = new SbmlFiles(graph, converter, failing, cache); - assertThrows(SbmlFiles.DiagramUnavailable.class, () -> files.get("R-HSA-1")); + ExportFiles files = sbml(converter, failing); + assertThrows(ExportFiles.DiagramUnavailable.class, () -> files.get("R-HSA-1")); assertTrue(calls.isEmpty()); assertFalse(Files.exists(cache.resolve("sbml/97/R-HSA-1.sbml"))); } @@ -102,11 +173,11 @@ void aDiagramThatCannotBeHadIsNeverSbmlWithoutItsLayout() throws Exception { @Test void aFailedConversionIsTriedAgain() throws Exception { AtomicInteger attempts = new AtomicInteger(); - SbmlFiles.Converter once = (stId, rel, d, into) -> { + ExportFiles.Converter once = (stId, rel, d, into) -> { if (attempts.incrementAndGet() == 1) throw new IllegalStateException("the first attempt fails"); Files.writeString(into.resolve(stId + ".sbml"), ""); }; - SbmlFiles files = new SbmlFiles(graph, once, diagrams, cache); + ExportFiles files = sbml(once, diagrams); assertThrows(IllegalStateException.class, () -> files.get("R-HSA-2")); assertEquals("", Files.readString(files.get("R-HSA-2").orElseThrow())); assertEquals(2, attempts.get()); @@ -114,15 +185,18 @@ void aFailedConversionIsTriedAgain() throws Exception { @Test void workLeftByAStoppedServiceIsRemoved() throws Exception { - Path stale = Files.createDirectories(cache.resolve("tmp/left-by-a-killed-run")); + Path stale = Files.createDirectories(cache.resolve("tmp/sbml/left-by-a-killed-run")); Files.writeString(stale.resolve("R-HSA-2.sbml"), " { + ExportFiles.Converter slow = (stId, rel, d, into) -> { conversions.incrementAndGet(); started.countDown(); assertTrue(release.await(10, TimeUnit.SECONDS)); Files.writeString(into.resolve(stId + ".sbml"), ""); }; - SbmlFiles files = new SbmlFiles(graph, slow, diagrams, cache); + ExportFiles files = sbml(slow, diagrams); ExecutorService pool = Executors.newFixedThreadPool(2); try { Future> a = pool.submit(() -> files.get("R-HSA-2")); diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java index 8b26e5793..ce88a0022 100644 --- a/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java +++ b/deploy/java-library/src/test/java/org/reactome/server/library/HttpDiagramSourceTest.java @@ -124,7 +124,7 @@ void writesTheReleasesFile() throws Exception { @Test void aMissingFileIsUnavailableAndLeavesNothing() throws Exception { - SbmlFiles.DiagramUnavailable e = assertThrows(SbmlFiles.DiagramUnavailable.class, + ExportFiles.DiagramUnavailable e = assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-2", 97, into)); assertTrue(e.getMessage().contains("answered 404"), e.getMessage()); try (var left = Files.list(into)) { @@ -134,7 +134,7 @@ void aMissingFileIsUnavailableAndLeavesNothing() throws Exception { @Test void aPageInPlaceOfTheFileIsUnavailableAndNotKept() throws Exception { - SbmlFiles.DiagramUnavailable e = assertThrows(SbmlFiles.DiagramUnavailable.class, + ExportFiles.DiagramUnavailable e = assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-3", 97, into)); assertTrue(e.getMessage().contains("not R-HSA-3's diagram"), e.getMessage()); try (var left = Files.list(into)) { @@ -144,7 +144,7 @@ void aPageInPlaceOfTheFileIsUnavailableAndNotKept() throws Exception { @Test void anotherPathwaysDiagramIsUnavailableAndNotKept() throws Exception { - SbmlFiles.DiagramUnavailable e = assertThrows(SbmlFiles.DiagramUnavailable.class, + ExportFiles.DiagramUnavailable e = assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-4", 97, into)); assertTrue(e.getMessage().contains("not R-HSA-4's diagram"), e.getMessage()); try (var left = Files.list(into)) { @@ -156,7 +156,7 @@ void anotherPathwaysDiagramIsUnavailableAndNotKept() throws Exception { void aBodyThatStallsIsUnavailableWithinTheTimeout() throws Exception { // The request's own timeout covers only the wait for the headers assertTimeoutPreemptively(Duration.ofSeconds(15), - () -> assertThrows(SbmlFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-5", 97, into))); + () -> assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-5", 97, into))); try (var left = Files.list(into)) { assertEquals(0, left.count()); } @@ -164,7 +164,7 @@ void aBodyThatStallsIsUnavailableWithinTheTimeout() throws Exception { @Test void aDownloadThatTimesOutIsAbandoned() throws Exception { - assertThrows(SbmlFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-6", 97, into)); + assertThrows(ExportFiles.DiagramUnavailable.class, () -> source().fetch("R-HSA-6", 97, into)); assertTrue(abandoned.await(10, TimeUnit.SECONDS), "the server was still sending"); } @@ -187,7 +187,7 @@ void anInterruptedDownloadIsAbandoned() throws Exception { fetching.interrupt(); fetching.join(10_000); assertFalse(fetching.isAlive(), "the fetch did not return"); - assertTrue(outcome.get() instanceof SbmlFiles.DiagramUnavailable, String.valueOf(outcome.get())); + assertTrue(outcome.get() instanceof ExportFiles.DiagramUnavailable, String.valueOf(outcome.get())); assertTrue(stillInterrupted.get(), "the interrupt was swallowed"); assertTrue(abandoned.await(10, TimeUnit.SECONDS), "the server was still sending"); } diff --git a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java b/deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java deleted file mode 100644 index f3ba26905..000000000 --- a/deploy/java-library/src/test/java/org/reactome/server/library/SbmlControllerTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.reactome.server.library; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.springframework.http.ResponseEntity; - -class SbmlControllerTest { - - @TempDir - Path cache; - - private final SbmlFiles.Graph graph = new SbmlFiles.Graph() { - @Override - public int release() { - return 97; - } - - @Override - public Optional event(String stId) { - return stId.equals("R-HSA-1") ? Optional.of(new SbmlFiles.Event(true)) : Optional.empty(); - } - }; - - private SbmlController controller(SbmlFiles.DiagramSource diagrams, SbmlFiles.Converter converter) { - return new SbmlController(new SbmlFiles(graph, converter, diagrams, cache), graph, new Properties()); - } - - private final SbmlFiles.Converter writes = (stId, rel, d, into) -> Files.writeString(into.resolve(stId + ".sbml"), ""); - private final SbmlFiles.DiagramSource fetches = (stId, rel, into) -> { - try { - Files.writeString(into.resolve(stId + ".json"), "{}"); - } catch (java.io.IOException e) { - throw new SbmlFiles.DiagramUnavailable(e.getMessage(), e); - } - }; - - @Test - void answersTheFileAsSbml() throws Exception { - ResponseEntity answer = controller(fetches, writes).sbml("R-HSA-1"); - assertEquals(200, answer.getStatusCodeValue()); - assertEquals(SbmlController.SBML, answer.getHeaders().getContentType()); - } - - @Test - void anIdNotInTheGraphIs404() throws Exception { - assertEquals(404, controller(fetches, writes).sbml("R-HSA-9").getStatusCodeValue()); - } - - @Test - void anythingButAStableIdIs400() throws Exception { - for (String id : new String[]{"R-HSA-1.2", "123", "../R-HSA-1", "r-hsa-1"}) { - assertEquals(400, controller(fetches, writes).sbml(id).getStatusCodeValue(), id); - } - } - - @Test - void aDiagramThatCannotBeHadIs503() throws Exception { - SbmlFiles.DiagramSource failing = (stId, rel, into) -> { - throw new SbmlFiles.DiagramUnavailable("answered 404", null); - }; - ResponseEntity answer = controller(failing, writes).sbml("R-HSA-1"); - assertEquals(503, answer.getStatusCodeValue()); - assertEquals(Map.of("error", "answered 404"), answer.getBody()); - } - - @Test - void aConversionThatFailsIs500() throws Exception { - SbmlFiles.Converter failing = (stId, rel, d, into) -> { - throw new IllegalStateException("no reactions"); - }; - assertEquals(500, controller(fetches, failing).sbml("R-HSA-1").getStatusCodeValue()); - } -} diff --git a/deploy/java/PINS.md b/deploy/java/PINS.md index f982e5b60..30fbf42ef 100644 --- a/deploy/java/PINS.md +++ b/deploy/java/PINS.md @@ -17,6 +17,7 @@ The Reactome Java this deployment builds (spec 010), each pinned to the commit t | reactome-parent 1.0.5-SNAPSHOT (and its BOM) | `reactome/reactome-parent` | `d93ad5a` (branch `feature/new-pwb`) | the BOM equal to the published snapshot; the parent differs only in Batik and FOP's versions (published 1.18 and 2.10, here 1.16 and 2.8), which nothing built on it here uses: the published snapshot matches no pushed commit | 17 | | graph-core 3.0.0-SNAPSHOT | `reactome/graph-core` | `62aef35` | the deployed ContentService's jar is the published snapshot of 25 Sep 2025, byte for byte; rebuilt here, 172/377 classes identical, 203 identical by `javap`, 2 differ only in order (a method's place; which of three identical handlers each exception jumps to) | 17 | | sbml-exporter-lib 3.0.0-SNAPSHOT | `reactome/sbml-exporter` | `f7b1034` (branch `graph-only`) | not Java's relational answer, so no byte comparison: proven by `ci/compare.py` against today's files (spec 015 T037, T039) | 17 | +| biopax3-exporter-lib 0.0.1-SNAPSHOT | `reactome/BioPAX3Exporter` | `64c716a` (`main`) | not Java's relational answer, so no byte comparison: proven by its own `ci/compare.py` and `ci/contained.py` against release 98's published files (its spec 001 T020, T021), and here by spec 015 T049 against the legacy service | 17 | ## Frozen on purpose @@ -27,9 +28,9 @@ The Reactome Java this deployment builds (spec 010), each pinned to the commit t - **What uses it:** only diagram-exporter's PPTX path, which no Java-homed operation uses, and it runs unlicensed today (research R8). - **To use it again:** a licence, and an operation that needs it. -## The library service (spec 015 T063) +## The library service (spec 015 T063, T069) -`deploy/java-library/Dockerfile` builds the last three rows from these commits, and checks that its jar carries the libraries it built, byte for byte. `deploy/java-library/pins.spec.ts` checks that the Dockerfile's commits are this table's. Changing `sbml-exporter-lib`'s pin means rerunning spec 015 T067's comparison, not `tools/java-parity`: its body is not Java's. +`deploy/java-library/Dockerfile` builds the last four rows from these commits, and checks that its jar carries the libraries it built, byte for byte. `deploy/java-library/pins.spec.ts` checks that the Dockerfile's commits are this table's. Changing `sbml-exporter-lib`'s pin means rerunning spec 015 T067's comparison, not `tools/java-parity`: its body is not Java's. Changing `biopax3-exporter-lib`'s likewise means rerunning spec 015 T049's comparison and T072's check. ## Changing a pin diff --git a/docker-compose.yml b/docker-compose.yml index 2a557d7ba..a1b6ad1dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -317,10 +317,11 @@ services: # fails to build its lists, and rebuilds on the next request -- which is # the behaviour specs/006 D10 asks for -- rather than refusing to start. - # The library service (spec 015 T063, T064): the website's SBML, made on - # demand from the graph by sbml-exporter's library. Spec 010's Java library - # service, started here with SBML as its only operation. Node is its only - # client (T065); nginx never names it. + # The library service (spec 015 T063, T064, T069): the website's SBML and + # BioPAX Level 3, made on demand from the graph by sbml-exporter's and + # BioPAX3Exporter's libraries. Spec 010's Java library service, started here + # with those two operations. Node is its only client (T065, T070); nginx + # never names it. # # Built here, on this host and in CI, from the commits deploy/java/PINS.md # pins, and never pushed to a registry: no `image:` to pull or push @@ -404,7 +405,7 @@ volumes: # Survives a container replacement, which is the point: a cached figure is a # file read, and rebuilding the cache means paying for every render again. render-cache: - # The library service's SBML and diagram files, per release: kept across a + # The library service's SBML, BioPAX and diagram files, per release: kept across a # container replacement, as render's figures are. java-library-cache: graphdb-release98: diff --git a/specs/015-graph-exports/tasks.md b/specs/015-graph-exports/tasks.md index c1a14540e..cffa18f28 100644 --- a/specs/015-graph-exports/tasks.md +++ b/specs/015-graph-exports/tasks.md @@ -265,7 +265,10 @@ Both exporters' adoption pull requests found that their own checks do not run ye - [ ] T050 [US2] (The species files are done: the validator gives no error a published file lacks, its T020, [BioPAX3Exporter#79](https://github.com/reactome/BioPAX3Exporter/pull/79).) What is left: validate the sample's per-event files from the library, and the legacy service's, and record any error the library's has that the legacy's lacks. - [x] T051 [US2] [ext] In `BioPAX3Exporter`'s CI: the validator over its output for a small fixture graph, failing on any error in the reports. — its T004 ([BioPAX3Exporter#15](https://github.com/reactome/BioPAX3Exporter/pull/15)) and T007 ([BioPAX3Exporter#27](https://github.com/reactome/BioPAX3Exporter/pull/27)). - [ ] T052 [US2] Serve the website's BioPAX through the library service in `deploy/java-library/`, as R10's amendment of 29 Oct 2026 decided: T069–T072, in that order. T071 waits for T049 and T050. -- [ ] T069 [US2] The BioPAX operation in the library service, on T063's terms: +- [x] T069 [US2] The BioPAX operation in the library service, on T063's terms: — #755. `SbmlFiles` and `SbmlController` became `ExportFiles` and `ExportController`, one `ExportFiles` per export, with the conversions' cap of two shared by both. BioPAX3Exporter `main` at `64c716a` is pinned (PINS.md; `pins.spec.ts`, whose ARG pattern now allows a digit). The Dockerfile checks the jar's `kotlin-stdlib` 2.4.21 and `paxtools-core` 6.0.0. `check.sh` gains a BioPAX stage and a `SERVICE_PORT`, so it runs beside a deployment's service. + - **Checked here:** the image built and `check.sh` passed over both fixtures: SBML's three files valid and equal to expected; five BioPAX events, one of each kind across both species, each contained and whole in its species' expected file, with the per-event base. Against beta's image, which has no BioPAX, it failed: five 404s. + - **Measured** on a private copy of release 98's graph, in a container held to 1.5 GiB with T064's 1 GiB heap: BioPAX and SBML of Signal Transduction and Metabolism, all four asked at once. Peak 1.10 GiB, no kill. The cap converted two at once: Metabolism's pair in 156–158 s, then Signal Transduction's pair, finished at 300–310 s. Alone, a conversion of either is about 150 s, under node's 290 s. In that worst case the queued pair's readers get node's 503 ("took too long"); the service finishes and keeps the files, so a second request is answered at once. Files: BioPAX 47 MB and 41 MB, SBML 23 MB and 19 MB. + - **Unit tests:** `ExportFilesTest` (11) and `ExportControllerTest` (8). The new ones: an export with no diagram source never fetches one; each export keeps its own files; the cap is shared, at most two at once across both; BioPAX answered as RDF, needing no diagram, with SBML's errors. From #754's review, in the same PR: node's comment on the shares, and BioPAX's 503s tested. - **The library from source:** the Dockerfile builds `biopax3-exporter-lib` from a pinned commit of `BioPAX3Exporter`'s `main`, with its row in `deploy/java/PINS.md` and `pins.spec.ts` following. Its graph-core must be the build already pinned, or the PR says why not. The service's pom sets `kotlin.version` to the library's: Spring Boot 2.7's dependency management would otherwise bring Kotlin's standard library down to 1.6.21 under a library compiled with 2.4. The Dockerfile's jar check `cmp`s `biopax3-exporter-lib` as it does `sbml-exporter-lib`, and requires the jar's `kotlin-stdlib` and `paxtools-core` to be the library's versions. - **Its operation:** `GET /biopax/{stId}` answers the library's one-event conversion, with the release's per-event `xml:base`, and 404 for an id that is not a pathway or reaction in the graph, a backstop: node decides 404 and 400 first (T070). - **The files** are kept on disk per release, written beside and moved into place. One id is never converted twice at the same time. diff --git a/tools/content-node/biopax-export.spec.ts b/tools/content-node/biopax-export.spec.ts index c492ce357..eaa9a83b3 100644 --- a/tools/content-node/biopax-export.spec.ts +++ b/tools/content-node/biopax-export.spec.ts @@ -105,15 +105,15 @@ describe('the file', () => { res.writeHead(200, { 'Content-Type': 'application/rdf+xml;charset=UTF-8' }); res.end(''); }; - for (const form of ['R-HSA-1', 'R-HSA-1.4', '1']) { - const answer = await answers(`event/${form}.biopax`); + for (const path of ['R-HSA-1.biopax', 'R-HSA-1.4.biopax', '1.biopax', 'R-HSA-1.biopax/']) { + const answer = await answers(`event/${path}`); expect(answer.status).toBe(200); expect(answer.type).toBe('application/rdf+xml;charset=UTF-8'); expect(answer.headers?.['Content-Disposition']).toBe('attachment; filename="R-HSA-1.biopax"'); expect(await text(answer)).toBe(''); } // The service is asked for BioPAX, by the bare stable id - expect(asked).toEqual(['/biopax/R-HSA-1', '/biopax/R-HSA-1', '/biopax/R-HSA-1']); + expect(asked).toEqual(Array(4).fill('/biopax/R-HSA-1')); }); it('the service’s 404 is a 404, and its failure a 500 without its own reason', async () => { @@ -134,6 +134,27 @@ describe('the file', () => { ]); }); + it('the service’s 503 is a 503 to try again, and a service not there is a 503, in BioPAX’s words', async () => { + withService(); + behave = (_path, res) => { + res.writeHead(503, { 'Content-Type': 'application/json' }); + res.end('{"error":"/cache busy"}'); + }; + const busy = await answers('event/R-HSA-1.biopax'); + expect(envelope(busy)).toEqual([ + 503, + 'SERVICE_UNAVAILABLE', + ['The BioPAX file cannot be made just now; try again shortly'], + ]); + expect(busy.headers?.['Retry-After']).toBe('30'); + process.env['SBML_SERVICE'] = 'http://127.0.0.1:9'; + expect(envelope(await answers('event/R-HSA-1.biopax'))).toEqual([ + 503, + 'SERVICE_UNAVAILABLE', + ['The BioPAX service cannot be reached'], + ]); + }); + it('a service that sends no headers in time is a 503', async () => { withService(); behave = () => undefined; diff --git a/tools/content-node/exporters-endpoints.ts b/tools/content-node/exporters-endpoints.ts index 89f1e2f2f..1fa0afc9c 100644 --- a/tools/content-node/exporters-endpoints.ts +++ b/tools/content-node/exporters-endpoints.ts @@ -370,11 +370,12 @@ async function exportEvent(raw: string): Promise { } /** - * Each export's own share (research R6): Java made one file at a time, and the - * service two at once across its exports (its backstop, spec 015 T069); past - * a share, a 503 with Retry-After, as the deep pool answers. Separate from the - * deep pool and from each other, so a burst of one cannot starve the queries - * the pages need, or the other export. + * Each export's own share (research R6): Java made one file at a time; past a + * share, a 503 with Retry-After, as the deep pool answers. Separate from the + * deep pool, so a burst of exports cannot starve the queries the pages need, + * and from each other, so one export cannot take the other's places here. The + * service converts two at once across both (spec 015 T069), for its heap: a + * BioPAX file may wait there behind two SBML ones, within the headers' time. */ const SBML_SHARE = deepShare(2); const BIOPAX_SHARE = deepShare(2);