Spec 015 T069: BioPAX in the library service

deploy/java-library/Dockerfile

@@ -1,11 +1,11 @@
-# Spec 010's library service, started here with SBML as its only operation
-# (spec 015 T063): sbml-exporter's library behind one loopback endpoint, which
-# node calls.
+# Spec 010's library service, started here with SBML (spec 015 T063) and
+# BioPAX Level 3 (T069): sbml-exporter's and BioPAX3Exporter's libraries behind
+# loopback endpoints, which node calls.
 #
 # Every Reactome library is built from the commit deploy/java/PINS.md records,
 # never taken from a published snapshot: a snapshot can be republished under the
-# same version, and the shared parent POM already was once. When the SBML library
-# is published at a fixed version, its build stage goes and the pom names it.
+# same version, and the shared parent POM already was once. When a library is
+# published at a fixed version, its build goes and the pom names it.
 #
 # Built only on the beta host and in CI, and never pushed to a registry: the
 # libraries have no licence yet (spec 015 research R10).
@@ -16,6 +16,7 @@ FROM maven:3.9.6-eclipse-temurin-17 AS libraries
 ARG REACTOME_PARENT_COMMIT=d93ad5a417724647f04810b476f6c2705a0e3aec
 ARG GRAPH_CORE_COMMIT=62aef35d7a89403b8a8800dc004d8de62f847aee
 ARG SBML_EXPORTER_COMMIT=f7b10342258d1aacd4f77f44bd928a29b516025e
+ARG BIOPAX3_EXPORTER_COMMIT=64c716a7f37081b42c1cf7b30bdebbf55ff580e2
 
 WORKDIR /build
 
@@ -28,37 +29,47 @@ RUN set -eu; \
     }; \
     fetch reactome-parent "$REACTOME_PARENT_COMMIT"; \
     fetch graph-core "$GRAPH_CORE_COMMIT"; \
-    fetch sbml-exporter "$SBML_EXPORTER_COMMIT"
+    fetch sbml-exporter "$SBML_EXPORTER_COMMIT"; \
+    fetch BioPAX3Exporter "$BIOPAX3_EXPORTER_COMMIT"
 
 # -nsu: a snapshot installed here is never replaced by a repository's.
 RUN set -eu; \
     cd /build/reactome-parent/boms && mvn -B -q -nsu install -N; \
     cd /build/reactome-parent/boms/reactome-bom && mvn -B -q -nsu install; \
     cd /build/reactome-parent/parents/reactome-parent && mvn -B -q -nsu install; \
     cd /build/graph-core && mvn -B -q -nsu install -DskipTests; \
-    cd /build/sbml-exporter && mvn -B -q -nsu install -pl lib -am -DskipTests
+    cd /build/sbml-exporter && mvn -B -q -nsu install -pl lib -am -DskipTests; \
+    cd /build/BioPAX3Exporter && mvn -B -q -nsu install -pl lib -am -DskipTests
 
 FROM libraries AS service
 
 ARG REACTOME_PARENT_COMMIT=d93ad5a417724647f04810b476f6c2705a0e3aec
 ARG GRAPH_CORE_COMMIT=62aef35d7a89403b8a8800dc004d8de62f847aee
 ARG SBML_EXPORTER_COMMIT=f7b10342258d1aacd4f77f44bd928a29b516025e
+ARG BIOPAX3_EXPORTER_COMMIT=64c716a7f37081b42c1cf7b30bdebbf55ff580e2
 
 WORKDIR /service
 COPY deploy/java-library/pom.xml ./
 COPY deploy/java-library/src ./src
 # The commits the health endpoint reports.
-RUN printf 'reactome-parent=%s\ngraph-core=%s\nsbml-exporter=%s\n' \
-      "$REACTOME_PARENT_COMMIT" "$GRAPH_CORE_COMMIT" "$SBML_EXPORTER_COMMIT" \
+RUN printf 'reactome-parent=%s\ngraph-core=%s\nsbml-exporter=%s\nBioPAX3Exporter=%s\n' \
+      "$REACTOME_PARENT_COMMIT" "$GRAPH_CORE_COMMIT" "$SBML_EXPORTER_COMMIT" "$BIOPAX3_EXPORTER_COMMIT" \
       > src/main/resources/pins.properties
 RUN mvn -B -q -nsu package
 
 # The jar must carry the libraries built above, not a repository's copies of
-# their versions: compare their bytes.
+# their versions: compare their bytes. And the BioPAX library's Kotlin and
+# Paxtools at its own versions, alone: Spring Boot's dependency management
+# would give Kotlin's standard library as 1.6.21 (the pom holds it).
 RUN set -eu; \
     mkdir /check && cd /check && jar -xf /service/target/java-library.jar BOOT-INF/lib; \
     cmp BOOT-INF/lib/graph-core-3.0.0-SNAPSHOT.jar /build/graph-core/target/graph-core.jar; \
-    cmp BOOT-INF/lib/sbml-exporter-lib-3.0.0-SNAPSHOT.jar /build/sbml-exporter/lib/target/sbml-exporter-lib-3.0.0-SNAPSHOT.jar
+    cmp BOOT-INF/lib/sbml-exporter-lib-3.0.0-SNAPSHOT.jar /build/sbml-exporter/lib/target/sbml-exporter-lib-3.0.0-SNAPSHOT.jar; \
+    cmp BOOT-INF/lib/biopax3-exporter-lib-0.0.1-SNAPSHOT.jar /build/BioPAX3Exporter/lib/target/biopax3-exporter-lib-0.0.1-SNAPSHOT.jar; \
+    kotlin=$(sed -n 's|.*<kotlin.version>\(.*\)</kotlin.version>.*|\1|p' /build/BioPAX3Exporter/pom.xml); \
+    paxtools=$(sed -n 's|.*<paxtools.version>\(.*\)</paxtools.version>.*|\1|p' /build/BioPAX3Exporter/pom.xml); \
+    [ "$(ls BOOT-INF/lib | grep '^kotlin-stdlib-[0-9]')" = "kotlin-stdlib-$kotlin.jar" ]; \
+    [ "$(ls BOOT-INF/lib | grep '^paxtools-core-')" = "paxtools-core-$paxtools.jar" ]
 
 FROM eclipse-temurin:17-jre
 

deploy/java-library/check.sh

@@ -1,26 +1,33 @@
 #!/usr/bin/env bash
-# The library service, checked end to end over sbml-exporter's fixture graph
-# (spec 015 T063): the image this repository builds, its answers for the
-# fixture's three events compared with the expected files of the sbml-exporter
-# commit it is pinned to, by that commit's own ci/compare.py.
+# The library service, checked end to end (spec 015 T063, T069): the image this
+# repository builds, over each exporter's own fixture graph, its answers
+# compared with the expected files of the commit it is pinned to, by that
+# commit's own instruments:
+# - SBML over sbml-exporter's fixture: the three events' files, by its
+#   ci/compare.py;
+# - BioPAX over BioPAX3Exporter's fixture: events of each kind and both
+#   species, each contained in its species' expected file, by its
+#   ci/contained.py, with the release's per-event xml:base.
 #
 # Usage: deploy/java-library/check.sh [image]
-# GRAPH_PORT (7687) and FILES_PORT (8097) move the fixture graph and the files
-# off ports a host already uses; the service keeps its own, 4360.
+# GRAPH_PORT (7687), FILES_PORT (8097) and SERVICE_PORT (4360) move the fixture
+# graph, the files and the service off ports a host already uses (a deployment's
+# own service is on 4360).
 # Without an image, it builds one. Needs docker, curl and python3 with
 # python-libsbml 5.21.2 (pip install python-libsbml==5.21.2).
 set -euo pipefail
 
 graph_port=${GRAPH_PORT:-7687}
 files_port=${FILES_PORT:-8097}
+service_port=${SERVICE_PORT:-4360}
 
-# The check's own service takes 4360 on the host's network. One already there
+# The check's own service takes its port on the host's network. One already there
 # (a deployment's) would answer the checks below in its place, over its graph.
 # Any answer refuses, a 503 too (a deployment whose graph is down), and it is
 # refused before the trap below, whose clean-up would remove another run's
 # containers.
-if curl -s -m 5 -o /dev/null http://127.0.0.1:4360/health; then
-  echo "Something already answers on 127.0.0.1:4360; stop it, or run this elsewhere" >&2
+if curl -s -m 5 -o /dev/null http://127.0.0.1:$service_port/health; then
+  echo "Something already answers on 127.0.0.1:$service_port; stop it, or set SERVICE_PORT" >&2
   exit 1
 fi
 
@@ -30,9 +37,11 @@ work=$(mktemp -d)
 # Named for this run, so a run that stops early removes only its own.
 neo=java-library-check-graph-$$
 service=java-library-check-$$
+neo2=java-library-check-graph2-$$
+service2=java-library-check2-$$
 # shellcheck disable=SC2317 # called by the trap
 cleanup() {
-  docker rm -f -v "$neo" "$service" >/dev/null 2>&1 || true
+  docker rm -f -v "$neo" "$service" "$neo2" "$service2" >/dev/null 2>&1 || true
   if [ -n "${server:-}" ]; then kill "$server" 2>/dev/null || true; fi
   rm -rf "$work"
 }
@@ -51,37 +60,47 @@ curl -fsSL "https://codeload.github.com/reactome/sbml-exporter/tar.gz/$commit" \
   | tar -xz -C "$work/sbml-exporter" --strip-components=1 --wildcards '*/ci/*'
 ci="$work/sbml-exporter/ci"
 
-# The fixture graph, on loopback.
-docker run -d --name "$neo" -p "127.0.0.1:$graph_port:7687" \
-  -e NEO4J_AUTH=neo4j/fixturepass -e NEO4J_dbms_default__database=graph.db \
-  -v "$ci/fixture/fixture.cypher:/fixture.cypher:ro" neo4j:4.4.48-community >/dev/null
-for _ in $(seq 90); do
-  docker exec "$neo" cypher-shell -u neo4j -p fixturepass -d graph.db 'RETURN 1' >/dev/null 2>&1 && break
-  sleep 2
-done
-docker exec "$neo" cypher-shell -u neo4j -p fixturepass -d graph.db -f /fixture.cypher >/dev/null
+# A fixture graph, on loopback: start_graph <container> <fixture.cypher>
+start_graph() {
+  docker run -d --name "$1" -p "127.0.0.1:$graph_port:7687" \
+    -e NEO4J_AUTH=neo4j/fixturepass -e NEO4J_dbms_default__database=graph.db \
+    -v "$2:/fixture.cypher:ro" neo4j:4.4.48-community >/dev/null
+  for _ in $(seq 90); do
+    docker exec "$1" cypher-shell -u neo4j -p fixturepass -d graph.db 'RETURN 1' >/dev/null 2>&1 && break
+    sleep 2
+  done
+  docker exec "$1" cypher-shell -u neo4j -p fixturepass -d graph.db -f /fixture.cypher >/dev/null
+}
+
+# The service over it, answering its health check: start_service <container>
+start_service() {
+  docker run -d --name "$1" --network host \
+    -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \
+    -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache \
+    -e SERVER_PORT="$service_port" "$image" >/dev/null
+  for _ in $(seq 90); do
+    curl -fs http://127.0.0.1:$service_port/health >/dev/null && break
+    sleep 2
+  done
+  if ! curl -fsS http://127.0.0.1:$service_port/health; then
+    echo "The service did not answer its health check. Its log:" >&2
+    docker logs "$1" 2>&1 | tail -60 >&2
+    exit 1
+  fi
+  echo
+}
+
+start_graph "$neo" "$ci/fixture/fixture.cypher"
 
 # The release's download directory, as far as the fixture needs it.
 mkdir -p "$work/www/97/diagram"
 cp "$ci"/fixture/R-*.json "$work/www/97/diagram/"
 python3 -m http.server "$files_port" --bind 127.0.0.1 --directory "$work/www" >/dev/null 2>&1 &
 server=$!
 
-docker run -d --name "$service" --network host \
-  -e NEO4J_URI="bolt://127.0.0.1:$graph_port" -e NEO4J_USER=neo4j -e NEO4J_PASSWORD=fixturepass \
-  -e DIAGRAM_BASE="http://127.0.0.1:$files_port/" -e SBML_CACHE=/cache "$image" >/dev/null
-for _ in $(seq 90); do
-  curl -fs http://127.0.0.1:4360/health >/dev/null && break
-  sleep 2
-done
-if ! curl -fsS http://127.0.0.1:4360/health; then
-  echo "The service did not answer its health check. Its log:" >&2
-  docker logs "$service" 2>&1 | tail -60 >&2
-  exit 1
-fi
-echo
+start_service "$service"
 
-status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:4360/sbml/$1"; }
+status() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:$service_port/sbml/$1"; }
 fail=0
 mkdir "$work/out"
 for id in $(find "$ci/fixture/expected" -name 'R-*.sbml' -exec basename {} .sbml \; | sort); do
@@ -93,6 +112,57 @@ code=$(status R-HSA-1 /dev/null)
 code=$(status R-HSA-9613354.1 /dev/null)
 [ "$code" = 400 ] || { echo "R-HSA-9613354.1 (not a bare stable id): $code, expected 400" >&2; fail=1; }
 
-python3 "$ci/validate.py" "$work/out"
-python3 "$ci/compare.py" "$ci/fixture/expected" "$work/out"
+python3 "$ci/validate.py" "$work/out" || fail=1
+python3 "$ci/compare.py" "$ci/fixture/expected" "$work/out" || fail=1
+
+# BioPAX, over BioPAX3Exporter's fixture graph (release 98), in a graph and a
+# service of their own.
+docker rm -f -v "$neo" "$service" >/dev/null
+bcommit=$(sed -n 's/^ARG BIOPAX3_EXPORTER_COMMIT=\([0-9a-f]\{40\}\)$/\1/p' "$root/deploy/java-library/Dockerfile" | sort -u)
+[ "$(printf '%s\n' "$bcommit" | wc -l)" -eq 1 ] || { echo "The Dockerfile names more than one BioPAX3Exporter commit" >&2; exit 1; }
+mkdir "$work/BioPAX3Exporter"
+curl -fsSL "https://codeload.github.com/reactome/BioPAX3Exporter/tar.gz/$bcommit" \
+  | tar -xz -C "$work/BioPAX3Exporter" --strip-components=1 --wildcards '*/ci/*'
+bci="$work/BioPAX3Exporter/ci"
+start_graph "$neo2" "$bci/fixture/fixture.cypher"
+start_service "$service2"
+
+# Events of each kind and both species: of the graph's, by database id, the
+# first of each kind that its species' expected file holds (the fixture has
+# events beyond those files, such as an inferred event's source). Each line:
+# stId, dbId, species.
+candidates=$(docker exec "$neo2" cypher-shell -u neo4j -p fixturepass -d graph.db --format plain '
+  CALL {
+    MATCH (e:TopLevelPathway {speciesName: "Homo sapiens"}) RETURN "pathway" AS kind, e
+    UNION ALL MATCH (e:Reaction {speciesName: "Homo sapiens"}) RETURN "reaction" AS kind, e
+    UNION ALL MATCH (e:BlackBoxEvent {speciesName: "Homo sapiens"}) RETURN "blackbox" AS kind, e
+    UNION ALL MATCH (e:ReactionLikeEvent {dbId: 174731}) RETURN "degradation" AS kind, e
+    UNION ALL MATCH (e:Pathway {speciesName: "Plasmodium falciparum"}) RETURN "falciparum" AS kind, e
+  }
+  RETURN kind + " " + e.stId + " " + e.dbId + " " + replace(e.speciesName, " ", "_")
+  ORDER BY kind, e.dbId' | tail -n +2 | tr -d '"')
+events=$(while read -r kind id dbid species; do
+  grep -q ">$dbid</bp:id>" "$bci/fixture/expected/$species.owl" && echo "$kind $id $dbid $species"
+done <<<"$candidates" | sort -u -k1,1 | cut -d' ' -f2-)
+[ "$(printf '%s\n' "$events" | wc -l)" -eq 5 ] || { echo "Expected one event of each of five kinds in the fixture's files, got: $events" >&2; exit 1; }
+
+bstatus() { curl -s -o "$2" -w '%{http_code}' "http://127.0.0.1:$service_port/biopax/$1"; }
+mkdir -p "$work/owl/Homo_sapiens" "$work/owl/Plasmodium_falciparum"
+while read -r id dbid species; do
+  file="$work/owl/$species/$id.owl"
+  code=$(bstatus "$id" "$file")
+  [ "$code" = 200 ] || { echo "$id: $code, expected 200" >&2; fail=1; continue; }
+  # The release's per-event base: Reactome's BioPAX namespace, the release, the event's database id
+  grep -q "xml:base=\"http://www.reactome.org/biopax/98/$dbid#\"" "$file" \
+    || { echo "$id: not the release's per-event xml:base" >&2; fail=1; }
+done <<<"$events"
+code=$(bstatus R-HSA-1 /dev/null)
+[ "$code" = 404 ] || { echo "R-HSA-1 (not in the graph): $code, expected 404" >&2; fail=1; }
+code=$(bstatus R-HSA-174731.1 /dev/null)
+[ "$code" = 400 ] || { echo "R-HSA-174731.1 (not a bare stable id): $code, expected 400" >&2; fail=1; }
+
+# Each event's file holds nothing its species' file lacks, and is whole
+for species in Homo_sapiens Plasmodium_falciparum; do
+  python3 -I "$bci/contained.py" "$bci/fixture/expected/$species.owl" "$work/owl/$species"/*.owl || fail=1
+done
 exit $fail

deploy/java-library/pins.spec.ts

@@ -15,7 +15,7 @@ const pins = readFileSync(join(__dirname, '../java/PINS.md'), 'utf8');
 /** The Dockerfile's ARG <NAME>_COMMIT=<sha> lines, by name, each stage's. */
 function commitArgs(text: string) {
   const found = new Map<string, Set<string>>();
-  for (const [, name = '', sha = ''] of text.matchAll(/^ARG ([A-Z_]+)_COMMIT=([0-9a-f]{40})$/gm))
+  for (const [, name = '', sha = ''] of text.matchAll(/^ARG ([A-Z0-9_]+)_COMMIT=([0-9a-f]{40})$/gm))
     found.set(name, (found.get(name) ?? new Set()).add(sha));
   return found;
 }
@@ -30,6 +30,7 @@ const REPOSITORIES: Record<string, string> = {
   REACTOME_PARENT: 'reactome-parent',
   GRAPH_CORE: 'graph-core',
   SBML_EXPORTER: 'sbml-exporter',
+  BIOPAX3_EXPORTER: 'BioPAX3Exporter',
 };
 
 describe('the library service pins', () => {

deploy/java-library/pom.xml

@@ -3,7 +3,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <!-- Spec 010's library service, started here with SBML as its only operation (spec 015 T063). -->
+    <!-- Spec 010's library service, started here with SBML (spec 015 T063) and BioPAX Level 3 (T069). -->
     <groupId>org.reactome.server</groupId>
     <artifactId>java-library</artifactId>
     <version>0.1.0</version>
@@ -26,6 +26,12 @@
         <!-- sbml-exporter's graph-only branch, built from its commit by the Dockerfile. Once the library is
              published, the Dockerfile's build stage goes and this names the published version. -->
         <sbml-exporter-lib.version>3.0.0-SNAPSHOT</sbml-exporter-lib.version>
+        <!-- BioPAX3Exporter's main, built from its commit by the Dockerfile, likewise. -->
+        <biopax3-exporter-lib.version>0.0.1-SNAPSHOT</biopax3-exporter-lib.version>
+        <!-- The BioPAX library's Kotlin (its pom's kotlin.version). Spring Boot 2.7's dependency management, which
+             the parent brings, would otherwise give its standard library as 1.6.21 under a library compiled with
+             2.4; the Dockerfile checks the jar carries this one. -->
+        <kotlin.version>2.4.21</kotlin.version>
     </properties>
 
     <dependencies>
@@ -34,6 +40,11 @@
             <artifactId>sbml-exporter-lib</artifactId>
             <version>${sbml-exporter-lib.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.reactome.server.tools</groupId>
+            <artifactId>biopax3-exporter-lib</artifactId>
+            <version>${biopax3-exporter-lib.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>

deploy/java-library/src/main/java/org/reactome/server/library/SbmlController.java → deploy/java-library/src/main/java/org/reactome/server/library/ExportController.java

@@ -9,6 +9,7 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.Resource;
 import org.springframework.http.HttpStatus;
@@ -20,35 +21,52 @@
 
 /**
  * The service's answers to node, over loopback. Node writes the public answer: its headers, its errors in the
- * service's envelope and its cache labels (spec 015 T065). This writes only the body, or a status and a reason.
+ * service's envelope and its cache labels (spec 015 T065, T070). This writes only the body, or a status and a reason.
  */
 @RestController
-public class SbmlController {
+public class ExportController {
 
-    private static final Logger logger = LoggerFactory.getLogger(SbmlController.class);
+    private static final Logger logger = LoggerFactory.getLogger(ExportController.class);
     private static final Pattern ST_ID = Pattern.compile("R-[A-Z]{3}-\\d+");
     static final MediaType SBML = MediaType.parseMediaType("application/sbml+xml;charset=UTF-8");
+    static final MediaType BIOPAX = MediaType.parseMediaType("application/rdf+xml;charset=UTF-8");
 
-    private final SbmlFiles files;
-    private final SbmlFiles.Graph graph;
+    private final ExportFiles sbml;
+    private final ExportFiles biopax;
+    private final ExportFiles.Graph graph;
     private final Properties pins;
 
-    public SbmlController(SbmlFiles files, SbmlFiles.Graph graph, Properties pins) {
-        this.files = files;
+    public ExportController(@Qualifier("sbmlFiles") ExportFiles sbml, @Qualifier("biopaxFiles") ExportFiles biopax,
+                            ExportFiles.Graph graph, Properties pins) {
+        this.sbml = sbml;
+        this.biopax = biopax;
         this.graph = graph;
         this.pins = pins;
     }
 
     /** An event's SBML, by its stable id without a version: node resolves any other form first. */
     @GetMapping("/sbml/{stId}")
     public ResponseEntity<?> sbml(@PathVariable String stId) throws InterruptedException {
+        return answer(sbml, SBML, stId);
+    }
+
+    /**
+     * An event's BioPAX Level 3, by its stable id without a version. Node decides 404 and 400 first (T070); the
+     * service's own 404 is a backstop.
+     */
+    @GetMapping("/biopax/{stId}")
+    public ResponseEntity<?> biopax(@PathVariable String stId) throws InterruptedException {
+        return answer(biopax, BIOPAX, stId);
+    }
+
+    private static ResponseEntity<?> answer(ExportFiles files, MediaType type, String stId) throws InterruptedException {
         if (!ST_ID.matcher(stId).matches()) return reason(HttpStatus.BAD_REQUEST, "Not a stable id: " + stId);
         try {
             Optional<Path> file = files.get(stId);
             if (file.isEmpty()) return reason(HttpStatus.NOT_FOUND, "No event " + stId + " in the graph");
             Resource body = new FileSystemResource(file.get());
-            return ResponseEntity.ok().contentType(SBML).body(body);
-        } catch (SbmlFiles.DiagramUnavailable e) {
+            return ResponseEntity.ok().contentType(type).body(body);
+        } catch (ExportFiles.DiagramUnavailable e) {
             logger.warn("{}: {}", stId, e.getMessage());
             return reason(HttpStatus.SERVICE_UNAVAILABLE, e.getMessage());
         } catch (RuntimeException e) {