perf: post codec and JDBC benchmark results on pull requests

.github/workflows/perf.yml

@@ -0,0 +1,113 @@
+name: Performance
+
+# Posts a sticky comment with driver-side benchmark numbers. The base commit and the PR merge
+# commit are measured on the same runner, three interleaved rounds each, so the comparison survives
+# shared-runner noise. The job is informational: regressions are flagged, not failed.
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: perf-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    name: Codec & JDBC benchmark
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Check out base
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          persist-credentials: false
+
+      # The default pull_request ref is the merge commit, which isolates the PR changes when the
+      # base is the base branch tip. It also works for pull requests from forks.
+      - name: Check out PR merge commit
+        uses: actions/checkout@v7
+        with:
+          path: head
+          persist-credentials: false
+
+      - name: Set up Amazon Corretto 25
+        uses: actions/setup-java@v6
+        with:
+          distribution: 'corretto'
+          java-version: '25'
+          cache: 'maven'
+
+      - name: Build benchmarkable modules
+        run: |
+          set -euo pipefail
+          if [ -d base/libsql-perf ]; then
+            (cd base && ./mvnw --batch-mode --quiet -Dmaven.test.skip=true -pl libsql-perf -am package)
+            echo "BASE_BENCHMARK=1" >> "$GITHUB_ENV"
+          else
+            echo "Base commit has no libsql-perf module yet; this run reports head numbers only."
+          fi
+          (cd head && ./mvnw --batch-mode --quiet -Dmaven.test.skip=true -pl libsql-perf -am package)
+
+      - name: Run benchmark (three interleaved rounds)
+        run: |
+          set -euo pipefail
+          mkdir -p results
+          run() {
+            local dir=$1 out=$2 sha=$3
+            java -Xms1g -Xmx1g \
+              -cp "$dir/libsql-perf/target/classes:$dir/libsql-perf/target/lib/*" \
+              io.github.magnusp.libsql.perf.PerfBenchmark \
+              --quiet --json "$out" --commit "$sha"
+          }
+          for i in 1 2 3; do
+            if [ "${BASE_BENCHMARK:-0}" = "1" ]; then
+              run base "results/base-$i.json" "${{ github.event.pull_request.base.sha }}"
+            fi
+            run head "results/head-$i.json" "${{ github.sha }}"
+          done
+
+      - name: Render comment
+        run: |
+          set -euo pipefail
+          if [ "${BASE_BENCHMARK:-0}" = "1" ]; then
+            python3 head/tools/perf/render-comment.py \
+              --base results/base-1.json results/base-2.json results/base-3.json \
+              --head results/head-1.json results/head-2.json results/head-3.json \
+              --base-sha "${{ github.event.pull_request.base.sha }}" \
+              --head-sha "${{ github.sha }}" \
+              --out comment.md
+          else
+            python3 head/tools/perf/render-comment.py \
+              --head results/head-1.json results/head-2.json results/head-3.json \
+              --base-sha "${{ github.event.pull_request.base.sha }}" \
+              --head-sha "${{ github.sha }}" \
+              --out comment.md
+          fi
+          cat comment.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Post or update PR comment
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          head/tools/perf/comment.sh \
+            "${{ github.repository }}" \
+            "${{ github.event.pull_request.number }}" \
+            comment.md
+
+      - name: Upload benchmark results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: perf-results
+          path: results/
+          if-no-files-found: warn

AGENTS.md

@@ -34,6 +34,7 @@ libsql-java/
 ├── libsql-jdbc/                # Standard JDBC 4.3 driver (java.sql.* implementation)
 ├── libsql-itests/              # Testcontainers (sqld) and Toxiproxy integration tests
 ├── libsql-conformance/         # Selected xerial/sqlite-jdbc tests running against libSQL
+├── libsql-perf/                # Driver-side codec/JDBC benchmark posted on pull requests
 ├── tools/                      # Maintainer scripts such as export-upstream
 ├── upstream-xerial_sqlite-jdbc/  # Vendored xerial/sqlite-jdbc sources for conformance tests
 └── vendor/                     # Upstream specifications and documentation (Hrana v3, sqld)
@@ -99,6 +100,23 @@ mise exec -- ./mvnw -pl libsql-itests -Dtest=HikariCpIntegrationIT test
 
 Under nono, set `DOCKER_CONFIG` before the command, as described in the workaround below.
 
+### Performance Benchmark
+
+`libsql-perf` measures codec decode/encode and the JDBC result-set getters without a server. The
+[`Performance`](.github/workflows/perf.yml) workflow builds the PR merge commit and its base
+commit, runs three interleaved rounds, and posts a sticky comparison comment. The job flags
+regressions over 10% but does not fail the check.
+
+```bash
+mise exec -- ./mvnw -pl libsql-perf -am -DskipTests package
+mise exec -- java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \
+  io.github.magnusp.libsql.perf.PerfBenchmark --quick
+```
+
+See [`libsql-perf/README.md`](libsql-perf/README.md) for the workloads and how to change them. The
+benchmark payload is a measured baseline: change it in a dedicated PR and call it out, so the first
+comment after the change is not read as a regression.
+
 ### nono Sandbox and Docker Config
 
 This environment may run inside a `nono` filesystem sandbox. The sandbox can block reads of `~/.docker/config.json` even when `docker info` works. Testcontainers then fails with:

CONTRIBUTING.md

@@ -59,6 +59,7 @@ The repository is organized as a multi-module Maven project:
 - **`libsql-jdbc`**: Standard JDBC 4.3 compliant driver implementation (`jdbc:libsql:...`).
 - **`libsql-itests`**: End-to-end integration tests using Testcontainers and sqld.
 - **`libsql-conformance`**: Selected xerial/sqlite-jdbc tests running against libSQL.
+- **`libsql-perf`**: Driver-side codec and JDBC benchmark used for the PR performance comment.
 
 ## Code Style & Quality
 

README.md

@@ -25,6 +25,7 @@ A pure-Java JDBC driver, client library, and protocol codec for **libSQL** and *
 | [`libsql-jdbc`](libsql-jdbc) | Standard JDBC 4.3 driver (`jdbc:libsql:...`) |
 | [`libsql-itests`](libsql-itests) | Integration tests against `sqld` running in Testcontainers |
 | [`libsql-conformance`](libsql-conformance/README.md) | Selected xerial/sqlite-jdbc tests running against libSQL |
+| [`libsql-perf`](libsql-perf) | Driver-side codec and JDBC benchmark used for the PR performance comment |
 
 ## Getting Started
 

libsql-perf/README.md

@@ -0,0 +1,69 @@
+# libsql-perf
+
+Driver-side performance benchmark for the Hrana JSON codec and the JDBC result-set getters.
+
+The [`Performance`](../.github/workflows/perf.yml) workflow runs it for every pull request, compares
+the PR merge commit with the base commit, and posts the result as a sticky comment. The job is
+informational: regressions are flagged in the comment, not failed in the check.
+
+## What it measures
+
+| Workload | What it exercises |
+|---|---|
+| `codec.direct.decode5k` | `DirectHranaJsonCodec` decoding a 5,000-row, six-column `execute` result |
+| `codec.jackson.decode5k` | The optional Jackson codec on the same payload, as a baseline |
+| `codec.direct.decode20k` | The direct codec on a 20,000-row result |
+| `codec.direct.encode` | The direct codec serializing one parameterized execute |
+| `codec.jackson.encode` | The Jackson codec on the same request, as a baseline |
+| `jdbc.scan.all` | `LibsqlResultSet.next()` plus typed getters over 20,000 rows |
+| `jdbc.scan.object` | `ResultSet.getObject()` over all six columns of 20,000 rows |
+
+Every workload reports operations per second, rows per second (where applicable), and allocated
+bytes per operation from `java.lang.management.ThreadMXBean`. The benchmark takes the median of
+several measurement windows and prints Markdown, or writes JSON for the PR comment.
+
+The workload is deliberately driver-side. Starting `sqld` would add server and network time that
+dominates the codec and result-set costs the benchmark is meant to track. The integration tests in
+`libsql-itests` cover end-to-end throughput.
+
+## Running it locally
+
+```bash
+./mvnw -pl libsql-perf -am -DskipTests package
+java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \
+  io.github.magnusp.libsql.perf.PerfBenchmark
+```
+
+Useful flags:
+
+| Flag | Effect |
+|---|---|
+| `--quick` | Shorter warmup and fewer, shorter windows for a rough check |
+| `--json <path>` | Write machine-readable results |
+| `--commit <sha>` | Set the commit recorded in the JSON output |
+| `--quiet` | Do not print the Markdown summary |
+
+To see the same comparison the workflow posts, run the base and head revisions and render them:
+
+```bash
+./tools/perf/render-comment.py \
+  --base base.json --head head.json \
+  --base-sha <base-sha> --head-sha <head-sha>
+```
+
+## Reading the comparison
+
+The workflow checks out the base commit and the PR merge commit, builds both, then runs three
+interleaved rounds per side on the same runner. The comment reports medians across those rounds.
+Throughput drops and allocation increases larger than 10% are marked with a warning; shared-runner
+noise below that band is not meaningful.
+
+The comparison is against the PR base, not a historical record. To follow a metric across releases,
+compare the comments or the `perf-results` artifact attached to each run.
+
+## Changing the benchmark
+
+Adding a workload means adding a `measure(...)` call in `PerfBenchmark` and a matching entry in
+`WORKLOADS` in [`tools/perf/render-comment.py`](../tools/perf/render-comment.py). Changing the
+payload, warmup, or window size changes the baseline; call that out in the PR description so the
+first comment after the change is not read as a regression.

libsql-perf/pom.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.github.magnusp.libsql</groupId>
+        <artifactId>libsql-java-parent</artifactId>
+        <version>${revision}${sha1}${changelist}</version>
+    </parent>
+
+    <artifactId>libsql-perf</artifactId>
+    <name>libSQL Java Performance Benchmark</name>
+    <description>Driver-side codec and JDBC benchmark used by the PR performance comment</description>
+
+    <properties>
+        <!-- Measurement tool, never a published artifact. -->
+        <maven.deploy.skip>true</maven.deploy.skip>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.github.magnusp.libsql</groupId>
+            <artifactId>libsql-hrana-codec</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.github.magnusp.libsql</groupId>
+            <artifactId>libsql-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.github.magnusp.libsql</groupId>
+            <artifactId>libsql-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>3.9.0</version>
+                <executions>
+                    <execution>
+                        <id>copy-runtime-dependencies</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                            <includeScope>runtime</includeScope>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java

@@ -0,0 +1,71 @@
+package io.github.magnusp.libsql.perf;
+
+import io.github.magnusp.libsql.hrana.codec.PipelineReqBody;
+import io.github.magnusp.libsql.hrana.codec.Stmt;
+import io.github.magnusp.libsql.hrana.codec.StreamRequest;
+import io.github.magnusp.libsql.hrana.codec.Value;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+/**
+ * Builds the fixed payloads the benchmark measures.
+ *
+ * <p>The tracked payload is deliberately plain ASCII without JSON escapes so that changes in the
+ * numbers come from the codec, not from a shifting test document. Codec correctness for escapes and
+ * non-ASCII text is covered by the unit and property tests.
+ */
+final class Payloads {
+
+    private static final String NOTE =
+            "lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor";
+    private static final String BLOB = Base64.getEncoder().encodeToString(new byte[16]);
+    private static final String[] NAMES = {"id", "name", "score", "payload", "note", "flag"};
+    private static final String[] DECL_TYPES = {"INTEGER", "TEXT", "REAL", "BLOB", "TEXT", "INTEGER"};
+
+    /** A Hrana {@code execute} response with six columns and the requested number of rows. */
+    static byte[] scan(int rows) {
+        StringBuilder sb = new StringBuilder(rows * 300 + 512);
+        sb.append("{\"baton\":\"baton-token\",\"base_url\":null,\"results\":[{\"type\":\"ok\",")
+                .append("\"response\":{\"type\":\"execute\",\"result\":{\"cols\":[");
+        for (int c = 0; c < NAMES.length; c++) {
+            if (c > 0) {
+                sb.append(',');
+            }
+            sb.append("{\"name\":\"").append(NAMES[c]).append("\",\"decltype\":\"").append(DECL_TYPES[c]).append("\"}");
+        }
+        sb.append("],\"rows\":[");
+        for (int r = 0; r < rows; r++) {
+            if (r > 0) {
+                sb.append(',');
+            }
+            sb.append("[{\"type\":\"integer\",\"value\":\"").append(r).append("\"},")
+                    .append("{\"type\":\"text\",\"value\":\"user-").append(r).append("-name\"},")
+                    .append("{\"type\":\"float\",\"value\":").append(r).append(".25},")
+                    .append("{\"type\":\"blob\",\"base64\":\"").append(BLOB).append("\"},")
+                    .append("{\"type\":\"text\",\"value\":\"").append(NOTE).append("\"},")
+                    .append("{\"type\":\"").append(r % 2 == 0 ? "integer\",\"value\":\"1" : "null").append("\"}]");
+        }
+        sb.append("],\"affected_row_count\":0,\"last_insert_rowid\":null}}}]}");
+        return sb.toString().getBytes(StandardCharsets.UTF_8);
+    }
+
+    /** A pipeline with one parameterized insert, the shape every prepared execute sends. */
+    static PipelineReqBody insertRequest() {
+        List<Value> args = new ArrayList<>();
+        for (int i = 0; i < 6; i++) {
+            args.add(switch (i % 3) {
+                case 0 -> Value.of(123456789L);
+                case 1 -> Value.of("some text value, with a comma and a \"quote\"");
+                default -> Value.of(12.5);
+            });
+        }
+        Stmt stmt = new Stmt(
+                "INSERT INTO users (a, b, c, d, e, f) VALUES (?, ?, ?, ?, ?, ?)", null, args, List.of(), false);
+        return new PipelineReqBody("baton-token", List.of(new StreamRequest.Execute(stmt)));
+    }
+
+    private Payloads() {}
+}