From f5a784aacf3c108a5d1bb4e4a35204e80fcce17d Mon Sep 17 00:00:00 2001 From: Magnus Persson <1431685+magnusp@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:55:24 +0100 Subject: [PATCH 1/2] perf: benchmark codec and JDBC paths on pull requests Add a driver-side benchmark module and a workflow that posts its result as a sticky comment on every pull request. The benchmark measures the direct and Jackson codec decode/encode paths and the JDBC result-set getters on fixed payloads. It starts no server, so server and network time cannot bury codec changes in shared-runner noise. The workflow measures the base commit and the PR merge commit, three interleaved rounds each, and reports medians. Throughput drops and allocation increases over 10% are flagged; the check stays green so reviewers judge the numbers. The base side is skipped until the module exists on the base branch, so this PR's first comment reports absolute head numbers. Refs #291. --- .github/workflows/perf.yml | 113 +++++++ AGENTS.md | 18 ++ CONTRIBUTING.md | 1 + README.md | 1 + libsql-perf/README.md | 69 ++++ libsql-perf/pom.xml | 62 ++++ .../github/magnusp/libsql/perf/Payloads.java | 71 +++++ .../magnusp/libsql/perf/PerfBenchmark.java | 301 ++++++++++++++++++ pom.xml | 1 + tools/perf/comment.sh | 35 ++ tools/perf/render-comment.py | 183 +++++++++++ 11 files changed, 855 insertions(+) create mode 100644 .github/workflows/perf.yml create mode 100644 libsql-perf/README.md create mode 100644 libsql-perf/pom.xml create mode 100644 libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java create mode 100644 libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java create mode 100755 tools/perf/comment.sh create mode 100755 tools/perf/render-comment.py diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000..85fa731 --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,113 @@ +name: Performance + +# Posts a sticky comment with driver-side benchmark numbers. The base commit and the PR merge +# commit are measured on the same runner, three interleaved rounds each, so the comparison survives +# shared-runner noise. The job is informational: regressions are flagged, not failed. + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: perf-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark: + name: Codec & JDBC benchmark + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: read + pull-requests: write + steps: + - name: Check out base + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + persist-credentials: false + + # The default pull_request ref is the merge commit, which isolates the PR changes when the + # base is the base branch tip. It also works for pull requests from forks. + - name: Check out PR merge commit + uses: actions/checkout@v7 + with: + path: head + persist-credentials: false + + - name: Set up Amazon Corretto 25 + uses: actions/setup-java@v6 + with: + distribution: 'corretto' + java-version: '25' + cache: 'maven' + + - name: Build benchmarkable modules + run: | + set -euo pipefail + if [ -d base/libsql-perf ]; then + (cd base && ./mvnw --batch-mode --quiet -Dmaven.test.skip=true -pl libsql-perf -am package) + echo "BASE_BENCHMARK=1" >> "$GITHUB_ENV" + else + echo "Base commit has no libsql-perf module yet; this run reports head numbers only." + fi + (cd head && ./mvnw --batch-mode --quiet -Dmaven.test.skip=true -pl libsql-perf -am package) + + - name: Run benchmark (three interleaved rounds) + run: | + set -euo pipefail + mkdir -p results + run() { + local dir=$1 out=$2 sha=$3 + java -Xms1g -Xmx1g \ + -cp "$dir/libsql-perf/target/classes:$dir/libsql-perf/target/lib/*" \ + io.github.magnusp.libsql.perf.PerfBenchmark \ + --quiet --json "$out" --commit "$sha" + } + for i in 1 2 3; do + if [ "${BASE_BENCHMARK:-0}" = "1" ]; then + run base "results/base-$i.json" "${{ github.event.pull_request.base.sha }}" + fi + run head "results/head-$i.json" "${{ github.sha }}" + done + + - name: Render comment + run: | + set -euo pipefail + if [ "${BASE_BENCHMARK:-0}" = "1" ]; then + python3 head/tools/perf/render-comment.py \ + --base results/base-1.json results/base-2.json results/base-3.json \ + --head results/head-1.json results/head-2.json results/head-3.json \ + --base-sha "${{ github.event.pull_request.base.sha }}" \ + --head-sha "${{ github.sha }}" \ + --out comment.md + else + python3 head/tools/perf/render-comment.py \ + --head results/head-1.json results/head-2.json results/head-3.json \ + --base-sha "${{ github.event.pull_request.base.sha }}" \ + --head-sha "${{ github.sha }}" \ + --out comment.md + fi + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Post or update PR comment + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + head/tools/perf/comment.sh \ + "${{ github.repository }}" \ + "${{ github.event.pull_request.number }}" \ + comment.md + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: perf-results + path: results/ + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 07b3f86..d726f90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,6 +34,7 @@ libsql-java/ ├── libsql-jdbc/ # Standard JDBC 4.3 driver (java.sql.* implementation) ├── libsql-itests/ # Testcontainers (sqld) and Toxiproxy integration tests ├── libsql-conformance/ # Selected xerial/sqlite-jdbc tests running against libSQL +├── libsql-perf/ # Driver-side codec/JDBC benchmark posted on pull requests ├── tools/ # Maintainer scripts such as export-upstream ├── upstream-xerial_sqlite-jdbc/ # Vendored xerial/sqlite-jdbc sources for conformance tests └── vendor/ # Upstream specifications and documentation (Hrana v3, sqld) @@ -99,6 +100,23 @@ mise exec -- ./mvnw -pl libsql-itests -Dtest=HikariCpIntegrationIT test Under nono, set `DOCKER_CONFIG` before the command, as described in the workaround below. +### Performance Benchmark + +`libsql-perf` measures codec decode/encode and the JDBC result-set getters without a server. The +[`Performance`](.github/workflows/perf.yml) workflow builds the PR merge commit and its base +commit, runs three interleaved rounds, and posts a sticky comparison comment. The job flags +regressions over 10% but does not fail the check. + +```bash +mise exec -- ./mvnw -pl libsql-perf -am -DskipTests package +mise exec -- java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \ + io.github.magnusp.libsql.perf.PerfBenchmark --quick +``` + +See [`libsql-perf/README.md`](libsql-perf/README.md) for the workloads and how to change them. The +benchmark payload is a measured baseline: change it in a dedicated PR and call it out, so the first +comment after the change is not read as a regression. + ### nono Sandbox and Docker Config This environment may run inside a `nono` filesystem sandbox. The sandbox can block reads of `~/.docker/config.json` even when `docker info` works. Testcontainers then fails with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43a32b5..ca87d84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,6 +59,7 @@ The repository is organized as a multi-module Maven project: - **`libsql-jdbc`**: Standard JDBC 4.3 compliant driver implementation (`jdbc:libsql:...`). - **`libsql-itests`**: End-to-end integration tests using Testcontainers and sqld. - **`libsql-conformance`**: Selected xerial/sqlite-jdbc tests running against libSQL. +- **`libsql-perf`**: Driver-side codec and JDBC benchmark used for the PR performance comment. ## Code Style & Quality diff --git a/README.md b/README.md index f07ead3..9aadbc9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ A pure-Java JDBC driver, client library, and protocol codec for **libSQL** and * | [`libsql-jdbc`](libsql-jdbc) | Standard JDBC 4.3 driver (`jdbc:libsql:...`) | | [`libsql-itests`](libsql-itests) | Integration tests against `sqld` running in Testcontainers | | [`libsql-conformance`](libsql-conformance/README.md) | Selected xerial/sqlite-jdbc tests running against libSQL | +| [`libsql-perf`](libsql-perf) | Driver-side codec and JDBC benchmark used for the PR performance comment | ## Getting Started diff --git a/libsql-perf/README.md b/libsql-perf/README.md new file mode 100644 index 0000000..875e58f --- /dev/null +++ b/libsql-perf/README.md @@ -0,0 +1,69 @@ +# libsql-perf + +Driver-side performance benchmark for the Hrana JSON codec and the JDBC result-set getters. + +The [`Performance`](../.github/workflows/perf.yml) workflow runs it for every pull request, compares +the PR merge commit with the base commit, and posts the result as a sticky comment. The job is +informational: regressions are flagged in the comment, not failed in the check. + +## What it measures + +| Workload | What it exercises | +|---|---| +| `codec.direct.decode5k` | `DirectHranaJsonCodec` decoding a 5,000-row, six-column `execute` result | +| `codec.jackson.decode5k` | The optional Jackson codec on the same payload, as a baseline | +| `codec.direct.decode20k` | The direct codec on a 20,000-row result | +| `codec.direct.encode` | The direct codec serializing one parameterized execute | +| `codec.jackson.encode` | The Jackson codec on the same request, as a baseline | +| `jdbc.scan.all` | `LibsqlResultSet.next()` plus typed getters over 20,000 rows | +| `jdbc.scan.object` | `ResultSet.getObject()` over all six columns of 20,000 rows | + +Every workload reports operations per second, rows per second (where applicable), and allocated +bytes per operation from `java.lang.management.ThreadMXBean`. The benchmark takes the median of +several measurement windows and prints Markdown, or writes JSON for the PR comment. + +The workload is deliberately driver-side. Starting `sqld` would add server and network time that +dominates the codec and result-set costs the benchmark is meant to track. The integration tests in +`libsql-itests` cover end-to-end throughput. + +## Running it locally + +```bash +./mvnw -pl libsql-perf -am -DskipTests package +java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \ + io.github.magnusp.libsql.perf.PerfBenchmark +``` + +Useful flags: + +| Flag | Effect | +|---|---| +| `--quick` | Shorter warmup and fewer, shorter windows for a rough check | +| `--json ` | Write machine-readable results | +| `--commit ` | Set the commit recorded in the JSON output | +| `--quiet` | Do not print the Markdown summary | + +To see the same comparison the workflow posts, run the base and head revisions and render them: + +```bash +./tools/perf/render-comment.py \ + --base base.json --head head.json \ + --base-sha --head-sha +``` + +## Reading the comparison + +The workflow checks out the base commit and the PR merge commit, builds both, then runs three +interleaved rounds per side on the same runner. The comment reports medians across those rounds. +Throughput drops and allocation increases larger than 10% are marked with a warning; shared-runner +noise below that band is not meaningful. + +The comparison is against the PR base, not a historical record. To follow a metric across releases, +compare the comments or the `perf-results` artifact attached to each run. + +## Changing the benchmark + +Adding a workload means adding a `measure(...)` call in `PerfBenchmark` and a matching entry in +`WORKLOADS` in [`tools/perf/render-comment.py`](../tools/perf/render-comment.py). Changing the +payload, warmup, or window size changes the baseline; call that out in the PR description so the +first comment after the change is not read as a regression. diff --git a/libsql-perf/pom.xml b/libsql-perf/pom.xml new file mode 100644 index 0000000..427f1d5 --- /dev/null +++ b/libsql-perf/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + io.github.magnusp.libsql + libsql-java-parent + ${revision}${sha1}${changelist} + + + libsql-perf + libSQL Java Performance Benchmark + Driver-side codec and JDBC benchmark used by the PR performance comment + + + + true + + + + + io.github.magnusp.libsql + libsql-hrana-codec + + + io.github.magnusp.libsql + libsql-client + + + io.github.magnusp.libsql + libsql-jdbc + + + com.fasterxml.jackson.core + jackson-core + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.9.0 + + + copy-runtime-dependencies + package + + copy-dependencies + + + ${project.build.directory}/lib + runtime + + + + + + + diff --git a/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java new file mode 100644 index 0000000..ef1a39c --- /dev/null +++ b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java @@ -0,0 +1,71 @@ +package io.github.magnusp.libsql.perf; + +import io.github.magnusp.libsql.hrana.codec.PipelineReqBody; +import io.github.magnusp.libsql.hrana.codec.Stmt; +import io.github.magnusp.libsql.hrana.codec.StreamRequest; +import io.github.magnusp.libsql.hrana.codec.Value; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * Builds the fixed payloads the benchmark measures. + * + *

The tracked payload is deliberately plain ASCII without JSON escapes so that changes in the + * numbers come from the codec, not from a shifting test document. Codec correctness for escapes and + * non-ASCII text is covered by the unit and property tests. + */ +final class Payloads { + + private static final String NOTE = + "lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor"; + private static final String BLOB = Base64.getEncoder().encodeToString(new byte[16]); + private static final String[] NAMES = {"id", "name", "score", "payload", "note", "flag"}; + private static final String[] DECL_TYPES = {"INTEGER", "TEXT", "REAL", "BLOB", "TEXT", "INTEGER"}; + + /** A Hrana {@code execute} response with six columns and the requested number of rows. */ + static byte[] scan(int rows) { + StringBuilder sb = new StringBuilder(rows * 300 + 512); + sb.append("{\"baton\":\"baton-token\",\"base_url\":null,\"results\":[{\"type\":\"ok\",") + .append("\"response\":{\"type\":\"execute\",\"result\":{\"cols\":["); + for (int c = 0; c < NAMES.length; c++) { + if (c > 0) { + sb.append(','); + } + sb.append("{\"name\":\"").append(NAMES[c]).append("\",\"decltype\":\"").append(DECL_TYPES[c]).append("\"}"); + } + sb.append("],\"rows\":["); + for (int r = 0; r < rows; r++) { + if (r > 0) { + sb.append(','); + } + sb.append("[{\"type\":\"integer\",\"value\":\"").append(r).append("\"},") + .append("{\"type\":\"text\",\"value\":\"user-").append(r).append("-name\"},") + .append("{\"type\":\"float\",\"value\":").append(r).append(".25},") + .append("{\"type\":\"blob\",\"base64\":\"").append(BLOB).append("\"},") + .append("{\"type\":\"text\",\"value\":\"").append(NOTE).append("\"},") + .append("{\"type\":\"").append(r % 2 == 0 ? "integer\",\"value\":\"1" : "null").append("\"}]"); + } + sb.append("],\"affected_row_count\":0,\"last_insert_rowid\":null}}}]}"); + return sb.toString().getBytes(StandardCharsets.UTF_8); + } + + /** A pipeline with one parameterized insert, the shape every prepared execute sends. */ + static PipelineReqBody insertRequest() { + List args = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + args.add(switch (i % 3) { + case 0 -> Value.of(123456789L); + case 1 -> Value.of("some text value, with a comma and a \"quote\""); + default -> Value.of(12.5); + }); + } + Stmt stmt = new Stmt( + "INSERT INTO users (a, b, c, d, e, f) VALUES (?, ?, ?, ?, ?, ?)", null, args, List.of(), false); + return new PipelineReqBody("baton-token", List.of(new StreamRequest.Execute(stmt))); + } + + private Payloads() {} +} diff --git a/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java new file mode 100644 index 0000000..c2795a9 --- /dev/null +++ b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java @@ -0,0 +1,301 @@ +package io.github.magnusp.libsql.perf; + +import io.github.magnusp.libsql.hrana.codec.DirectHranaJsonCodec; +import io.github.magnusp.libsql.hrana.codec.HranaJsonCodec; +import io.github.magnusp.libsql.hrana.codec.JacksonHranaJsonCodec; +import io.github.magnusp.libsql.hrana.codec.PipelineReqBody; +import io.github.magnusp.libsql.hrana.codec.PipelineRespBody; +import io.github.magnusp.libsql.hrana.codec.StmtResult; +import io.github.magnusp.libsql.hrana.codec.StreamResponse; +import io.github.magnusp.libsql.hrana.codec.StreamResult; +import io.github.magnusp.libsql.jdbc.LibsqlResultSet; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** + * Driver-side benchmark for the JSON codec and the JDBC result-set getters. + * + *

This measures work that happens in the client process. It intentionally starts no server: the + * PR comparison runs on shared CI hardware, and server and network time would bury codec changes in + * noise. The workloads use fixed payloads and report a median over several windows. + * + *

Run it after {@code ./mvnw -pl libsql-perf -am -DskipTests package}: + * + * 

{@code
+ * java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \
+ *   io.github.magnusp.libsql.perf.PerfBenchmark --json /tmp/perf.json
+ * }
+ */ +public final class PerfBenchmark { + + private static final int SMALL_ROWS = 5_000; + private static final int LARGE_ROWS = 20_000; + + private static volatile Object blackhole; + private static final com.sun.management.ThreadMXBean THREADS = + (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + + public static void main(String[] args) throws Exception { + Options options = Options.parse(args); + Config config = options.quick() ? new Config(500, 150, 3, 300) : new Config(1_000, 250, 5, 500); + + HranaJsonCodec direct = new DirectHranaJsonCodec(); + HranaJsonCodec jackson = new JacksonHranaJsonCodec(); + byte[] small = Payloads.scan(SMALL_ROWS); + byte[] large = Payloads.scan(LARGE_ROWS); + PipelineReqBody insert = Payloads.insertRequest(); + verifyCodecsAgree(direct, jackson, small, insert); + + List metrics = new ArrayList<>(); + metrics.add(measure( + "codec.direct.decode5k", + SMALL_ROWS, + () -> direct.deserializePipelineResponse(small), + config)); + metrics.add(measure( + "codec.jackson.decode5k", + SMALL_ROWS, + () -> jackson.deserializePipelineResponse(small), + config)); + metrics.add(measure( + "codec.direct.decode20k", + LARGE_ROWS, + () -> direct.deserializePipelineResponse(large), + config)); + metrics.add(measure("codec.direct.encode", 0, () -> direct.serializePipelineRequest(insert), config)); + metrics.add(measure("codec.jackson.encode", 0, () -> jackson.serializePipelineRequest(insert), config)); + + StmtResult result = firstResult(direct.deserializePipelineResponse(large)); + metrics.add(measure("jdbc.scan.all", LARGE_ROWS, () -> scanAll(result), config)); + metrics.add(measure("jdbc.scan.object", LARGE_ROWS, () -> scanObject(result), config)); + + if (!options.quiet()) { + printMarkdown(metrics, options, config); + } + if (options.json() != null) { + writeJson(options.json(), metrics, options, config); + } + } + + private static void verifyCodecsAgree( + HranaJsonCodec direct, HranaJsonCodec jackson, byte[] scanPayload, PipelineReqBody insert) throws IOException { + PipelineRespBody directBody = direct.deserializePipelineResponse(scanPayload); + PipelineRespBody jacksonBody = jackson.deserializePipelineResponse(scanPayload); + if (!directBody.equals(jacksonBody)) { + throw new IllegalStateException("the codecs disagree on the scan payload"); + } + if (!Arrays.equals( + direct.serializePipelineRequest(insert), jackson.serializePipelineRequest(insert))) { + throw new IllegalStateException("the codecs disagree on the insert request"); + } + } + + private static StmtResult firstResult(PipelineRespBody body) { + StreamResult streamResult = body.results().get(0); + if (streamResult instanceof StreamResult.Ok ok + && ok.response() instanceof StreamResponse.Execute execute) { + return execute.result(); + } + throw new IllegalStateException("unexpected payload: " + streamResult); + } + + private static Metric measure(String name, int rows, Task task, Config config) throws Exception { + long deadline = System.nanoTime() + config.warmupMs() * 1_000_000L; + while (System.nanoTime() < deadline) { + for (int i = 0; i < 10; i++) { + blackhole = task.run(); + } + } + + long allocBefore = THREADS.getCurrentThreadAllocatedBytes(); + long allocOps = 0; + deadline = System.nanoTime() + config.allocMs() * 1_000_000L; + while (System.nanoTime() < deadline) { + for (int i = 0; i < 10; i++) { + blackhole = task.run(); + } + allocOps += 10; + } + long allocAfter = THREADS.getCurrentThreadAllocatedBytes(); + double bytesPerOp = allocOps == 0 ? 0 : (double) (allocAfter - allocBefore) / allocOps; + + double[] rates = new double[config.windows()]; + for (int window = 0; window < rates.length; window++) { + long count = 0; + long start = System.nanoTime(); + long end = start + config.windowMs() * 1_000_000L; + long now; + do { + for (int i = 0; i < 10; i++) { + blackhole = task.run(); + } + count += 10; + now = System.nanoTime(); + } while (now < end); + rates[window] = count / ((now - start) / 1_000_000_000.0); + } + Arrays.sort(rates); + double median = rates[rates.length / 2]; + return new Metric(name, median, rows == 0 ? 0 : median * rows, bytesPerOp); + } + + private static Object scanAll(StmtResult result) throws Exception { + LibsqlResultSet rs = new LibsqlResultSet(null, result); + long sum = 0; + while (rs.next()) { + sum += rs.getLong(1); + sum += rs.getString(2).length(); + sum += (long) rs.getDouble(3); + sum += rs.getBytes(4).length; + sum += rs.getString(5).length(); + sum += rs.getInt(6); + } + return sum; + } + + private static Object scanObject(StmtResult result) throws Exception { + LibsqlResultSet rs = new LibsqlResultSet(null, result); + long sum = 0; + while (rs.next()) { + for (int column = 1; column <= 6; column++) { + Object value = rs.getObject(column); + if (value != null) { + sum += value.hashCode(); + } + } + } + return sum; + } + + private static void printMarkdown(List metrics, Options options, Config config) { + System.out.printf(Locale.ROOT, "Benchmark commit: %s%n", options.commit()); + System.out.printf( + Locale.ROOT, + "Warmup %d ms, allocation %d ms, %d windows of %d ms%n%n", + config.warmupMs(), + config.allocMs(), + config.windows(), + config.windowMs()); + System.out.println("| Workload | ops/s | rows/s | bytes/op |"); + System.out.println("|---|---:|---:|---:|"); + for (Metric metric : metrics) { + System.out.printf( + Locale.ROOT, + "| %s | %.1f | %.0f | %.0f |%n", + metric.name(), + metric.opsPerSec(), + metric.rowsPerSec(), + metric.bytesPerOp()); + } + } + + private static void writeJson(Path path, List metrics, Options options, Config config) + throws IOException { + Path parent = path.toAbsolutePath().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + StringBuilder json = new StringBuilder(1_024); + json.append("{\n"); + json.append(" \"schemaVersion\": 1,\n"); + json.append(" \"commit\": \"").append(escape(options.commit())).append("\",\n"); + json.append(" \"javaVersion\": \"") + .append(escape(System.getProperty("java.version"))) + .append("\",\n"); + json.append(" \"osName\": \"") + .append(escape(System.getProperty("os.name"))) + .append("\",\n"); + json.append(" \"osArch\": \"") + .append(escape(System.getProperty("os.arch"))) + .append("\",\n"); + json.append(" \"availableProcessors\": ") + .append(Runtime.getRuntime().availableProcessors()) + .append(",\n"); + json.append(" \"config\": {") + .append("\"warmupMs\": ") + .append(config.warmupMs()) + .append(", \"allocMs\": ") + .append(config.allocMs()) + .append(", \"windows\": ") + .append(config.windows()) + .append(", \"windowMs\": ") + .append(config.windowMs()) + .append("},\n"); + json.append(" \"metrics\": {\n"); + for (int i = 0; i < metrics.size(); i++) { + Metric metric = metrics.get(i); + json.append(" \"") + .append(escape(metric.name())) + .append("\": {\"opsPerSec\": ") + .append(number(metric.opsPerSec())) + .append(", \"rowsPerSec\": ") + .append(number(metric.rowsPerSec())) + .append(", \"bytesPerOp\": ") + .append(number(metric.bytesPerOp())) + .append("}"); + json.append(i + 1 < metrics.size() ? ",\n" : "\n"); + } + json.append(" }\n"); + json.append("}\n"); + Files.writeString(path, json.toString()); + } + + private static String number(double value) { + return String.format(Locale.ROOT, "%.3f", value); + } + + private static String escape(String value) { + if (value == null) { + return "unknown"; + } + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private record Metric(String name, double opsPerSec, double rowsPerSec, double bytesPerOp) {} + + private record Config(long warmupMs, long allocMs, int windows, long windowMs) {} + + private record Options(Path json, String commit, boolean quiet, boolean quick) { + static Options parse(String[] args) { + Path json = null; + String commit = System.getenv().getOrDefault("GITHUB_SHA", "unknown"); + boolean quiet = false; + boolean quick = false; + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--json" -> json = Path.of(requireValue(args, ++i, "--json")); + case "--commit" -> commit = requireValue(args, ++i, "--commit"); + case "--quiet" -> quiet = true; + case "--quick" -> quick = true; + case "--help", "-h" -> { + System.out.println("Usage: PerfBenchmark [--json PATH] [--commit SHA] [--quiet] [--quick]"); + System.exit(0); + } + default -> throw new IllegalArgumentException("unknown argument: " + args[i]); + } + } + return new Options(json, commit, quiet, quick); + } + + private static String requireValue(String[] args, int index, String flag) { + if (index >= args.length) { + throw new IllegalArgumentException(flag + " requires a value"); + } + return args[index]; + } + } + + @FunctionalInterface + private interface Task { + Object run() throws Exception; + } + + private PerfBenchmark() {} +} diff --git a/pom.xml b/pom.xml index 67beb7a..91b87ae 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ libsql-jdbc libsql-itests libsql-conformance + libsql-perf diff --git a/tools/perf/comment.sh b/tools/perf/comment.sh new file mode 100755 index 0000000..2221c7c --- /dev/null +++ b/tools/perf/comment.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Create or update the sticky performance comment on a pull request. +# +# usage: comment.sh +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +repo=$1 +pr=$2 +body_file=$3 +marker='' + +if [[ ! -f "$body_file" ]]; then + echo "comment file not found: $body_file" >&2 + exit 2 +fi + +existing=$(gh api --paginate "repos/${repo}/issues/${pr}/comments" \ + --jq ".[] | select(.body | startswith(\"${marker}\")) | .id" | head -n 1 || true) + +payload=$(mktemp) +trap 'rm -f "$payload"' EXIT +jq -Rs '{body: .}' "$body_file" >"$payload" + +if [[ -n "$existing" ]]; then + gh api --method PATCH "repos/${repo}/issues/comments/${existing}" --input "$payload" >/dev/null + echo "Updated performance comment ${existing} on #${pr}" +else + gh api --method POST "repos/${repo}/issues/${pr}/comments" --input "$payload" >/dev/null + echo "Created performance comment on #${pr}" +fi diff --git a/tools/perf/render-comment.py b/tools/perf/render-comment.py new file mode 100755 index 0000000..f9a42b4 --- /dev/null +++ b/tools/perf/render-comment.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Render the PR performance comment from PerfBenchmark JSON output. + +The base and head sides are each measured in several runs. Per metric, this script takes the median +across runs and compares head with base. Throughput drops and allocation increases beyond the +threshold are flagged; the job stays green either way, so reviewers judge the numbers. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sys +from pathlib import Path + +MARKER = "" +DEFAULT_THRESHOLD = 10.0 + +# metric name, display label, rate column ("rows" or "ops") +WORKLOADS = [ + ("codec.direct.decode5k", "Decode 5k rows (direct)", "rows"), + ("codec.jackson.decode5k", "Decode 5k rows (Jackson baseline)", "rows"), + ("codec.direct.decode20k", "Decode 20k rows (direct)", "rows"), + ("codec.direct.encode", "Encode one execute (direct)", "ops"), + ("codec.jackson.encode", "Encode one execute (Jackson baseline)", "ops"), + ("jdbc.scan.all", "JDBC scan + getters (20k rows)", "rows"), + ("jdbc.scan.object", "JDBC scan + getObject (20k rows)", "rows"), +] + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", nargs="+", type=Path, default=None, help="base JSON files") + parser.add_argument("--head", nargs="+", type=Path, required=True, help="head JSON files") + parser.add_argument("--base-sha", default="unknown") + parser.add_argument("--head-sha", default="unknown") + parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD) + parser.add_argument("--out", type=Path, default=None) + return parser.parse_args(argv) + + +def load_runs(paths: list[Path] | None) -> tuple[dict, dict] | tuple[None, None]: + if not paths: + return None, {} + runs = [json.loads(path.read_text()) for path in paths] + names = set(runs[0]["metrics"]) + metrics = {} + for name in names: + metrics[name] = { + key: statistics.median(run["metrics"][name][key] for run in runs if name in run["metrics"]) + for key in ("opsPerSec", "rowsPerSec", "bytesPerOp") + } + return runs[0], metrics + + +def fmt_rate(value: float, kind: str) -> str: + unit = "rows/s" if kind == "rows" else "req/s" + if value >= 1_000_000: + return f"{value / 1_000_000:.2f}M {unit}" + if value >= 1_000: + return f"{value / 1_000:.1f}k {unit}" + return f"{value:.0f} {unit}" + + +def fmt_bytes(value: float) -> str: + if value >= 1_000_000: + return f"{value / 1_000_000:.2f} MB/op" + if value >= 1_000: + return f"{value / 1_000:.2f} kB/op" + return f"{value:.0f} B/op" + + +def pct(base: float, head: float) -> float | None: + if base == 0: + return None + return (head - base) / base * 100.0 + + +def fmt_delta(delta: float | None, lower_is_better: bool, threshold: float) -> str: + if delta is None: + return "n/a" + text = f"{delta:+.1f}%" + regressed = delta > threshold if lower_is_better else delta < -threshold + if regressed: + text += " ⚠️" + return text + + +def short(sha: str) -> str: + return sha[:12] if sha and sha != "unknown" else "unknown" + + +def plural(count: int, word: str) -> str: + return word if count == 1 else word + "s" + + +def render(args: argparse.Namespace) -> str: + base_meta, base_metrics = load_runs(args.base) + head_meta, head_metrics = load_runs(args.head) + if head_meta is None: + raise SystemExit("no head results to render") + + head_runs = len(args.head) + base_runs = len(args.base) if args.base else 0 + + lines = [MARKER, "## Performance benchmark", ""] + if base_meta is None: + lines.append( + f"Head `{short(args.head_sha)}`: absolute numbers only, because the benchmark is not " + "present in the base commit. Deltas start with the next PR." + ) + else: + lines.append( + f"Base `{short(args.base_sha)}` → head `{short(args.head_sha)}`. " + f"Medians of {base_runs} base {plural(base_runs, 'run')} and " + f"{head_runs} head {plural(head_runs, 'run')}, interleaved on the same runner." + ) + lines.append("") + lines.append( + f"Driver-side microbenchmarks only; server and network time are excluded. " + f"Shared-runner noise is about ±{args.threshold:.0f}%, so only larger regressions are flagged." + ) + lines.append("") + lines.append(f"JDK {head_meta.get('javaVersion', '?')}, {head_meta.get('osName', '?')} " + f"{head_meta.get('osArch', '?')}, {head_meta.get('availableProcessors', '?')} CPUs.") + lines.append("") + + lines.append("### Throughput") + lines.append("") + lines.append("| Workload | Base | Head | Δ |") + lines.append("|---|---:|---:|---:|") + for name, label, kind in WORKLOADS: + if name not in head_metrics: + continue + head_value = head_metrics[name]["rowsPerSec" if kind == "rows" else "opsPerSec"] + if base_metrics and name in base_metrics: + base_value = base_metrics[name]["rowsPerSec" if kind == "rows" else "opsPerSec"] + base_text = fmt_rate(base_value, kind) + delta = fmt_delta(pct(base_value, head_value), False, args.threshold) + else: + base_text = "—" + delta = "n/a" + lines.append(f"| {label} | {base_text} | {fmt_rate(head_value, kind)} | {delta} |") + lines.append("") + + lines.append("### Allocation") + lines.append("") + lines.append("| Workload | Base | Head | Δ |") + lines.append("|---|---:|---:|---:|") + for name, label, _ in WORKLOADS: + if name not in head_metrics: + continue + head_value = head_metrics[name]["bytesPerOp"] + if base_metrics and name in base_metrics: + base_value = base_metrics[name]["bytesPerOp"] + base_text = fmt_bytes(base_value) + delta = fmt_delta(pct(base_value, head_value), True, args.threshold) + else: + base_text = "—" + delta = "n/a" + lines.append(f"| {label} | {base_text} | {fmt_bytes(head_value)} | {delta} |") + lines.append("") + lines.append( + "Workloads and thresholds live in `tools/perf/render-comment.py`. " + "Run locally with `./mvnw -pl libsql-perf -am -DskipTests package` and see " + "[`libsql-perf/README.md`](libsql-perf/README.md)." + ) + return "\n".join(lines) + "\n" + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + comment = render(args) + if args.out: + args.out.write_text(comment) + else: + sys.stdout.write(comment) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) From ba1ac610055be2c551510ef3b4870001a5fc2cb6 Mon Sep 17 00:00:00 2001 From: Magnus Persson <1431685+magnusp@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:58:47 +0100 Subject: [PATCH 2/2] ci(perf): use upload-artifact v7 --- .github/workflows/perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 85fa731..57381ce 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -106,7 +106,7 @@ jobs: - name: Upload benchmark results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: perf-results path: results/