diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml new file mode 100644 index 0000000..57381ce --- /dev/null +++ b/.github/workflows/perf.yml @@ -0,0 +1,113 @@ +name: Performance + +# Posts a sticky comment with driver-side benchmark numbers. The base commit and the PR merge +# commit are measured on the same runner, three interleaved rounds each, so the comparison survives +# shared-runner noise. The job is informational: regressions are flagged, not failed. + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: perf-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + benchmark: + name: Codec & JDBC benchmark + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: read + pull-requests: write + steps: + - name: Check out base + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + persist-credentials: false + + # The default pull_request ref is the merge commit, which isolates the PR changes when the + # base is the base branch tip. It also works for pull requests from forks. + - name: Check out PR merge commit + uses: actions/checkout@v7 + with: + path: head + persist-credentials: false + + - name: Set up Amazon Corretto 25 + uses: actions/setup-java@v6 + with: + distribution: 'corretto' + java-version: '25' + cache: 'maven' + + - name: Build benchmarkable modules + run: | + set -euo pipefail + if [ -d base/libsql-perf ]; then + (cd base && ./mvnw --batch-mode --quiet -Dmaven.test.skip=true -pl libsql-perf -am package) + echo "BASE_BENCHMARK=1" >> "$GITHUB_ENV" + else + echo "Base commit has no libsql-perf module yet; this run reports head numbers only." + fi + (cd head && ./mvnw --batch-mode --quiet -Dmaven.test.skip=true -pl libsql-perf -am package) + + - name: Run benchmark (three interleaved rounds) + run: | + set -euo pipefail + mkdir -p results + run() { + local dir=$1 out=$2 sha=$3 + java -Xms1g -Xmx1g \ + -cp "$dir/libsql-perf/target/classes:$dir/libsql-perf/target/lib/*" \ + io.github.magnusp.libsql.perf.PerfBenchmark \ + --quiet --json "$out" --commit "$sha" + } + for i in 1 2 3; do + if [ "${BASE_BENCHMARK:-0}" = "1" ]; then + run base "results/base-$i.json" "${{ github.event.pull_request.base.sha }}" + fi + run head "results/head-$i.json" "${{ github.sha }}" + done + + - name: Render comment + run: | + set -euo pipefail + if [ "${BASE_BENCHMARK:-0}" = "1" ]; then + python3 head/tools/perf/render-comment.py \ + --base results/base-1.json results/base-2.json results/base-3.json \ + --head results/head-1.json results/head-2.json results/head-3.json \ + --base-sha "${{ github.event.pull_request.base.sha }}" \ + --head-sha "${{ github.sha }}" \ + --out comment.md + else + python3 head/tools/perf/render-comment.py \ + --head results/head-1.json results/head-2.json results/head-3.json \ + --base-sha "${{ github.event.pull_request.base.sha }}" \ + --head-sha "${{ github.sha }}" \ + --out comment.md + fi + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Post or update PR comment + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + head/tools/perf/comment.sh \ + "${{ github.repository }}" \ + "${{ github.event.pull_request.number }}" \ + comment.md + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v7 + with: + name: perf-results + path: results/ + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 07b3f86..d726f90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,6 +34,7 @@ libsql-java/ ├── libsql-jdbc/ # Standard JDBC 4.3 driver (java.sql.* implementation) ├── libsql-itests/ # Testcontainers (sqld) and Toxiproxy integration tests ├── libsql-conformance/ # Selected xerial/sqlite-jdbc tests running against libSQL +├── libsql-perf/ # Driver-side codec/JDBC benchmark posted on pull requests ├── tools/ # Maintainer scripts such as export-upstream ├── upstream-xerial_sqlite-jdbc/ # Vendored xerial/sqlite-jdbc sources for conformance tests └── vendor/ # Upstream specifications and documentation (Hrana v3, sqld) @@ -99,6 +100,23 @@ mise exec -- ./mvnw -pl libsql-itests -Dtest=HikariCpIntegrationIT test Under nono, set `DOCKER_CONFIG` before the command, as described in the workaround below. +### Performance Benchmark + +`libsql-perf` measures codec decode/encode and the JDBC result-set getters without a server. The +[`Performance`](.github/workflows/perf.yml) workflow builds the PR merge commit and its base +commit, runs three interleaved rounds, and posts a sticky comparison comment. The job flags +regressions over 10% but does not fail the check. + +```bash +mise exec -- ./mvnw -pl libsql-perf -am -DskipTests package +mise exec -- java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \ + io.github.magnusp.libsql.perf.PerfBenchmark --quick +``` + +See [`libsql-perf/README.md`](libsql-perf/README.md) for the workloads and how to change them. The +benchmark payload is a measured baseline: change it in a dedicated PR and call it out, so the first +comment after the change is not read as a regression. + ### nono Sandbox and Docker Config This environment may run inside a `nono` filesystem sandbox. The sandbox can block reads of `~/.docker/config.json` even when `docker info` works. Testcontainers then fails with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43a32b5..ca87d84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,6 +59,7 @@ The repository is organized as a multi-module Maven project: - **`libsql-jdbc`**: Standard JDBC 4.3 compliant driver implementation (`jdbc:libsql:...`). - **`libsql-itests`**: End-to-end integration tests using Testcontainers and sqld. - **`libsql-conformance`**: Selected xerial/sqlite-jdbc tests running against libSQL. +- **`libsql-perf`**: Driver-side codec and JDBC benchmark used for the PR performance comment. ## Code Style & Quality diff --git a/README.md b/README.md index f07ead3..9aadbc9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ A pure-Java JDBC driver, client library, and protocol codec for **libSQL** and * | [`libsql-jdbc`](libsql-jdbc) | Standard JDBC 4.3 driver (`jdbc:libsql:...`) | | [`libsql-itests`](libsql-itests) | Integration tests against `sqld` running in Testcontainers | | [`libsql-conformance`](libsql-conformance/README.md) | Selected xerial/sqlite-jdbc tests running against libSQL | +| [`libsql-perf`](libsql-perf) | Driver-side codec and JDBC benchmark used for the PR performance comment | ## Getting Started diff --git a/libsql-perf/README.md b/libsql-perf/README.md new file mode 100644 index 0000000..875e58f --- /dev/null +++ b/libsql-perf/README.md @@ -0,0 +1,69 @@ +# libsql-perf + +Driver-side performance benchmark for the Hrana JSON codec and the JDBC result-set getters. + +The [`Performance`](../.github/workflows/perf.yml) workflow runs it for every pull request, compares +the PR merge commit with the base commit, and posts the result as a sticky comment. The job is +informational: regressions are flagged in the comment, not failed in the check. + +## What it measures + +| Workload | What it exercises | +|---|---| +| `codec.direct.decode5k` | `DirectHranaJsonCodec` decoding a 5,000-row, six-column `execute` result | +| `codec.jackson.decode5k` | The optional Jackson codec on the same payload, as a baseline | +| `codec.direct.decode20k` | The direct codec on a 20,000-row result | +| `codec.direct.encode` | The direct codec serializing one parameterized execute | +| `codec.jackson.encode` | The Jackson codec on the same request, as a baseline | +| `jdbc.scan.all` | `LibsqlResultSet.next()` plus typed getters over 20,000 rows | +| `jdbc.scan.object` | `ResultSet.getObject()` over all six columns of 20,000 rows | + +Every workload reports operations per second, rows per second (where applicable), and allocated +bytes per operation from `java.lang.management.ThreadMXBean`. The benchmark takes the median of +several measurement windows and prints Markdown, or writes JSON for the PR comment. + +The workload is deliberately driver-side. Starting `sqld` would add server and network time that +dominates the codec and result-set costs the benchmark is meant to track. The integration tests in +`libsql-itests` cover end-to-end throughput. + +## Running it locally + +```bash +./mvnw -pl libsql-perf -am -DskipTests package +java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \ + io.github.magnusp.libsql.perf.PerfBenchmark +``` + +Useful flags: + +| Flag | Effect | +|---|---| +| `--quick` | Shorter warmup and fewer, shorter windows for a rough check | +| `--json ` | Write machine-readable results | +| `--commit ` | Set the commit recorded in the JSON output | +| `--quiet` | Do not print the Markdown summary | + +To see the same comparison the workflow posts, run the base and head revisions and render them: + +```bash +./tools/perf/render-comment.py \ + --base base.json --head head.json \ + --base-sha --head-sha +``` + +## Reading the comparison + +The workflow checks out the base commit and the PR merge commit, builds both, then runs three +interleaved rounds per side on the same runner. The comment reports medians across those rounds. +Throughput drops and allocation increases larger than 10% are marked with a warning; shared-runner +noise below that band is not meaningful. + +The comparison is against the PR base, not a historical record. To follow a metric across releases, +compare the comments or the `perf-results` artifact attached to each run. + +## Changing the benchmark + +Adding a workload means adding a `measure(...)` call in `PerfBenchmark` and a matching entry in +`WORKLOADS` in [`tools/perf/render-comment.py`](../tools/perf/render-comment.py). Changing the +payload, warmup, or window size changes the baseline; call that out in the PR description so the +first comment after the change is not read as a regression. diff --git a/libsql-perf/pom.xml b/libsql-perf/pom.xml new file mode 100644 index 0000000..427f1d5 --- /dev/null +++ b/libsql-perf/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + io.github.magnusp.libsql + libsql-java-parent + ${revision}${sha1}${changelist} + + + libsql-perf + libSQL Java Performance Benchmark + Driver-side codec and JDBC benchmark used by the PR performance comment + + + + true + + + + + io.github.magnusp.libsql + libsql-hrana-codec + + + io.github.magnusp.libsql + libsql-client + + + io.github.magnusp.libsql + libsql-jdbc + + + com.fasterxml.jackson.core + jackson-core + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.9.0 + + + copy-runtime-dependencies + package + + copy-dependencies + + + ${project.build.directory}/lib + runtime + + + + + + + diff --git a/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java new file mode 100644 index 0000000..ef1a39c --- /dev/null +++ b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/Payloads.java @@ -0,0 +1,71 @@ +package io.github.magnusp.libsql.perf; + +import io.github.magnusp.libsql.hrana.codec.PipelineReqBody; +import io.github.magnusp.libsql.hrana.codec.Stmt; +import io.github.magnusp.libsql.hrana.codec.StreamRequest; +import io.github.magnusp.libsql.hrana.codec.Value; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * Builds the fixed payloads the benchmark measures. + * + *

The tracked payload is deliberately plain ASCII without JSON escapes so that changes in the + * numbers come from the codec, not from a shifting test document. Codec correctness for escapes and + * non-ASCII text is covered by the unit and property tests. + */ +final class Payloads { + + private static final String NOTE = + "lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor"; + private static final String BLOB = Base64.getEncoder().encodeToString(new byte[16]); + private static final String[] NAMES = {"id", "name", "score", "payload", "note", "flag"}; + private static final String[] DECL_TYPES = {"INTEGER", "TEXT", "REAL", "BLOB", "TEXT", "INTEGER"}; + + /** A Hrana {@code execute} response with six columns and the requested number of rows. */ + static byte[] scan(int rows) { + StringBuilder sb = new StringBuilder(rows * 300 + 512); + sb.append("{\"baton\":\"baton-token\",\"base_url\":null,\"results\":[{\"type\":\"ok\",") + .append("\"response\":{\"type\":\"execute\",\"result\":{\"cols\":["); + for (int c = 0; c < NAMES.length; c++) { + if (c > 0) { + sb.append(','); + } + sb.append("{\"name\":\"").append(NAMES[c]).append("\",\"decltype\":\"").append(DECL_TYPES[c]).append("\"}"); + } + sb.append("],\"rows\":["); + for (int r = 0; r < rows; r++) { + if (r > 0) { + sb.append(','); + } + sb.append("[{\"type\":\"integer\",\"value\":\"").append(r).append("\"},") + .append("{\"type\":\"text\",\"value\":\"user-").append(r).append("-name\"},") + .append("{\"type\":\"float\",\"value\":").append(r).append(".25},") + .append("{\"type\":\"blob\",\"base64\":\"").append(BLOB).append("\"},") + .append("{\"type\":\"text\",\"value\":\"").append(NOTE).append("\"},") + .append("{\"type\":\"").append(r % 2 == 0 ? "integer\",\"value\":\"1" : "null").append("\"}]"); + } + sb.append("],\"affected_row_count\":0,\"last_insert_rowid\":null}}}]}"); + return sb.toString().getBytes(StandardCharsets.UTF_8); + } + + /** A pipeline with one parameterized insert, the shape every prepared execute sends. */ + static PipelineReqBody insertRequest() { + List args = new ArrayList<>(); + for (int i = 0; i < 6; i++) { + args.add(switch (i % 3) { + case 0 -> Value.of(123456789L); + case 1 -> Value.of("some text value, with a comma and a \"quote\""); + default -> Value.of(12.5); + }); + } + Stmt stmt = new Stmt( + "INSERT INTO users (a, b, c, d, e, f) VALUES (?, ?, ?, ?, ?, ?)", null, args, List.of(), false); + return new PipelineReqBody("baton-token", List.of(new StreamRequest.Execute(stmt))); + } + + private Payloads() {} +} diff --git a/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java new file mode 100644 index 0000000..c2795a9 --- /dev/null +++ b/libsql-perf/src/main/java/io/github/magnusp/libsql/perf/PerfBenchmark.java @@ -0,0 +1,301 @@ +package io.github.magnusp.libsql.perf; + +import io.github.magnusp.libsql.hrana.codec.DirectHranaJsonCodec; +import io.github.magnusp.libsql.hrana.codec.HranaJsonCodec; +import io.github.magnusp.libsql.hrana.codec.JacksonHranaJsonCodec; +import io.github.magnusp.libsql.hrana.codec.PipelineReqBody; +import io.github.magnusp.libsql.hrana.codec.PipelineRespBody; +import io.github.magnusp.libsql.hrana.codec.StmtResult; +import io.github.magnusp.libsql.hrana.codec.StreamResponse; +import io.github.magnusp.libsql.hrana.codec.StreamResult; +import io.github.magnusp.libsql.jdbc.LibsqlResultSet; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** + * Driver-side benchmark for the JSON codec and the JDBC result-set getters. + * + *

This measures work that happens in the client process. It intentionally starts no server: the + * PR comparison runs on shared CI hardware, and server and network time would bury codec changes in + * noise. The workloads use fixed payloads and report a median over several windows. + * + *

Run it after {@code ./mvnw -pl libsql-perf -am -DskipTests package}: + * + * 

{@code
+ * java -cp 'libsql-perf/target/classes:libsql-perf/target/lib/*' \
+ *   io.github.magnusp.libsql.perf.PerfBenchmark --json /tmp/perf.json
+ * }
+ */ +public final class PerfBenchmark { + + private static final int SMALL_ROWS = 5_000; + private static final int LARGE_ROWS = 20_000; + + private static volatile Object blackhole; + private static final com.sun.management.ThreadMXBean THREADS = + (com.sun.management.ThreadMXBean) ManagementFactory.getThreadMXBean(); + + public static void main(String[] args) throws Exception { + Options options = Options.parse(args); + Config config = options.quick() ? new Config(500, 150, 3, 300) : new Config(1_000, 250, 5, 500); + + HranaJsonCodec direct = new DirectHranaJsonCodec(); + HranaJsonCodec jackson = new JacksonHranaJsonCodec(); + byte[] small = Payloads.scan(SMALL_ROWS); + byte[] large = Payloads.scan(LARGE_ROWS); + PipelineReqBody insert = Payloads.insertRequest(); + verifyCodecsAgree(direct, jackson, small, insert); + + List metrics = new ArrayList<>(); + metrics.add(measure( + "codec.direct.decode5k", + SMALL_ROWS, + () -> direct.deserializePipelineResponse(small), + config)); + metrics.add(measure( + "codec.jackson.decode5k", + SMALL_ROWS, + () -> jackson.deserializePipelineResponse(small), + config)); + metrics.add(measure( + "codec.direct.decode20k", + LARGE_ROWS, + () -> direct.deserializePipelineResponse(large), + config)); + metrics.add(measure("codec.direct.encode", 0, () -> direct.serializePipelineRequest(insert), config)); + metrics.add(measure("codec.jackson.encode", 0, () -> jackson.serializePipelineRequest(insert), config)); + + StmtResult result = firstResult(direct.deserializePipelineResponse(large)); + metrics.add(measure("jdbc.scan.all", LARGE_ROWS, () -> scanAll(result), config)); + metrics.add(measure("jdbc.scan.object", LARGE_ROWS, () -> scanObject(result), config)); + + if (!options.quiet()) { + printMarkdown(metrics, options, config); + } + if (options.json() != null) { + writeJson(options.json(), metrics, options, config); + } + } + + private static void verifyCodecsAgree( + HranaJsonCodec direct, HranaJsonCodec jackson, byte[] scanPayload, PipelineReqBody insert) throws IOException { + PipelineRespBody directBody = direct.deserializePipelineResponse(scanPayload); + PipelineRespBody jacksonBody = jackson.deserializePipelineResponse(scanPayload); + if (!directBody.equals(jacksonBody)) { + throw new IllegalStateException("the codecs disagree on the scan payload"); + } + if (!Arrays.equals( + direct.serializePipelineRequest(insert), jackson.serializePipelineRequest(insert))) { + throw new IllegalStateException("the codecs disagree on the insert request"); + } + } + + private static StmtResult firstResult(PipelineRespBody body) { + StreamResult streamResult = body.results().get(0); + if (streamResult instanceof StreamResult.Ok ok + && ok.response() instanceof StreamResponse.Execute execute) { + return execute.result(); + } + throw new IllegalStateException("unexpected payload: " + streamResult); + } + + private static Metric measure(String name, int rows, Task task, Config config) throws Exception { + long deadline = System.nanoTime() + config.warmupMs() * 1_000_000L; + while (System.nanoTime() < deadline) { + for (int i = 0; i < 10; i++) { + blackhole = task.run(); + } + } + + long allocBefore = THREADS.getCurrentThreadAllocatedBytes(); + long allocOps = 0; + deadline = System.nanoTime() + config.allocMs() * 1_000_000L; + while (System.nanoTime() < deadline) { + for (int i = 0; i < 10; i++) { + blackhole = task.run(); + } + allocOps += 10; + } + long allocAfter = THREADS.getCurrentThreadAllocatedBytes(); + double bytesPerOp = allocOps == 0 ? 0 : (double) (allocAfter - allocBefore) / allocOps; + + double[] rates = new double[config.windows()]; + for (int window = 0; window < rates.length; window++) { + long count = 0; + long start = System.nanoTime(); + long end = start + config.windowMs() * 1_000_000L; + long now; + do { + for (int i = 0; i < 10; i++) { + blackhole = task.run(); + } + count += 10; + now = System.nanoTime(); + } while (now < end); + rates[window] = count / ((now - start) / 1_000_000_000.0); + } + Arrays.sort(rates); + double median = rates[rates.length / 2]; + return new Metric(name, median, rows == 0 ? 0 : median * rows, bytesPerOp); + } + + private static Object scanAll(StmtResult result) throws Exception { + LibsqlResultSet rs = new LibsqlResultSet(null, result); + long sum = 0; + while (rs.next()) { + sum += rs.getLong(1); + sum += rs.getString(2).length(); + sum += (long) rs.getDouble(3); + sum += rs.getBytes(4).length; + sum += rs.getString(5).length(); + sum += rs.getInt(6); + } + return sum; + } + + private static Object scanObject(StmtResult result) throws Exception { + LibsqlResultSet rs = new LibsqlResultSet(null, result); + long sum = 0; + while (rs.next()) { + for (int column = 1; column <= 6; column++) { + Object value = rs.getObject(column); + if (value != null) { + sum += value.hashCode(); + } + } + } + return sum; + } + + private static void printMarkdown(List metrics, Options options, Config config) { + System.out.printf(Locale.ROOT, "Benchmark commit: %s%n", options.commit()); + System.out.printf( + Locale.ROOT, + "Warmup %d ms, allocation %d ms, %d windows of %d ms%n%n", + config.warmupMs(), + config.allocMs(), + config.windows(), + config.windowMs()); + System.out.println("| Workload | ops/s | rows/s | bytes/op |"); + System.out.println("|---|---:|---:|---:|"); + for (Metric metric : metrics) { + System.out.printf( + Locale.ROOT, + "| %s | %.1f | %.0f | %.0f |%n", + metric.name(), + metric.opsPerSec(), + metric.rowsPerSec(), + metric.bytesPerOp()); + } + } + + private static void writeJson(Path path, List metrics, Options options, Config config) + throws IOException { + Path parent = path.toAbsolutePath().getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + StringBuilder json = new StringBuilder(1_024); + json.append("{\n"); + json.append(" \"schemaVersion\": 1,\n"); + json.append(" \"commit\": \"").append(escape(options.commit())).append("\",\n"); + json.append(" \"javaVersion\": \"") + .append(escape(System.getProperty("java.version"))) + .append("\",\n"); + json.append(" \"osName\": \"") + .append(escape(System.getProperty("os.name"))) + .append("\",\n"); + json.append(" \"osArch\": \"") + .append(escape(System.getProperty("os.arch"))) + .append("\",\n"); + json.append(" \"availableProcessors\": ") + .append(Runtime.getRuntime().availableProcessors()) + .append(",\n"); + json.append(" \"config\": {") + .append("\"warmupMs\": ") + .append(config.warmupMs()) + .append(", \"allocMs\": ") + .append(config.allocMs()) + .append(", \"windows\": ") + .append(config.windows()) + .append(", \"windowMs\": ") + .append(config.windowMs()) + .append("},\n"); + json.append(" \"metrics\": {\n"); + for (int i = 0; i < metrics.size(); i++) { + Metric metric = metrics.get(i); + json.append(" \"") + .append(escape(metric.name())) + .append("\": {\"opsPerSec\": ") + .append(number(metric.opsPerSec())) + .append(", \"rowsPerSec\": ") + .append(number(metric.rowsPerSec())) + .append(", \"bytesPerOp\": ") + .append(number(metric.bytesPerOp())) + .append("}"); + json.append(i + 1 < metrics.size() ? ",\n" : "\n"); + } + json.append(" }\n"); + json.append("}\n"); + Files.writeString(path, json.toString()); + } + + private static String number(double value) { + return String.format(Locale.ROOT, "%.3f", value); + } + + private static String escape(String value) { + if (value == null) { + return "unknown"; + } + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + private record Metric(String name, double opsPerSec, double rowsPerSec, double bytesPerOp) {} + + private record Config(long warmupMs, long allocMs, int windows, long windowMs) {} + + private record Options(Path json, String commit, boolean quiet, boolean quick) { + static Options parse(String[] args) { + Path json = null; + String commit = System.getenv().getOrDefault("GITHUB_SHA", "unknown"); + boolean quiet = false; + boolean quick = false; + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "--json" -> json = Path.of(requireValue(args, ++i, "--json")); + case "--commit" -> commit = requireValue(args, ++i, "--commit"); + case "--quiet" -> quiet = true; + case "--quick" -> quick = true; + case "--help", "-h" -> { + System.out.println("Usage: PerfBenchmark [--json PATH] [--commit SHA] [--quiet] [--quick]"); + System.exit(0); + } + default -> throw new IllegalArgumentException("unknown argument: " + args[i]); + } + } + return new Options(json, commit, quiet, quick); + } + + private static String requireValue(String[] args, int index, String flag) { + if (index >= args.length) { + throw new IllegalArgumentException(flag + " requires a value"); + } + return args[index]; + } + } + + @FunctionalInterface + private interface Task { + Object run() throws Exception; + } + + private PerfBenchmark() {} +} diff --git a/pom.xml b/pom.xml index 67beb7a..91b87ae 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ libsql-jdbc libsql-itests libsql-conformance + libsql-perf diff --git a/tools/perf/comment.sh b/tools/perf/comment.sh new file mode 100755 index 0000000..2221c7c --- /dev/null +++ b/tools/perf/comment.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Create or update the sticky performance comment on a pull request. +# +# usage: comment.sh +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +repo=$1 +pr=$2 +body_file=$3 +marker='' + +if [[ ! -f "$body_file" ]]; then + echo "comment file not found: $body_file" >&2 + exit 2 +fi + +existing=$(gh api --paginate "repos/${repo}/issues/${pr}/comments" \ + --jq ".[] | select(.body | startswith(\"${marker}\")) | .id" | head -n 1 || true) + +payload=$(mktemp) +trap 'rm -f "$payload"' EXIT +jq -Rs '{body: .}' "$body_file" >"$payload" + +if [[ -n "$existing" ]]; then + gh api --method PATCH "repos/${repo}/issues/comments/${existing}" --input "$payload" >/dev/null + echo "Updated performance comment ${existing} on #${pr}" +else + gh api --method POST "repos/${repo}/issues/${pr}/comments" --input "$payload" >/dev/null + echo "Created performance comment on #${pr}" +fi diff --git a/tools/perf/render-comment.py b/tools/perf/render-comment.py new file mode 100755 index 0000000..f9a42b4 --- /dev/null +++ b/tools/perf/render-comment.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Render the PR performance comment from PerfBenchmark JSON output. + +The base and head sides are each measured in several runs. Per metric, this script takes the median +across runs and compares head with base. Throughput drops and allocation increases beyond the +threshold are flagged; the job stays green either way, so reviewers judge the numbers. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import sys +from pathlib import Path + +MARKER = "" +DEFAULT_THRESHOLD = 10.0 + +# metric name, display label, rate column ("rows" or "ops") +WORKLOADS = [ + ("codec.direct.decode5k", "Decode 5k rows (direct)", "rows"), + ("codec.jackson.decode5k", "Decode 5k rows (Jackson baseline)", "rows"), + ("codec.direct.decode20k", "Decode 20k rows (direct)", "rows"), + ("codec.direct.encode", "Encode one execute (direct)", "ops"), + ("codec.jackson.encode", "Encode one execute (Jackson baseline)", "ops"), + ("jdbc.scan.all", "JDBC scan + getters (20k rows)", "rows"), + ("jdbc.scan.object", "JDBC scan + getObject (20k rows)", "rows"), +] + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", nargs="+", type=Path, default=None, help="base JSON files") + parser.add_argument("--head", nargs="+", type=Path, required=True, help="head JSON files") + parser.add_argument("--base-sha", default="unknown") + parser.add_argument("--head-sha", default="unknown") + parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD) + parser.add_argument("--out", type=Path, default=None) + return parser.parse_args(argv) + + +def load_runs(paths: list[Path] | None) -> tuple[dict, dict] | tuple[None, None]: + if not paths: + return None, {} + runs = [json.loads(path.read_text()) for path in paths] + names = set(runs[0]["metrics"]) + metrics = {} + for name in names: + metrics[name] = { + key: statistics.median(run["metrics"][name][key] for run in runs if name in run["metrics"]) + for key in ("opsPerSec", "rowsPerSec", "bytesPerOp") + } + return runs[0], metrics + + +def fmt_rate(value: float, kind: str) -> str: + unit = "rows/s" if kind == "rows" else "req/s" + if value >= 1_000_000: + return f"{value / 1_000_000:.2f}M {unit}" + if value >= 1_000: + return f"{value / 1_000:.1f}k {unit}" + return f"{value:.0f} {unit}" + + +def fmt_bytes(value: float) -> str: + if value >= 1_000_000: + return f"{value / 1_000_000:.2f} MB/op" + if value >= 1_000: + return f"{value / 1_000:.2f} kB/op" + return f"{value:.0f} B/op" + + +def pct(base: float, head: float) -> float | None: + if base == 0: + return None + return (head - base) / base * 100.0 + + +def fmt_delta(delta: float | None, lower_is_better: bool, threshold: float) -> str: + if delta is None: + return "n/a" + text = f"{delta:+.1f}%" + regressed = delta > threshold if lower_is_better else delta < -threshold + if regressed: + text += " ⚠️" + return text + + +def short(sha: str) -> str: + return sha[:12] if sha and sha != "unknown" else "unknown" + + +def plural(count: int, word: str) -> str: + return word if count == 1 else word + "s" + + +def render(args: argparse.Namespace) -> str: + base_meta, base_metrics = load_runs(args.base) + head_meta, head_metrics = load_runs(args.head) + if head_meta is None: + raise SystemExit("no head results to render") + + head_runs = len(args.head) + base_runs = len(args.base) if args.base else 0 + + lines = [MARKER, "## Performance benchmark", ""] + if base_meta is None: + lines.append( + f"Head `{short(args.head_sha)}`: absolute numbers only, because the benchmark is not " + "present in the base commit. Deltas start with the next PR." + ) + else: + lines.append( + f"Base `{short(args.base_sha)}` → head `{short(args.head_sha)}`. " + f"Medians of {base_runs} base {plural(base_runs, 'run')} and " + f"{head_runs} head {plural(head_runs, 'run')}, interleaved on the same runner." + ) + lines.append("") + lines.append( + f"Driver-side microbenchmarks only; server and network time are excluded. " + f"Shared-runner noise is about ±{args.threshold:.0f}%, so only larger regressions are flagged." + ) + lines.append("") + lines.append(f"JDK {head_meta.get('javaVersion', '?')}, {head_meta.get('osName', '?')} " + f"{head_meta.get('osArch', '?')}, {head_meta.get('availableProcessors', '?')} CPUs.") + lines.append("") + + lines.append("### Throughput") + lines.append("") + lines.append("| Workload | Base | Head | Δ |") + lines.append("|---|---:|---:|---:|") + for name, label, kind in WORKLOADS: + if name not in head_metrics: + continue + head_value = head_metrics[name]["rowsPerSec" if kind == "rows" else "opsPerSec"] + if base_metrics and name in base_metrics: + base_value = base_metrics[name]["rowsPerSec" if kind == "rows" else "opsPerSec"] + base_text = fmt_rate(base_value, kind) + delta = fmt_delta(pct(base_value, head_value), False, args.threshold) + else: + base_text = "—" + delta = "n/a" + lines.append(f"| {label} | {base_text} | {fmt_rate(head_value, kind)} | {delta} |") + lines.append("") + + lines.append("### Allocation") + lines.append("") + lines.append("| Workload | Base | Head | Δ |") + lines.append("|---|---:|---:|---:|") + for name, label, _ in WORKLOADS: + if name not in head_metrics: + continue + head_value = head_metrics[name]["bytesPerOp"] + if base_metrics and name in base_metrics: + base_value = base_metrics[name]["bytesPerOp"] + base_text = fmt_bytes(base_value) + delta = fmt_delta(pct(base_value, head_value), True, args.threshold) + else: + base_text = "—" + delta = "n/a" + lines.append(f"| {label} | {base_text} | {fmt_bytes(head_value)} | {delta} |") + lines.append("") + lines.append( + "Workloads and thresholds live in `tools/perf/render-comment.py`. " + "Run locally with `./mvnw -pl libsql-perf -am -DskipTests package` and see " + "[`libsql-perf/README.md`](libsql-perf/README.md)." + ) + return "\n".join(lines) + "\n" + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + comment = render(args) + if args.out: + args.out.write_text(comment) + else: + sys.stdout.write(comment) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:]))