From aa9cf86b07004d6c8d40a431ac13bbf30507ebd2 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 23 Oct 2026 20:19:35 +0000 Subject: [PATCH 01/19] Share configurable release asset checks and snapshot publication --- .github/buildcheck/releaseassets.py | 192 +++++++++++ .github/buildcheck/snapshot.py | 124 +++++++ .../buildcheck/tests/test_releaseassets.py | 305 ++++++++++++++++++ .github/buildcheck/tests/test_snapshot.py | 205 ++++++++++++ .github/collect-release-assets.py | 49 +++ .github/release-assets.json | 3 + .github/release-assets.json.license | 3 + .github/report-asset-signatures.py | 33 ++ .github/shared-files.sha256 | 12 +- .github/update-snapshot.py | 32 ++ .github/workflows/publish.yml | 150 ++------- CLAUDE.md | 19 +- 12 files changed, 984 insertions(+), 143 deletions(-) create mode 100644 .github/buildcheck/releaseassets.py create mode 100644 .github/buildcheck/snapshot.py create mode 100644 .github/buildcheck/tests/test_releaseassets.py create mode 100644 .github/buildcheck/tests/test_snapshot.py create mode 100644 .github/collect-release-assets.py create mode 100644 .github/release-assets.json create mode 100644 .github/release-assets.json.license create mode 100644 .github/report-asset-signatures.py create mode 100644 .github/update-snapshot.py diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py new file mode 100644 index 0000000..36d36fe --- /dev/null +++ b/.github/buildcheck/releaseassets.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Collect signed Maven release assets and explicitly configured extra files. + +No repository names or native/Android layouts live here. See the workspace policy +policies/release-asset-scripts.md for the JSON contract and staging options. +""" + +import csv +import json +import re +import shutil +import xml.etree.ElementTree as ET +from pathlib import Path + + +def _keys(value, allowed, context): + if not isinstance(value, dict) or set(value) - set(allowed): + raise ValueError(f"invalid {context}; allowed keys: {', '.join(allowed)}") + + +def _inside(root, relative): + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ValueError(f"expected a repository-relative path: {relative!r}") + path = (root / relative).resolve() + if not path.is_relative_to(root): + raise ValueError(f"path escapes repository: {relative}") + return path + + +def configuration(root, path=None): + """Read the explicit contract; unknown keys fail instead of silently dropping assets.""" + path = Path(path) if path is not None else Path(root) / ".github/release-assets.json" + config = json.loads(path.read_text()) + _keys(config, ("schema", "maven", "files"), "release asset configuration") + if type(config.get("schema")) is not int or config["schema"] != 1: + raise ValueError("release asset configuration requires schema: 1") + maven = config.get("maven", {}) + _keys(maven, ("standalone", "cyclonedx"), "maven configuration") + if not isinstance(maven.get("standalone", []), list): + raise ValueError("maven.standalone must be a list of module paths") + if type(maven.get("cyclonedx", False)) is not bool: + raise ValueError("maven.cyclonedx must be a boolean") + if not isinstance(config.get("files", []), list): + raise ValueError("files must be a list") + return config + + +def projects(root, standalone=()): + """Discover the reactor recursively, then optional separately published Maven projects.""" + root = Path(root).resolve() + seen, result = set(), [] + + def visit(folder, reactor=True): + folder = folder.resolve() + if not folder.is_relative_to(root) or folder in seen: + raise ValueError(f"invalid or repeated Maven module: {folder}") + seen.add(folder) + pom = ET.parse(folder / "pom.xml").getroot() + artifact = pom.findtext("{*}artifactId") + version = pom.findtext("{*}version") or pom.findtext("{*}parent/{*}version") + packaging = pom.findtext("{*}packaging", "jar") + if (not artifact or not version or not re.fullmatch(r"[\w.-]+", artifact) + or not re.fullmatch(r"[\w.+-]+", version) + or packaging not in ("pom", "jar", "maven-plugin")): + raise ValueError(f"unsupported release coordinates in {folder / 'pom.xml'}") + if result and version != result[0][2]: + raise ValueError(f"release version mismatch in {folder / 'pom.xml'}: {version}") + result.append((folder, artifact, version, packaging, reactor)) + for module in pom.findall("{*}modules/{*}module"): + visit(_inside(root, str(folder.relative_to(root) / (module.text or "").strip())), reactor) + + visit(root) + for module in standalone: + visit(_inside(root, module), reactor=False) + return result + + +def _classifiers(root, spec): + if isinstance(spec, dict): + _keys(spec, ("csv", "column"), "classifier CSV") + if not isinstance(spec.get("column"), str) or not spec["column"]: + raise ValueError("classifier CSV requires a column") + with _inside(root, spec.get("csv")).open() as stream: + rows = csv.DictReader(line for line in stream if line.strip() and not line.lstrip().startswith("#")) + if spec["column"] not in (rows.fieldnames or []): + raise ValueError(f"missing classifier CSV column: {spec['column']}") + spec = [row[spec["column"]] for row in rows] + if (not isinstance(spec, list) or not spec or any(not isinstance(c, str) + or not re.fullmatch(r"[\w.-]*", c) for c in spec) or len(set(spec)) != len(spec)): + raise ValueError("classifiers must be a nonempty list of unique classifier names") + return spec + + +def _format(template, version, classifier): + if not isinstance(template, str) or not template: + raise ValueError("file source/name must be a nonempty string") + try: + return template.format(version=version, classifier=classifier) + except (KeyError, ValueError, IndexError, AttributeError) as error: + raise ValueError(f"invalid release filename template: {template}") from error + + +def file_classifiers(root, index, config_path=None): + """Let packaging loops consume the same classifier list the collector checks.""" + if index < 0: + raise ValueError("file rule index must not be negative") + config = configuration(root, config_path) + try: + item = config.get("files", [])[index] + except IndexError as error: + raise ValueError(f"no file rule at index {index}") from error + _keys(item, ("source", "name", "classifiers"), "file rule") + return _classifiers(Path(root).resolve(), item.get("classifiers", [""])) + + +def collect(root, destination, config_path=None, *, maven_only=False, maven_assets=None): + """Copy available outputs, returning all missing files/signatures. Never hide partial output. + + maven_only stages deploy-signed files before later packaging overwrites target/. + maven_assets reads those staged files by their final names during the final collection. + """ + root, destination = Path(root).resolve(), Path(destination).resolve() + config = configuration(root, config_path) + modules = projects(root, config.get("maven", {}).get("standalone", [])) + version, expected, errors = modules[0][2], [], [] + for folder, artifact, _, packaging, reactor in modules: + base = f"{artifact}-{version}" + outputs = [(folder / "target" / f"{base}.pom", f"{base}.pom")] + if packaging != "pom": + for classifier in ("", "-sources", "-javadoc"): + name = f"{base}{classifier}.jar" + outputs.append((folder / "target" / name, name)) + if reactor and config.get("maven", {}).get("cyclonedx", False): + outputs.append((folder / "target/bom.json", f"{base}-cyclonedx.json")) + for source, name in outputs: + expected.append((Path(maven_assets) / name if maven_assets is not None else source, name)) + + for item in [] if maven_only else config.get("files", []): + _keys(item, ("source", "name", "classifiers"), "file rule") + for classifier in _classifiers(root, item.get("classifiers", [""])): + pattern = _format(item.get("source"), version, classifier) + _inside(root, pattern) + matches = sorted(root.glob(pattern)) + if len(matches) != 1: + errors.append(f"expected one release output matching {pattern}, found {len(matches)}") + continue + source = matches[0] + _inside(root, str(source.relative_to(root))) + name = _format(item["name"], version, classifier) if "name" in item else source.name + expected.append((source, name)) + + names = set() + for source, name in expected: + if (Path(name).name != name or name in (".", "..") or name.endswith(".asc") + or name in names or name + ".asc" in names): + raise ValueError(f"invalid or duplicate release asset filename: {name}") + names.update((name, name + ".asc")) + if destination == source.parent.resolve(): + raise ValueError("asset destination overlaps release inputs") + destination.mkdir(parents=True, exist_ok=True) + if any(destination.iterdir()): + raise ValueError(f"asset destination must be empty: {destination}") + for source, name in expected: + for suffix in ("", ".asc"): + path = Path(str(source) + suffix) + if not path.is_file() or path.stat().st_size == 0: + errors.append(f"missing or empty release output: {path}") + continue + shutil.copyfile(path, destination / (name + suffix)) + return errors + + +def signature_gaps(directory): + """Presence/size check for every file type; signing itself verifies cryptographic validity.""" + directory = Path(directory) + files = sorted(directory.iterdir()) if directory.is_dir() else [] + artifacts = [path for path in files if not path.name.endswith(".asc")] + if not artifacts: + return -1, [f"no artifacts in {directory} -- the collection step produced nothing"] + errors = [] + for path in artifacts: + signature = Path(str(path) + ".asc") + if not path.is_file() or path.stat().st_size == 0: + errors.append(f"missing or empty artifact: {path.name}") + elif not signature.is_file() or signature.stat().st_size == 0: + errors.append(f"unsigned: {path.name} has no nonempty detached .asc") + for path in files: + if path.name.endswith(".asc") and not path.with_suffix("").is_file(): + errors.append(f"orphan signature: {path.name}") + return len(errors), errors diff --git a/.github/buildcheck/snapshot.py b/.github/buildcheck/snapshot.py new file mode 100644 index 0000000..0c13dbb --- /dev/null +++ b/.github/buildcheck/snapshot.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Update the rolling snapshot without replacing a newer publish or deleting the release. + +The workflow serializes this entire operation with a job concurrency group. The ancestry/run +checks also reject an older build which finishes after a newer build has already published. +Available artifacts are uploaded even after an incomplete collection; only complete uploads +advance the source tag/notes and prune obsolete assets. +""" + +import json +import re +import subprocess +from pathlib import Path + +from .releaseassets import signature_gaps + + +class GitHub: + def command(self, *args): + subprocess.run(["gh", *map(str, args)], check=True) + + def api(self, method, endpoint, payload=None, optional=False): + args = ["gh", "api", "--method", method, endpoint] + if payload is not None: + args += ["--input", "-"] + result = subprocess.run(args, input=json.dumps(payload) if payload is not None else None, + text=True, capture_output=True) + if result.returncode: + if optional and "HTTP 404" in result.stderr: + return None + raise RuntimeError(f"GitHub {method} {endpoint}: {result.stderr.strip()}") + return json.loads(result.stdout) if result.stdout.strip() else None + + +def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, + *, tag="snapshot", title="Snapshot (latest)"): + """Return False for a superseded run; otherwise upload and update the rolling release.""" + if not re.fullmatch(r"[\w.-]+/[\w.-]+", repository) or not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ValueError("invalid repository or published commit") + if not re.fullmatch(r"[\w][\w.-]*", tag) or ".." in tag or tag.endswith((".", ".lock")): + raise ValueError("invalid rolling release tag") + run_id, attempt = int(run_id), int(attempt) + if run_id < 1 or attempt < 1: + raise ValueError("run ID and attempt must be positive") + files = sorted(Path(assets_dir).iterdir()) + if not files or any(not path.is_file() for path in files): + raise ValueError("refusing to update snapshot without a nonempty, flat asset directory") + artifacts = [path for path in files if not path.name.endswith(".asc")] + if not artifacts: + raise ValueError("refusing to update snapshot with signatures but no artifacts") + names = {path.name for path in files} + complete = complete and signature_gaps(assets_dir)[0] == 0 + gh = github or GitHub() + repo = f"repos/{repository}" + ref = gh.api("GET", f"{repo}/git/ref/tags/{tag}", optional=True) + release = gh.api("GET", f"{repo}/releases/tags/{tag}", optional=True) + published = ref["object"] if ref else None + # Annotated tags point to tag objects, not directly to commits. + for _ in range(8): + if not published or published["type"] != "tag": + break + published = gh.api("GET", f"{repo}/git/tags/{published['sha']}")["object"] + if published and published["type"] != "commit": + raise ValueError("snapshot tag does not resolve to a commit") + if published and published["sha"] != sha: + comparison = gh.api("GET", f"{repo}/compare/{published['sha']}...{sha}") + if comparison["status"] == "behind": + print("Snapshot already names a newer commit; skipping this older build.") + return False + if comparison["status"] not in ("ahead", "identical"): + raise ValueError("refusing to replace snapshot with a divergent commit") + if release: + marker = re.search(r"", + release.get("body") or "") + if marker and marker[3] == sha and (int(marker[1]), int(marker[2])) > (run_id, attempt): + print("Snapshot already names a newer run/attempt of this commit; skipping.") + return False + + body = (f"Snapshot from [{sha[:8]}](https://github.com/{repository}/commit/{sha}).\n\n" + f"[Publish run {run_id}, attempt {attempt}]" + f"(https://github.com/{repository}/actions/runs/{run_id}/attempts/{attempt}).\n\n" + f"") + if not release: + if not ref: + ref = gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) + release = gh.api("POST", f"{repo}/releases", { + "tag_name": tag, "target_commitish": sha, "name": title, + "prerelease": True, "body": body if complete else body + "\n\nIncomplete artifact collection." + }) + gh.command("release", "upload", tag, *files, "--repo", repository, "--clobber") + if not complete: + raise ValueError("available files uploaded, but collection is incomplete; " + "existing snapshot metadata and assets were retained") + + if ref: + gh.api("PATCH", f"{repo}/git/refs/tags/{tag}", {"sha": sha, "force": True}) + else: + gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) + gh.api("PATCH", f"{repo}/releases/{release['id']}", { + "target_commitish": sha, "name": title, "prerelease": True, "body": body + }) + + # Read every page before deleting: deleting while paginating could skip shifted entries. + assets = [] + page = 1 + while True: + batch = gh.api("GET", f"{repo}/releases/{release['id']}/assets?per_page=100&page={page}") + assets.extend(batch) + if len(batch) < 100: + break + page += 1 + failures = [] + for asset in assets: + if asset["name"] not in names: + try: + print(f"Removing obsolete snapshot asset: {asset['name']}") + gh.api("DELETE", f"{repo}/releases/assets/{asset['id']}", optional=True) + except RuntimeError as error: + failures.append(str(error)) + if failures: + raise RuntimeError("; ".join(failures)) + return True diff --git a/.github/buildcheck/tests/test_releaseassets.py b/.github/buildcheck/tests/test_releaseassets.py new file mode 100644 index 0000000..a4ed72a --- /dev/null +++ b/.github/buildcheck/tests/test_releaseassets.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from buildcheck.releaseassets import collect, configuration, file_classifiers, projects, signature_gaps +from buildcheck import workflow +from buildcheck.tests.helpers import REPO + + +VERSION = "2.0.0-SNAPSHOT" + + +class ReleaseAssetsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.output = self.root / "assets" + self.config = {"schema": 1, "maven": {"standalone": ["tool"], "cyclonedx": True}, "files": [ + {"source": "app/target/app-{version}-{classifier}.jar", + "classifiers": {"csv": ".github/classifiers.csv", "column": "classifier"}}, + {"source": "mobile/publication/pom.xml", "name": "mobile-{version}.pom"}, + {"source": "mobile/build/mobile-{version}.aar"}, + {"source": "archive/upstream-*-{version}.tar.gz"}]} + self.save_config() + self.write("pom.xml", f""" +parent{VERSION}pom +appplugin""") + self.outputs(".", "parent", "pom") + self.module("app", "jar") + self.module("plugin", "maven-plugin") + self.module("tool", "jar", reactor=False) + self.write(".github/classifiers.csv", "# comment\nclassifier\ncpu-linux\ngpu-linux\n") + for classifier in ("cpu-linux", "gpu-linux"): + self.asset(f"app/target/app-{VERSION}-{classifier}.jar") + self.asset("mobile/publication/pom.xml") + self.asset(f"mobile/build/mobile-{VERSION}.aar") + self.asset(f"archive/upstream-build1-{VERSION}.tar.gz") + + def write(self, name, content="artifact"): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + def save_config(self): + self.write(".github/release-assets.json", json.dumps(self.config)) + + def asset(self, name): + self.write(name) + self.write(name + ".asc", "signature of " + name) + + def outputs(self, folder, artifact, packaging, reactor=True): + self.asset(f"{folder}/target/{artifact}-{VERSION}.pom") + if packaging != "pom": + for classifier in ("", "-sources", "-javadoc"): + self.asset(f"{folder}/target/{artifact}-{VERSION}{classifier}.jar") + if reactor: + self.asset(f"{folder}/target/bom.json") + + def module(self, name, packaging, reactor=True): + self.write(f"{name}/pom.xml", f""" +{VERSION}{name} +{packaging}""") + self.outputs(name, name, packaging, reactor) + + def test_reactor_plugin_standalone_and_bom_names_match_central(self): + self.assertEqual(collect(self.root, self.output), []) + for artifact in ("app", "plugin", "tool"): + for classifier in ("", "-sources", "-javadoc"): + self.assertTrue((self.output / f"{artifact}-{VERSION}{classifier}.jar.asc").is_file()) + for artifact in ("parent", "app", "plugin"): + self.assertTrue((self.output / f"{artifact}-{VERSION}-cyclonedx.json.asc").is_file()) + self.assertFalse((self.output / "bom.json").exists()) + self.assertFalse((self.output / f"tool-{VERSION}-cyclonedx.json").exists()) + self.assertTrue((self.output / f"mobile-{VERSION}.pom.asc").is_file()) + + def test_new_reactor_module_is_automatic(self): + pom = self.root / "pom.xml" + pom.write_text(pom.read_text().replace("", "new")) + self.module("new", "jar") + self.assertEqual(collect(self.root, self.output), []) + self.assertTrue((self.output / f"new-{VERSION}.jar").is_file()) + + def test_single_library_needs_no_native_android_or_bom_configuration(self): + self.config = {"schema": 1} + self.save_config() + self.write("pom.xml", f"library{VERSION}") + self.outputs(".", "library", "jar") + self.assertEqual(collect(self.root, self.output), []) + self.assertEqual(len(list(self.output.iterdir())), 8) + + def test_missing_artifacts_signatures_boms_and_extras_preserve_available_files(self): + missing = ["app/target/bom.json", f"plugin/target/plugin-{VERSION}.pom.asc", + f"app/target/app-{VERSION}-gpu-linux.jar", f"mobile/build/mobile-{VERSION}.aar"] + for name in missing: + (self.root / name).unlink() + errors = collect(self.root, self.output) + for name in missing: + self.assertTrue(any(name in error for error in errors), errors) + self.assertTrue((self.output / f"app-{VERSION}.jar").is_file()) + + def test_empty_signature_fails(self): + self.write(f"app/target/app-{VERSION}.jar.asc", "") + self.assertTrue(any("missing or empty" in e for e in collect(self.root, self.output))) + + def test_stale_versions_unconfigured_fat_jars_and_plugin_dependencies_are_ignored(self): + for name in ("app/target/app-1.0.0.jar", f"app/target/app-{VERSION}-jar-with-dependencies.jar", + "app/target/spotbugs/third-party.jar"): + self.asset(name) + self.assertEqual(collect(self.root, self.output), []) + self.assertFalse(any("1.0.0" in p.name or "dependencies" in p.name or "third-party" in p.name + for p in self.output.iterdir())) + + def test_nonempty_destination_and_version_mismatch_fail(self): + self.write("assets/stale.jar") + with self.assertRaisesRegex(ValueError, "must be empty"): + collect(self.root, self.output) + (self.output / "stale.jar").unlink() + pom = self.root / "plugin/pom.xml" + pom.write_text(pom.read_text().replace(VERSION, "1.0.0")) + with self.assertRaisesRegex(ValueError, "version mismatch"): + collect(self.root, self.output) + + def test_ambiguous_archive_is_reported(self): + self.asset(f"archive/upstream-build2-{VERSION}.tar.gz") + self.assertTrue(any("found 2" in error for error in collect(self.root, self.output))) + + def test_staged_maven_files_survive_rebuild_and_missing_stage_file_fails_final_collection(self): + stage = self.root / "central-assets" + self.assertEqual(collect(self.root, stage, maven_only=True), []) + original = (stage / f"app-{VERSION}.jar").read_bytes() + self.write(f"app/target/app-{VERSION}.jar", "rebuilt unsigned jar") + self.assertEqual(collect(self.root, self.output, maven_assets=stage), []) + self.assertEqual((self.output / f"app-{VERSION}.jar").read_bytes(), original) + (stage / f"app-{VERSION}-javadoc.jar").unlink() + errors = collect(self.root, self.root / "partial", maven_assets=stage) + self.assertTrue(any("javadoc.jar" in e for e in errors)) + + def test_typo_or_wrong_schema_fails(self): + for config in ({"schema": 2}, {"schema": 1, "filez": []}, + {"schema": 1, "maven": {"cyclonedx": "yes"}}): + with self.subTest(config=config): + self.config = config + self.save_config() + with self.assertRaises(ValueError): + collect(self.root, self.output) + + def test_duplicate_names_traversal_bad_templates_and_empty_classifier_list_fail(self): + for rule in ({"source": "mobile/publication/pom.xml", "name": f"app-{VERSION}.jar"}, + {"source": "../outside.jar"}, {"source": "mobile/{unknown}.jar"}, + {"source": "mobile/build/*.aar", "classifiers": []}, + {"source": "mobile/build/*.aar", "name": "../outside.jar"}): + with self.subTest(rule=rule): + self.config["files"] = [rule] + self.save_config() + with self.assertRaises(ValueError): + collect(self.root, self.output) + self.assertFalse(self.output.exists()) + + def test_repeated_or_outside_module_fails(self): + for module in ("app", "../../outside"): + self.write("pom.xml", f"""parent{VERSION} +app{module}""") + with self.assertRaises(ValueError): + projects(self.root) + + def test_classifier_list_is_shared_with_packaging_loop(self): + self.assertEqual(file_classifiers(self.root, 0), ["cpu-linux", "gpu-linux"]) + self.config["files"][0]["classifiers"] = ["gpu-one", "gpu-two"] + self.save_config() + self.assertEqual(file_classifiers(self.root, 0), ["gpu-one", "gpu-two"]) + with self.assertRaisesRegex(ValueError, "no file rule"): + file_classifiers(self.root, 99) + with self.assertRaisesRegex(ValueError, "must not be negative"): + file_classifiers(self.root, -1) + + def test_current_repository_workflow_passes_collection_result_and_checks_both_uploads(self): + jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + for kind in ("snapshot", "release"): + job = jobs[f"publish-{kind}"].text + self.assertIn(f"python3 .github/collect-release-assets.py signed-{kind}-assets", job) + self.assertIn("assets_complete: $" + "{{ steps.collect.outputs.complete }}", job) + self.assertIn(" id: collect", job) + attach = jobs["github-snapshot" if kind == "snapshot" else + "github-release" if "github-release" in jobs else "github-release-signed"].text + self.assertIn(f"python3 .github/report-asset-signatures.py {'snapshot' if kind == 'snapshot' else 'release'}-assets", attach) + self.assertLess(attach.index("id: signatures"), attach.index(" - name: Fail if anything")) + if configuration(REPO).get("maven", {}).get("cyclonedx", False): + for kind in ("snapshot", "release"): + self.assertIn(f"signed-{kind}-assets/*.json", jobs[f"publish-{kind}"].text) + + def test_current_repository_contract_collects_exact_complete_and_partial_fixture(self): + # Exercise this checkout's real POMs and JSON, not just a hand-written generic fixture. + repo = Path(REPO) + config = configuration(repo) + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + expected = set() + + def copy(relative): + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(repo / relative, target) + + def artifact(relative, name): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + for suffix in ("", ".asc"): + Path(str(path) + suffix).write_text("fixture " + str(relative) + suffix) + expected.add(name + suffix) + + copy(".github/release-assets.json") + modules = projects(repo, config.get("maven", {}).get("standalone", [])) + version = modules[0][2] + for folder, name, _, packaging, reactor in modules: + relative = folder.relative_to(repo) + copy(relative / "pom.xml") + base = f"{name}-{version}" + artifact(relative / "target" / f"{base}.pom", f"{base}.pom") + if packaging != "pom": + for classifier in ("", "-sources", "-javadoc"): + artifact(relative / "target" / f"{base}{classifier}.jar", f"{base}{classifier}.jar") + if reactor and config.get("maven", {}).get("cyclonedx", False): + artifact(relative / "target/bom.json", f"{base}-cyclonedx.json") + for index, rule in enumerate(config.get("files", [])): + if isinstance(rule.get("classifiers"), dict): + copy(rule["classifiers"]["csv"]) + for classifier in file_classifiers(repo, index): + source = rule["source"].format(version=version, classifier=classifier).replace("*", "fixture") + name = rule.get("name", Path(source).name).format(version=version, classifier=classifier) + artifact(source, name) + destination = root / "collected" + self.assertEqual(collect(root, destination), []) + self.assertEqual({p.name for p in destination.iterdir()}, expected) + (destination / next(n for n in expected if n.endswith(".asc"))).unlink() + self.assertNotEqual(signature_gaps(destination)[0], 0) + # An incomplete reactor must still produce an upload directory with available files. + first = modules[0][1] + "-" + version + ".pom" + (root / "target" / first).unlink() + self.assertTrue(collect(root, root / "partial")) + self.assertTrue(any((root / "partial").iterdir())) + + def test_cli_reports_completeness_even_on_failure_and_accepts_parameters(self): + scripts = self.root / ".github" + (scripts / "buildcheck").mkdir() + for name in ("collect-release-assets.py", "buildcheck/releaseassets.py", "buildcheck/__init__.py"): + shutil.copyfile(Path(REPO) / ".github" / name, scripts / name) + cli = [sys.executable, str(scripts / "collect-release-assets.py")] + listing = subprocess.run(cli + ["--list-classifiers", "0"], capture_output=True, text=True) + self.assertEqual(listing.returncode, 0, listing.stderr) + self.assertEqual(listing.stdout.splitlines(), ["cpu-linux", "gpu-linux"]) + for complete in (True, False): + if not complete: + (self.root / f"plugin/target/plugin-{VERSION}.jar").unlink() + output = self.root / f"github-output-{complete}" + destination = self.root / f"collected-{complete}" + result = subprocess.run(cli + [str(destination), "--config", str(scripts / "release-assets.json")], + env={**os.environ, "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0 if complete else 1, result.stdout + result.stderr) + self.assertEqual(output.read_text(), f"complete={str(complete).lower()}\n") + self.assertTrue((destination / f"app-{VERSION}.jar").is_file()) + + +class SignatureReportTest(unittest.TestCase): + def test_all_file_types_spaces_empty_orphan_and_missing_signatures(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + self.assertEqual(signature_gaps(root)[0], -1) + for name in ("file.jar", "mobile.aar", "parent.pom", "bom.json", "source.tar.gz", "with spaces.zip"): + (root / name).write_text("artifact") + (root / (name + ".asc")).write_text("signature") + self.assertEqual(signature_gaps(root), (0, [])) + (root / "bom.json.asc").write_text("") + (root / "source.tar.gz.asc").unlink() + (root / "orphan.jar.asc").write_text("signature") + (root / "file.jar").write_text("") + self.assertEqual(signature_gaps(root)[0], 4) + + def test_cli_always_returns_zero_and_writes_missing_for_post_upload_assertion(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + assets = root / "assets" + assets.mkdir() + (assets / "unsigned.json").write_text("bom") + output = root / "github-output" + result = subprocess.run([sys.executable, str(Path(REPO) / ".github/report-asset-signatures.py"), + str(assets)], env={**os.environ, "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("::error::unsigned", result.stdout) + self.assertEqual(output.read_text(), "missing=1\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py new file mode 100644 index 0000000..e14f357 --- /dev/null +++ b/.github/buildcheck/tests/test_snapshot.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 + +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from buildcheck.snapshot import GitHub, publish +from buildcheck import workflow +from buildcheck.tests.helpers import REPO + + +OLD, NEW = "a" * 40, "b" * 40 + + +class FakeGitHub: + def __init__(self): + self.ref = {"object": {"type": "commit", "sha": OLD}} + self.release = {"id": 7, "body": "Snapshot from old commit"} + self.comparison = "ahead" + self.pages = {1: [{"id": 1, "name": "old.jar"}, {"id": 2, "name": "new.json"}]} + self.calls = [] + self.fail_upload = False + self.fail_deletes = set() + self.tag = "snapshot" + + def api(self, method, endpoint, payload=None, optional=False): + self.calls.append((method, endpoint, payload)) + if method == "DELETE" and int(endpoint.rsplit("/", 1)[1]) in self.fail_deletes: + raise RuntimeError("delete failed") + if method == "GET": + if endpoint.endswith(f"git/ref/tags/{self.tag}"): + return self.ref + if endpoint.endswith(f"releases/tags/{self.tag}"): + return self.release + if "/compare/" in endpoint: + return {"status": self.comparison} + if "/git/tags/" in endpoint: + return {"object": {"type": "commit", "sha": OLD}} + if "/assets?" in endpoint: + return self.pages[int(endpoint.split("&page=")[1])] + if method == "POST" and endpoint.endswith("/releases"): + return {"id": 7} + if method == "POST" and endpoint.endswith("/git/refs"): + return {"object": {"type": "commit", "sha": payload["sha"]}} + return {} + + def command(self, *args): + self.calls.append(("upload", args, None)) + if self.fail_upload: + raise subprocess.CalledProcessError(1, "gh release upload") + + +class SnapshotTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.assets = Path(self.temp.name) + for name in ("new.json", "new.json.asc"): + (self.assets / name).write_text("artifact") + self.gh = FakeGitHub() + quiet = patch("builtins.print") + quiet.start() + self.addCleanup(quiet.stop) + + def publish(self, **kwargs): + return publish("owner/repo", NEW, kwargs.pop("run_id", 10), kwargs.pop("attempt", 1), + self.assets, kwargs.pop("complete", True), self.gh, **kwargs) + + def test_upload_precedes_tag_notes_and_prune_and_release_itself_survives(self): + self.assertTrue(self.publish()) + calls = self.gh.calls + upload = next(i for i, c in enumerate(calls) if c[0] == "upload") + tag = next(i for i, c in enumerate(calls) if c[0] == "PATCH" and "/git/refs/" in c[1]) + notes = next(c[2] for c in calls if c[0] == "PATCH" and c[1].endswith("/releases/7")) + self.assertLess(upload, tag) + self.assertIn(NEW, notes["body"]) + self.assertIn("/actions/runs/10/attempts/1", notes["body"]) + self.assertEqual(notes["target_commitish"], NEW) + deletes = [c[1] for c in calls if c[0] == "DELETE"] + self.assertEqual(deletes, ["repos/owner/repo/releases/assets/1"]) + + def test_incomplete_collection_uploads_available_files_without_promoting_or_pruning(self): + with self.assertRaisesRegex(ValueError, "collection is incomplete"): + self.publish(complete=False) + self.assertTrue(any(c[0] == "upload" for c in self.gh.calls)) + self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + + def test_missing_signature_also_prevents_promotion(self): + (self.assets / "new.json.asc").unlink() + with self.assertRaisesRegex(ValueError, "collection is incomplete"): + self.publish() + self.assertFalse(any(c[0] == "PATCH" for c in self.gh.calls)) + + def test_late_finishing_older_commit_cannot_overwrite_newer_snapshot(self): + self.gh.comparison = "behind" + self.assertFalse(self.publish()) + self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) + + def test_empty_signature_or_orphan_also_prevents_promotion(self): + for name in ("new.json.asc", "orphan.jar.asc"): + with self.subTest(name=name): + (self.assets / name).write_text("") + with self.assertRaisesRegex(ValueError, "collection is incomplete"): + self.publish() + self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + + def test_rolling_tag_and_title_are_parameters(self): + self.gh.tag = "nightly" + self.assertTrue(self.publish(tag="nightly", title="Nightly build")) + upload = next(c for c in self.gh.calls if c[0] == "upload") + self.assertEqual(upload[1][2], "nightly") + notes = next(c[2] for c in self.gh.calls if c[0] == "PATCH" and c[1].endswith("/releases/7")) + self.assertEqual(notes["name"], "Nightly build") + + def test_same_commit_older_run_or_attempt_is_skipped(self): + self.gh.ref["object"]["sha"] = NEW + for run, attempt in ((11, 1), (10, 2)): + self.gh.release["body"] = f"" + self.gh.calls.clear() + self.assertFalse(self.publish()) + self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) + + def test_new_attempt_of_same_commit_can_replace_old_artifacts(self): + self.gh.ref["object"]["sha"] = NEW + self.gh.release["body"] = f"" + self.assertTrue(self.publish(attempt=2)) + + def test_annotated_tags_are_resolved_before_comparing_commits(self): + self.gh.ref["object"]["type"] = "tag" + self.assertTrue(self.publish()) + self.assertTrue(any("/git/tags/" in c[1] for c in self.gh.calls if c[0] == "GET")) + + def test_creates_missing_tag_and_release_once(self): + self.gh.ref, self.gh.release = None, None + self.assertTrue(self.publish()) + self.assertEqual(sum(c[0] == "POST" and c[1].endswith("/git/refs") for c in self.gh.calls), 1) + self.assertTrue(any(c[0] == "POST" and c[1].endswith("/releases") for c in self.gh.calls)) + + def test_reads_all_asset_pages_before_deleting(self): + self.gh.pages = {1: [{"id": i, "name": f"old-{i}.jar"} for i in range(100)], + 2: [{"id": 101, "name": "last-old.jar"}, {"id": 102, "name": "new.json"}]} + self.assertTrue(self.publish()) + calls = self.gh.calls + page2 = next(i for i, c in enumerate(calls) if c[0] == "GET" and "&page=2" in c[1]) + deletion = next(i for i, c in enumerate(calls) if c[0] == "DELETE") + self.assertLess(page2, deletion) + self.assertEqual(sum(c[0] == "DELETE" for c in calls), 101) + + def test_divergent_commit_and_empty_upload_are_refused(self): + self.gh.comparison = "diverged" + with self.assertRaisesRegex(ValueError, "divergent"): + self.publish() + self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) + for path in self.assets.iterdir(): + path.unlink() + self.gh.calls.clear() + with self.assertRaisesRegex(ValueError, "nonempty"): + self.publish() + self.assertEqual(self.gh.calls, []) + + def test_upload_failure_does_not_move_tag_or_prune(self): + self.gh.fail_upload = True + with self.assertRaises(subprocess.CalledProcessError): + self.publish() + self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + + def test_prune_tries_remaining_assets_after_a_deletion_failure(self): + self.gh.pages[1].append({"id": 3, "name": "another-old.jar"}) + self.gh.fail_deletes.add(1) + with self.assertRaisesRegex(RuntimeError, "delete failed"): + self.publish() + self.assertTrue(any(c[0] == "DELETE" and c[1].endswith("/assets/3") for c in self.gh.calls)) + + def test_signature_only_directory_does_not_mutate_snapshot(self): + (self.assets / "new.json").unlink() + with self.assertRaisesRegex(ValueError, "no artifacts"): + self.publish() + self.assertEqual(self.gh.calls, []) + + def test_workflow_serializes_entire_update_and_passes_collection_result(self): + jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + job = jobs["github-snapshot"].text + self.assertIn(" concurrency:\n group: snapshot-release-${{ github.repository }}", job) + self.assertIn(" cancel-in-progress: false", job) + self.assertIn("ASSETS_COMPLETE: ${{ needs.publish-snapshot.outputs.assets_complete }}", job) + self.assertIn("python3 .github/update-snapshot.py snapshot-assets", job) + + def test_optional_api_failure_accepts_only_404_not_authentication_failure(self): + gh = GitHub() + for status in (403, 404): + response = subprocess.CompletedProcess([], 1, "", f"gh: failure (HTTP {status})") + with patch("buildcheck.snapshot.subprocess.run", return_value=response): + if status == 404: + self.assertIsNone(gh.api("GET", "repos/owner/repo/releases/tags/snapshot", optional=True)) + else: + with self.assertRaisesRegex(RuntimeError, "403"): + gh.api("GET", "repos/owner/repo/releases/tags/snapshot", optional=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/collect-release-assets.py b/.github/collect-release-assets.py new file mode 100644 index 0000000..88de5b8 --- /dev/null +++ b/.github/collect-release-assets.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Collect assets using the repository's explicit release-assets.json contract.""" + +import argparse +import os +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +from buildcheck.releaseassets import collect, file_classifiers + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("destination", nargs="?") + parser.add_argument("--config", help="JSON contract (default: .github/release-assets.json)") + parser.add_argument("--maven-only", action="store_true", help="stage signed Maven files before a rebuild") + parser.add_argument("--maven-assets", help="read Maven files from a previous staging directory") + parser.add_argument("--list-classifiers", type=int, metavar="RULE", help="print classifiers of a file rule (zero based)") + args = parser.parse_args() + if args.list_classifiers is None and args.destination is None: + parser.error("destination is required unless --list-classifiers is used") + errors = [] + try: + root = Path(__file__).resolve().parent.parent + if args.list_classifiers is not None: + if args.list_classifiers < 0: + raise ValueError("file rule index must not be negative") + print("\n".join(file_classifiers(root, args.list_classifiers, args.config))) + return 0 + errors = collect(root, args.destination, args.config, + maven_only=args.maven_only, maven_assets=args.maven_assets) + except (ValueError, OSError, ET.ParseError) as error: + errors.append(str(error)) + for error in errors: + print(f"::error::{error}") + if not errors: + print(f"Collected {len(list(Path(args.destination).iterdir()))} release files including signatures.") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"complete={'false' if errors else 'true'}\n") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/release-assets.json b/.github/release-assets.json new file mode 100644 index 0000000..8d8244c --- /dev/null +++ b/.github/release-assets.json @@ -0,0 +1,3 @@ +{ + "schema": 1 +} diff --git a/.github/release-assets.json.license b/.github/release-assets.json.license new file mode 100644 index 0000000..eca5b04 --- /dev/null +++ b/.github/release-assets.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 Bernard Ladenthin + +SPDX-License-Identifier: MIT OR Apache-2.0 diff --git a/.github/report-asset-signatures.py b/.github/report-asset-signatures.py new file mode 100644 index 0000000..a8fbcef --- /dev/null +++ b/.github/report-asset-signatures.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Report signature gaps without blocking uploads; assert the missing output afterwards.""" + +import argparse +import os + +from buildcheck.releaseassets import signature_gaps + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory") + args = parser.parse_args() + try: + missing, errors = signature_gaps(args.directory) + except OSError as error: + missing, errors = -1, [str(error)] + for error in errors: + print(f"::error::{error}") + if not missing: + print("All collected artifacts have nonempty detached signatures.") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"missing={missing}\n") + # The workflow uploads first and fails on the output afterwards. + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 5f5ae67..94cb163 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -68,5 +68,13 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -c2684a3f93aaf552c13f845307536c2a14dda315c7d7b1e416bc66951b811dc1 .github/workflows/publish.yml#github-snapshot -c09a030706dbea4221851ca0fcbfa6fc686165b408724724ade61724da6e123b .github/workflows/publish.yml#github-release +7f96fc1b0cffa9844379162c2f82e5f26454ab0fdc7349019ddc05423b4320af .github/workflows/publish.yml#github-snapshot +68d41a4a97d26438f5649f9c6f24a884d1448192486b386271d49051c58f6ee1 .github/workflows/publish.yml#github-release +b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py +bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py +8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py +c0401d973100c4ab0ec7728ec5d3a841831863cd307112c88f8cfe172316316a .github/buildcheck/tests/test_snapshot.py +bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py +6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py +930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py +71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license diff --git a/.github/update-snapshot.py b/.github/update-snapshot.py new file mode 100644 index 0000000..03dcd45 --- /dev/null +++ b/.github/update-snapshot.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Usage: update-snapshot.py ; requires the serialized github-snapshot job.""" + +import argparse +import os +import subprocess +import sys + +from buildcheck.snapshot import publish + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory") + parser.add_argument("--tag", default="snapshot") + parser.add_argument("--title", default="Snapshot (latest)") + args = parser.parse_args() + try: + publish(os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_SHA"], + os.environ["GITHUB_RUN_ID"], os.environ["GITHUB_RUN_ATTEMPT"], args.directory, + os.environ.get("ASSETS_COMPLETE") == "true", tag=args.tag, title=args.title) + return 0 + except (ValueError, OSError, RuntimeError, KeyError, subprocess.CalledProcessError) as error: + print(f"::error::{error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c5e7d9a..29e6e0d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -403,6 +403,8 @@ jobs: if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central + outputs: + assets_complete: ${{ steps.collect.outputs.complete }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 @@ -445,14 +447,9 @@ jobs: # the signed jars and poms + .asc files already exist in target/ (signing happens at # verify). Collecting on failure lets the github-snapshot job still attach them. - name: Collect signed artifacts + id: collect if: ${{ !cancelled() }} - run: | - mkdir -p signed-snapshot-assets - cp target/*.jar signed-snapshot-assets/ 2>/dev/null || true - cp target/*.jar.asc signed-snapshot-assets/ 2>/dev/null || true - # maven-gpg writes target/-.pom next to its .pom.asc; Central has both. - cp target/*.pom signed-snapshot-assets/ 2>/dev/null || true - cp target/*.pom.asc signed-snapshot-assets/ 2>/dev/null || true + run: python3 .github/collect-release-assets.py signed-snapshot-assets - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 if: ${{ !cancelled() }} with: @@ -467,9 +464,14 @@ jobs: # the GitHub pre-release assets must not be lost in that case. if: ${{ !cancelled() && (needs.publish-snapshot.result == 'success' || needs.publish-snapshot.result == 'failure') }} runs-on: ubuntu-latest + # Serialize the complete upload, tag/notes update and pruning operation. + concurrency: + group: snapshot-release-${{ github.repository }} + cancel-in-progress: false permissions: contents: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets @@ -483,104 +485,12 @@ jobs: # still loudly red rather than quietly wrong. # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". id: signatures - run: | - set -uo pipefail - dir="snapshot-assets" - # Every attachable artifact type, not only jars: with '*.jar' alone the AARs were attached - # unsigned for months and nothing counted them, so the job reported success. - artifacts=$(find "$dir" -maxdepth 1 \( -name '*.jar' -o -name '*.aar' -o -name '*.pom' \) | sort) - if [ -z "$artifacts" ]; then - echo "::error::no artifacts in $dir -- the collection step produced nothing" - echo "missing=-1" >> "$GITHUB_OUTPUT" - exit 0 - fi - missing=0 - for artifact in $artifacts; do - if [ ! -e "$artifact.asc" ]; then - echo "::error::unsigned: $(basename "$artifact") has no detached .asc" - missing=$((missing + 1)) - fi - done - echo "missing=$missing" >> "$GITHUB_OUTPUT" - [ "$missing" -eq 0 ] && echo "all $(echo "$artifacts" | wc -l) artifact(s) signed" - exit 0 - - name: Update snapshot pre-release + run: python3 .github/report-asset-signatures.py snapshot-assets + - name: Update snapshot assets, source tag and notes env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release view snapshot --repo ${{ github.repository }} 2>/dev/null \ - || gh release create snapshot \ - --repo ${{ github.repository }} \ - --prerelease \ - --title "Snapshot (latest)" \ - --notes "Latest snapshot build from the main branch." - gh release upload snapshot snapshot-assets/* \ - --repo ${{ github.repository }} \ - --clobber - - name: Remove the assets this run did not upload - # `snapshot` is a rolling "latest" pre-release, and `gh release upload --clobber` replaces - # same-named assets but never removes any other, so every asset of every past version stayed - # forever: 208 on java-llama.cpp (109 of them from 5.0.x, 43 in shapes that no longer exist), - # and SpotBugs plugin jars an old collect step attached in 2026-05 on three repositories. A - # non-current asset on this release is noise by definition. The keep-set is what this run - # uploaded -- the files in the asset directory -- never a version pattern, which would have to - # be re-taught at every classifier added or dropped (how the stale shapes arose). AFTER the - # upload on purpose: a failed upload must not leave the release emptier than it was. And only - # here: a versioned v* release is history, github-release never deletes anything. Every - # deletion is printed by name, so the job log is the audit trail. - # - # Asset by asset, never the release. The first form of this job (2026-05-07) deleted the whole - # `snapshot` release with its tag and recreated it; on 2026-05-10 the runs of several main - # pushes minutes apart deleted it under each other, and every job of each burst but the last - # failed in that step with its artifacts present. The job then went with the Central - # migration and came back as `view || create` + `upload --clobber` without any cleanup, which - # is how the backlog grew (workspace/policies/fat-jar-release-assets.md has the runs). Hence - # three guards: an empty upload is refused (every asset would count as stale; the download - # and upload steps above fail first, the guard keeps it so), an asset that is already gone - # counts as removed (two overlapping dispatches may both pick it), and a failed deletion is - # reported and the rest still tried -- the assets of this upload are on the release either - # way, so this is "attach first, then red" like the signature check. - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - keep="$(ls -1 snapshot-assets | sort)" - if [ -z "$keep" ]; then - echo "::error::refusing to prune: this run uploaded nothing, so every asset of the release would count as stale" - exit 1 - fi - list_assets() { - gh release view snapshot --repo ${{ github.repository }} --json assets --jq '.assets[].name' | sort - } - current="$(list_assets)" - stale="$(comm -13 <(printf '%s\n' "$keep") <(printf '%s\n' "$current"))" - if [ -z "$stale" ]; then - echo "nothing to remove: the release holds exactly the $(printf '%s\n' "$keep" | wc -l) asset(s) of this upload" - exit 0 - fi - echo "removing $(printf '%s\n' "$stale" | wc -l) asset(s) that are not part of this upload:" - failed=0 - while IFS= read -r name; do - [ -n "$name" ] || continue - echo " deleting $name" - if gh release delete-asset snapshot "$name" --repo ${{ github.repository }} --yes; then - continue - fi - if ! remaining="$(list_assets)"; then - echo "::error::could not delete $name, and could not re-read the release to tell whether it is gone" - failed=$((failed + 1)) - elif printf '%s\n' "$remaining" | grep -qxF -- "$name"; then - echo "::error::could not delete $name" - failed=$((failed + 1)) - else - echo " $name was already gone" - fi - done <<< "$stale" - if [ "$failed" -ne 0 ]; then - echo "::error::$failed asset(s) could not be removed; the $(printf '%s\n' "$keep" | wc -l) asset(s) of this upload are on the release" - exit 1 - fi - echo "kept the $(printf '%s\n' "$keep" | wc -l) asset(s) of this upload" + ASSETS_COMPLETE: ${{ needs.publish-snapshot.outputs.assets_complete }} + run: python3 .github/update-snapshot.py snapshot-assets - name: Fail if anything was attached unsigned # After the upload on purpose: the assets must exist even when the signature does not. if: ${{ always() && steps.signatures.outputs.missing != '0' }} @@ -594,6 +504,8 @@ jobs: if: needs.check-tag.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central + outputs: + assets_complete: ${{ steps.collect.outputs.complete }} permissions: contents: write steps: @@ -625,14 +537,9 @@ jobs: # the signed jars and poms + .asc files already exist in target/ (signing happens at # verify). Collecting on failure lets the github-release job still attach them. - name: Collect signed artifacts + id: collect if: ${{ !cancelled() }} - run: | - mkdir -p signed-release-assets - cp target/*.jar signed-release-assets/ 2>/dev/null || true - cp target/*.jar.asc signed-release-assets/ 2>/dev/null || true - # maven-gpg writes target/-.pom next to its .pom.asc; Central has both. - cp target/*.pom signed-release-assets/ 2>/dev/null || true - cp target/*.pom.asc signed-release-assets/ 2>/dev/null || true + run: python3 .github/collect-release-assets.py signed-release-assets - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 if: ${{ !cancelled() }} with: @@ -650,6 +557,7 @@ jobs: permissions: contents: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets @@ -663,27 +571,7 @@ jobs: # still loudly red rather than quietly wrong. # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". id: signatures - run: | - set -uo pipefail - dir="release-assets" - # Every attachable artifact type, not only jars: with '*.jar' alone the AARs were attached - # unsigned for months and nothing counted them, so the job reported success. - artifacts=$(find "$dir" -maxdepth 1 \( -name '*.jar' -o -name '*.aar' -o -name '*.pom' \) | sort) - if [ -z "$artifacts" ]; then - echo "::error::no artifacts in $dir -- the collection step produced nothing" - echo "missing=-1" >> "$GITHUB_OUTPUT" - exit 0 - fi - missing=0 - for artifact in $artifacts; do - if [ ! -e "$artifact.asc" ]; then - echo "::error::unsigned: $(basename "$artifact") has no detached .asc" - missing=$((missing + 1)) - fi - done - echo "missing=$missing" >> "$GITHUB_OUTPUT" - [ "$missing" -eq 0 ] && echo "all $(echo "$artifacts" | wc -l) artifact(s) signed" - exit 0 + run: python3 .github/report-asset-signatures.py release-assets - name: Upload release assets uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3 with: diff --git a/CLAUDE.md b/CLAUDE.md index 020566e..9b138ea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -263,16 +263,15 @@ enforcer default gotcha are in The thin jar, its sources and javadoc jars and the pom are attached to the GitHub `snapshot` pre-release and to every versioned release with the `.asc` maven-gpg wrote at deploy; no fat jar -exists here. The attach jobs are shared text with the two sibling Maven-only repositories -(`publish.yml#github-snapshot` / `#github-release` in `shared-files.sha256`): the collect steps copy -`-.pom` + `.pom.asc` with the jars (maven-gpg writes both into `target/`), the -signature check counts every attachable type (`*.jar`, `*.aar`, `*.pom`), and `github-snapshot` ends -by deleting every asset of the `snapshot` pre-release that the run did not upload, each printed by -name -- `gh release upload --clobber` never removed anything, so the release accumulated every past -version. The prune refuses an empty upload, treats an asset that is already gone as removed and -fails only after trying every deletion; versioned releases are never pruned. See the policy's -"Attach first, then go red" and "The `snapshot` pre-release holds exactly the current upload", which -also records the delete-and-recreate form of this job (May 2026) and why it is gone. +exists here. Release collection, signature reporting and rolling-snapshot updates use the byte-identical +`collect-release-assets.py`, `report-asset-signatures.py`, `update-snapshot.py` and their buildcheck +library/tests, all listed in `.github/shared-files.sha256`. Repository-specific extras live in +`.github/release-assets.json`; Maven modules are discovered from POMs. Missing artifacts/signatures +fail collection while preserving available files for upload. The whole `github-snapshot` job is +shared across all four Java repos and serializes upload, source-tag/notes updates and pruning; +only complete signed uploads advance/prune the snapshot. Versioned releases are never pruned. +The `github-release` job remains shared across the three Maven-only repos. Interface, staging +options and synchronization rules: [`../workspace/policies/release-asset-scripts.md`](../workspace/policies/release-asset-scripts.md). ## Shared files and the release gate (`shared-files` job) From 537a427eda13b248b9aaad07d070cafe9f6a77ff Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Fri, 23 Oct 2026 20:51:19 +0000 Subject: [PATCH 02/19] Disable saved checkout credentials in release upload jobs --- .github/buildcheck/tests/test_snapshot.py | 10 ++++++++++ .github/shared-files.sha256 | 6 +++--- .github/workflows/publish.yml | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py index e14f357..bf148a6 100644 --- a/.github/buildcheck/tests/test_snapshot.py +++ b/.github/buildcheck/tests/test_snapshot.py @@ -200,6 +200,16 @@ def test_optional_api_failure_accepts_only_404_not_authentication_failure(self): with self.assertRaisesRegex(RuntimeError, "403"): gh.api("GET", "repos/owner/repo/releases/tags/snapshot", optional=True) + def test_release_upload_jobs_do_not_persist_checkout_credentials(self): + jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + versioned = "github-release" if "github-release" in jobs else "github-release-signed" + for name in ("github-snapshot", versioned): + checkouts = [step for step in jobs[name].steps() + if any("uses: actions/checkout@" in line for line in step)] + self.assertTrue(checkouts, name) + for step in checkouts: + self.assertIn(" persist-credentials: false", step, name) + if __name__ == "__main__": unittest.main() diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 94cb163..e0b4e92 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -68,12 +68,12 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -7f96fc1b0cffa9844379162c2f82e5f26454ab0fdc7349019ddc05423b4320af .github/workflows/publish.yml#github-snapshot -68d41a4a97d26438f5649f9c6f24a884d1448192486b386271d49051c58f6ee1 .github/workflows/publish.yml#github-release +f28843b1fc88b8876dfc6cc0cd451e5003087e904f4f20cefcf7c645e7ebea44 .github/workflows/publish.yml#github-snapshot +eaa2911c2c754ae0b1c9acff9aaba347e5d7fdfb841b4b5f2d12b271cb71d7fe .github/workflows/publish.yml#github-release b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py 8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py -c0401d973100c4ab0ec7728ec5d3a841831863cd307112c88f8cfe172316316a .github/buildcheck/tests/test_snapshot.py +4bab88300dc2bc06d33d659a2118d81070118e65300b9cfbacdc0313d11bee43 .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 29e6e0d..5416c0f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -472,6 +472,8 @@ jobs: contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets @@ -558,6 +560,8 @@ jobs: contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets From 0502739b44ca80194a3c35fee171da4655f33983 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 07:23:00 +0000 Subject: [PATCH 03/19] docs: add CodeRabbit badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a583b84..6b7f282 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ **AI:** [![Claude](https://img.shields.io/badge/Claude-D97757?logo=claude&logoColor=fff)](#) +[![CodeRabbit](https://img.shields.io/badge/CodeRabbit-AI%20Reviews-FF570A)](https://coderabbit.ai/) **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) From e59fd856fafc818cbf8b8bcba3defbac6beab7d1 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 07:56:36 +0000 Subject: [PATCH 04/19] docs: use repository-specific CodeRabbit reviews badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b7f282..e3efc71 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **AI:** [![Claude](https://img.shields.io/badge/Claude-D97757?logo=claude&logoColor=fff)](#) -[![CodeRabbit](https://img.shields.io/badge/CodeRabbit-AI%20Reviews-FF570A)](https://coderabbit.ai/) +![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/bernardladenthin/streambuffer?utm_source=oss&utm_medium=github&utm_campaign=bernardladenthin%2Fstreambuffer&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) From 5352d01bfcc937a756acf3141fc229766c9824f1 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 11:14:55 +0000 Subject: [PATCH 05/19] Preserve runtime license notices and document scoped FOSSA review --- .fossa.yml | 11 +++ .github/collect-third-party-licenses.py | 89 +++++++++++++++++++++++++ .github/shared-files.sha256 | 3 + THIRD_PARTY.md | 31 +++++++++ docs/dependency-license-review.md | 69 +++++++++++++++++++ pom.xml | 13 ++++ 6 files changed, 216 insertions(+) create mode 100644 .fossa.yml create mode 100644 .github/collect-third-party-licenses.py create mode 100644 THIRD_PARTY.md create mode 100644 docs/dependency-license-review.md diff --git a/.fossa.yml b/.fossa.yml new file mode 100644 index 0000000..c51f407 --- /dev/null +++ b/.fossa.yml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 + +version: 3 +project: + locator: git+github.com/bernardladenthin/streambuffer + +# No fixed revision.branch: preserve the actual PR/main branch and commit. +# No license overrides or scope exclusions: test and runtime findings stay visible. +maven: + scope-exclude: [] diff --git a/.github/collect-third-party-licenses.py b/.github/collect-third-party-licenses.py new file mode 100644 index 0000000..df33f5d --- /dev/null +++ b/.github/collect-third-party-licenses.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Preserve legal files from resolved runtime JARs without filename collisions. + +Input is maven-dependency-plugin:build-classpath output on this operating system. +This performs no license approvals, downloads, source rewriting or file filtering +by license type. Archives without legal files remain visible in inventory.json. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import zipfile + + +def legal_file(name): + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or "\\" in name or any(":" in part for part in path.parts): + raise ValueError("Unsafe archive entry: " + name) + if name.endswith("/"): + return False + lower = path.name.lower() + # A provider's LICENSE.class is executable code, not a license document. + if path.suffix.lower() in {".class", ".java", ".kt", ".png", ".jpg", ".gif", ".jar"}: + return False + return (any(part.lower() in {"licenses", "licences"} for part in path.parts) + or re.fullmatch(r"(?:licen[sc]e|notice|copying)(?:[-_.].*)?", lower) is not None) + + +def collect(classpath, output): + marker = output / ".generated-by-collect-third-party-licenses" + if output.is_symlink(): + raise ValueError("Output must not be a symlink") + if output.exists(): + if not marker.is_file(): + raise ValueError("Refusing to replace an output directory without our marker") + shutil.rmtree(output) + output.mkdir(parents=True) + marker.write_text("Generated runtime dependency license inventory.\n", encoding="utf-8") + entries = [] + for item in sorted(set(classpath.read_text(encoding="utf-8").strip().split(os.pathsep))): + if not item: + continue + source = Path(item) + if not source.is_file() or source.suffix.lower() != ".jar": + raise ValueError("Classpath entry is not a readable JAR: " + item) + with source.open("rb") as stream: + hasher = hashlib.sha256() + for block in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(block) + digest = hasher.hexdigest() + # The filename preserves classifiers; the digest also distinguishes equal + # filenames from different groups or different bytes of a snapshot. + directory = re.sub(r"[^a-zA-Z0-9._-]", "_", source.name) + "-" + digest[:16] + files = [] + with zipfile.ZipFile(source) as archive: + for info in sorted(archive.infolist(), key=lambda entry: entry.filename): + if not legal_file(info.filename): + continue + destination = output / directory / info.filename + destination.parent.mkdir(parents=True, exist_ok=True) + data = archive.read(info) + if destination.exists() and destination.read_bytes() != data: + raise ValueError("Conflicting duplicate entry: " + info.filename) + destination.write_bytes(data) + files.append(info.filename) + entries.append({"artifact": source.name, "sha256": digest, + "directory": directory, "legal_files": files}) + (output / "inventory.json").write_text( + json.dumps({"format": 1, "scope": "runtime", "artifacts": entries}, indent=2) + "\n", + encoding="utf-8") + missing = sum(not entry["legal_files"] for entry in entries) + print(f"Runtime legal inventory: {len(entries)} JARs, {missing} without embedded legal files") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--classpath", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + collect(args.classpath, args.output) + + +if __name__ == "__main__": + main() diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index e0b4e92..36f1369 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -78,3 +78,6 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license +7d82df16062877522da6a3d7a41e4b28f43ce4776ae89b6f7c7933cbb9f41aa3 docs/dependency-license-review.md +8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo +a6a4d6efe20634a62f860294b3fbd30c71d017a42fc55421ae83b8a7e8aba572 .github/collect-third-party-licenses.py diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md new file mode 100644 index 0000000..711e990 --- /dev/null +++ b/THIRD_PARTY.md @@ -0,0 +1,31 @@ + + +# Third-party license and attribution notes + +The project is Apache-2.0. The library has no declared runtime dependencies. +JMH 1.37, JCStress 0.16, Lincheck 3.7, JUnit and jqwik are test-only tools. +Lincheck includes LGPL-3.0-or-later source files as well as its declared MPL license. +A FOSSA scan of the old master revision does not cover this current build. + +## Review evidence + +See [the dependency license review](docs/dependency-license-review.md) in the source repository +for exact-version evidence and remaining decisions. This inventory is not an approval of all +licenses found by a scanner. Test-tool redistribution requires a separate review. + +For unchanged Maven components, source is available from +`https://repo.maven.apache.org/maven2////` +as `--sources.jar`; POMs and upstream source repositories provide provenance. +Preserve copyright, license and NOTICE files when redistributing dependencies. + +Runnable BAF/srcmorph CLI packaging uses Python 3.8+ to collect original dependency +legal files before the project JAR and fat JAR are assembled. Override `-Dlicense.python=python` +if that is the Python command on the build host. Legal files are under +`META-INF/third-party/dependencies/-/`. +`inventory.json` maps exact JAR hashes/classifiers to preserved files and also lists JARs +without embedded legal material, which require review or supplemental notices. +This source-location statement does not clear unknown proprietary or historical Git findings. diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md new file mode 100644 index 0000000..cab3418 --- /dev/null +++ b/docs/dependency-license-review.md @@ -0,0 +1,69 @@ + + +# Dependency license review and distribution + +## Scope and review rules + +The Maven repositories share FOSSA policy 85127 (Standard Bundle Distribution). A flagged +license is a review request. Repository configuration does not grant a license or approve a +FOSSA finding. Decisions below are version-specific evidence, not organization-wide exceptions. + +1. Analyze the actual commit and branch. Never label a PR scan as main. FOSSA's default/tracked + branch is a server setting; `.fossa.yml` does not repair that setting. After changing it, + analyze current main and verify the recorded commit. Streambuffer's previous May 7 master + revision is not evidence for its current dependencies. +2. Keep test dependencies visible. Do not hide GPL/LGPL findings by excluding tests globally. + Internal execution and redistribution of benchmark/test JARs are separate use cases. +3. Inspect original source headers as well as inherited POM licenses. A permissive POM does + not erase another license inside the same component. Record the exact version and file. +4. Preserve dependency license/notice files under artifact-specific paths in fat JARs, since + unpacking all dependencies into one directory can overwrite identically named legal files. +5. Include attribution and license texts missing from upstream binaries before distribution. + Full dependency/source provenance must remain available; bundled notices are not a statement + that every file of an upstream package has been legally cleared. +6. Enter a documented component/version decision in FOSSA only after its usage and notices + have been checked. Broad approval of GPL, AGPL, LGPL or proprietary findings is not appropriate. + +## Verified evidence (2026-10-24) + +| Component | Evidence and decision boundary | +|---|---| +| JMH 1.37, JCStress 0.16 | Parent POMs and binary LICENSE explicitly include GPL v2 with Classpath exception. Review these exact versions for internal test/benchmark use; do not treat two exception aliases as independent obligations or approve arbitrary GPL-only code. | +| JUnit Jupiter 6.1.3, jqwik 1.9.3 | EPL-2.0; test scope. Preserve license notices if distributing these tools. | +| Lincheck 3.7 | POM declares MPL-2.0, but source files `org/jetbrains/kotlinx/lincheck/strategy/managed/MemoryLocation.kt` and `MemoryTracker.kt` explicitly say LGPL-3.0-or-later. The LGPL finding is real; never override the entire component to MPL-only. Internal test use is the reviewed boundary; redistribution requires further review. | +| Logback Classic 1.6.5 | Source headers of `AsyncAppender.java` and `BasicConfigurator.java` explicitly allow EPL-2.0 OR LGPL-2.1. Use the EPL-2.0 alternative for these unchanged components where that grant applies. The extra EPL-1.0 finding still requires its exact matched file; do not silently overwrite it. | +| Jetty EE10 12.1.14 | Parent declares EPL-2.0 and Apache-2.0. Verify the applicable grant and notices of the actual files before recording an alternative-license selection. Used by the separate jllama agent, not the core. | +| JeroMQ 0.6.0 | POM declares MPL-2.0. Runtime in BAF; preserve notices and provide the exact upstream source location for unchanged covered code. | +| jsr305 3.0.2 | POM declares Apache-2.0 but `javax/annotation/concurrent/{GuardedBy,Immutable,NotThreadSafe,ThreadSafe}.java` contain CC-BY-2.5. The attribution finding is valid. | +| jcip-annotations 1.0 | Original sources explicitly contain CC-BY-2.5 and Brian Goetz/Tim Peierls attribution. Source republication must retain the notice. | +| Bouncy Castle provider 1.86 | Source `org/bouncycastle/LICENSE.java` contains the permissive Bouncy Castle copyright/permission/warranty text. Preserve it; a custom FOSSA license name alone does not imply a usage prohibition. | + +Source archives are obtainable at `https://repo.maven.apache.org/maven2/` using the component's +Maven coordinates and exact version (`-sources.jar`). Keep the POM and source archive together +as evidence. A new version requires checking again. + +## Outstanding service decisions + +FOSSA issue approvals/license corrections are SaaS settings, not Git settings. These branches +record evidence and distribution changes; they do not claim that server-side approvals have +been applied. Inspect exact file matches for the historical Git/proprietary findings and the +remaining Logback EPL-1.0 finding. Historical revisions are not changed by deleting current files. + +The unlicensed SevenForums material in BroomCabinet needs a documented redistribution grant +or a separate owner decision to remove it. A LicenseRef is documentation, not permission. + +## Releasing + +Review `THIRD_PARTY.md` per repository, the resolved runtime graph, and the produced archive. +For fat JARs inspect `META-INF/third-party/dependencies/inventory.json` and compare +copied legal entries with the original JAR bytes. The collector reads the resolved runtime +classpath, preserves classifiers and distinguishes equal filenames by content hash. +It never treats a `LICENSE.class` bytecode file as a text license. BAF's assembly profile and +the srcmorph CLI run it before packaging; Python 3.8+ is needed on that build host. +JARs with no legal entries remain explicitly listed. No network requests or approvals are made. +Missing legal material must be supplied explicitly. This check does not replace +source-availability obligations or the review of native third-party components. diff --git a/pom.xml b/pom.xml index a676a45..2b119e4 100644 --- a/pom.xml +++ b/pom.xml @@ -226,6 +226,19 @@ SPDX-License-Identifier: Apache-2.0 + + + ${project.basedir}/src/main/resources + + + ${project.basedir} + META-INF/third-party/net.ladenthin/${project.artifactId} + + LICENSE + THIRD_PARTY.md + + + From bd1e683a4783f196e0752d1dab4f2564f8582f41 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 14:30:54 +0000 Subject: [PATCH 06/19] Record SevenForums assessment as unresolved without asserting a release blocker --- .github/shared-files.sha256 | 2 +- docs/dependency-license-review.md | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 36f1369..7311e83 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -78,6 +78,6 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license -7d82df16062877522da6a3d7a41e4b28f43ce4776ae89b6f7c7933cbb9f41aa3 docs/dependency-license-review.md +0dabc638fcf4a505029e8718fca0d6434e66874d8029cfd9da2626003de6a5d7 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo a6a4d6efe20634a62f860294b3fbd30c71d017a42fc55421ae83b8a7e8aba572 .github/collect-third-party-licenses.py diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md index cab3418..f96627e 100644 --- a/docs/dependency-license-review.md +++ b/docs/dependency-license-review.md @@ -53,8 +53,13 @@ record evidence and distribution changes; they do not claim that server-side app been applied. Inspect exact file matches for the historical Git/proprietary findings and the remaining Logback EPL-1.0 finding. Historical revisions are not changed by deleting current files. -The unlicensed SevenForums material in BroomCabinet needs a documented redistribution grant -or a separate owner decision to remove it. A LicenseRef is documentation, not permission. +BroomCabinet records the SevenForums registry files as finding `BROOM-LIC-001` in its +`THIRD_PARTY.md`, with status unresolved. The reviewed contents are predominantly Microsoft +Windows default configuration; no substantial independent program logic was identified. +Whether protectable third-party contributions exist and require a redistribution grant remains +unresolved. The files and provenance markings are retained pending assessment; no release +blocker has been established from this finding. A LicenseRef documents provenance, not a grant +or scanner approval. This finding concerns BroomCabinet only. ## Releasing From 76dc704e96931769f5bb5e21c14c9174d4b6f28e Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 15:19:07 +0000 Subject: [PATCH 07/19] Preserve COPYRIGHT documents in shared runtime license collector --- .github/collect-third-party-licenses.py | 2 +- .github/shared-files.sha256 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/collect-third-party-licenses.py b/.github/collect-third-party-licenses.py index df33f5d..d337a08 100644 --- a/.github/collect-third-party-licenses.py +++ b/.github/collect-third-party-licenses.py @@ -28,7 +28,7 @@ def legal_file(name): if path.suffix.lower() in {".class", ".java", ".kt", ".png", ".jpg", ".gif", ".jar"}: return False return (any(part.lower() in {"licenses", "licences"} for part in path.parts) - or re.fullmatch(r"(?:licen[sc]e|notice|copying)(?:[-_.].*)?", lower) is not None) + or re.fullmatch(r"(?:licen[sc]e|notice|copying|copyright)(?:[-_.].*)?", lower) is not None) def collect(classpath, output): diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 7311e83..5bfbb2b 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -80,4 +80,4 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license 0dabc638fcf4a505029e8718fca0d6434e66874d8029cfd9da2626003de6a5d7 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo -a6a4d6efe20634a62f860294b3fbd30c71d017a42fc55421ae83b8a7e8aba572 .github/collect-third-party-licenses.py +31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py From d1caf722bf85818a61208f3f20f85916c2b659fa Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 15:23:54 +0000 Subject: [PATCH 08/19] Address review feedback on docs, badge links and signature subtests --- .github/buildcheck/tests/test_snapshot.py | 6 ++++-- .github/shared-files.sha256 | 4 ++-- README.md | 2 +- docs/dependency-license-review.md | 10 +++------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py index bf148a6..4b74f8c 100644 --- a/.github/buildcheck/tests/test_snapshot.py +++ b/.github/buildcheck/tests/test_snapshot.py @@ -101,9 +101,11 @@ def test_late_finishing_older_commit_cannot_overwrite_newer_snapshot(self): self.assertTrue(all(c[0] == "GET" for c in self.gh.calls)) def test_empty_signature_or_orphan_also_prevents_promotion(self): - for name in ("new.json.asc", "orphan.jar.asc"): + for name, content in (("new.json.asc", ""), ("orphan.jar.asc", "signature")): with self.subTest(name=name): - (self.assets / name).write_text("") + (self.assets / "new.json.asc").write_text("signature") + (self.assets / name).write_text(content) + self.gh.calls.clear() with self.assertRaisesRegex(ValueError, "collection is incomplete"): self.publish() self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 5bfbb2b..9ca8fa5 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -73,11 +73,11 @@ eaa2911c2c754ae0b1c9acff9aaba347e5d7fdfb841b4b5f2d12b271cb71d7fe .github/workfl b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py 8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py -4bab88300dc2bc06d33d659a2118d81070118e65300b9cfbacdc0313d11bee43 .github/buildcheck/tests/test_snapshot.py +8b6ae70f43560a0095b6f16012b80ed694b07633c1cbde94dbe8c21c529461ea .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license -0dabc638fcf4a505029e8718fca0d6434e66874d8029cfd9da2626003de6a5d7 docs/dependency-license-review.md +0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py diff --git a/README.md b/README.md index e3efc71..fb79c22 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **AI:** [![Claude](https://img.shields.io/badge/Claude-D97757?logo=claude&logoColor=fff)](#) -![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/bernardladenthin/streambuffer?utm_source=oss&utm_medium=github&utm_campaign=bernardladenthin%2Fstreambuffer&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) +[![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/bernardladenthin/streambuffer?utm_source=oss&utm_medium=github&utm_campaign=bernardladenthin%2Fstreambuffer&labelColor=171717&color=FF570A&label=CodeRabbit+Reviews)](https://coderabbit.ai) **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md index f96627e..76b82d0 100644 --- a/docs/dependency-license-review.md +++ b/docs/dependency-license-review.md @@ -53,13 +53,9 @@ record evidence and distribution changes; they do not claim that server-side app been applied. Inspect exact file matches for the historical Git/proprietary findings and the remaining Logback EPL-1.0 finding. Historical revisions are not changed by deleting current files. -BroomCabinet records the SevenForums registry files as finding `BROOM-LIC-001` in its -`THIRD_PARTY.md`, with status unresolved. The reviewed contents are predominantly Microsoft -Windows default configuration; no substantial independent program logic was identified. -Whether protectable third-party contributions exist and require a redistribution grant remains -unresolved. The files and provenance markings are retained pending assessment; no release -blocker has been established from this finding. A LicenseRef documents provenance, not a grant -or scanner approval. This finding concerns BroomCabinet only. +Keep repository-specific assessments in that repository's `THIRD_PARTY.md`, including +affected files, evidence, review status and the current decision. A LicenseRef identifier +alone does not establish permission or scanner approval. ## Releasing From dfef3046c3ebb009beda6b00c73cc17ce2d463fd Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 16:06:54 +0000 Subject: [PATCH 09/19] Gate signed publications locally and decouple Central and GitHub uploads --- .github/buildcheck/publication.py | 292 ++++++++++++++++++ .github/buildcheck/releaseassets.py | 15 +- .github/buildcheck/tests/test_publication.py | 126 ++++++++ .../buildcheck/tests/test_releaseassets.py | 6 +- .github/publication.py | 14 + .github/release-gate-exemptions.txt | 2 + .github/shared-files.sha256 | 14 +- .github/workflows/publish.yml | 222 +++++++------ docs/publication-preflight.md | 84 +++++ pom.xml | 6 +- 10 files changed, 668 insertions(+), 113 deletions(-) create mode 100644 .github/buildcheck/publication.py create mode 100644 .github/buildcheck/tests/test_publication.py create mode 100644 .github/publication.py create mode 100644 docs/publication-preflight.md diff --git a/.github/buildcheck/publication.py b/.github/buildcheck/publication.py new file mode 100644 index 0000000..a8d0db5 --- /dev/null +++ b/.github/buildcheck/publication.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Freeze signed publication bytes once; independent consumers never rebuild them.""" +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import subprocess +import tempfile +import xml.etree.ElementTree as ET +import zipfile + +from .releaseassets import configuration, projects, signature_gaps, _classifiers, _format, _inside + +PORTAL = "https://central.sonatype.com/api/v1/publisher/upload" +SNAPSHOTS = "https://central.sonatype.com/repository/maven-snapshots/" +DEPLOY_PLUGIN = "3.1.4" + + +def digest(path): + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def safe_path(root, name): + p = PurePosixPath(name) + if not name or p.is_absolute() or ".." in p.parts or "\\" in name or ":" in name: + raise ValueError("unsafe publication path: " + name) + target = root / name + if target.is_symlink() or any(parent.is_symlink() for parent in target.parents): + raise ValueError("publication symlink: " + name) + return target + + +def expected_central(root, config, modules, *, only_central=True): + names = set() + for _, artifact, version, packaging, reactor in modules: + base = f"{artifact}-{version}" + names.add(base + ".pom") + if packaging != "pom": + names.update(base + suffix + ".jar" for suffix in ("", "-sources", "-javadoc")) + if reactor and config.get("maven", {}).get("cyclonedx", False): + names.add(base + "-cyclonedx.json") + for rule in config.get("files", []): + if only_central and not rule.get("central", False): + continue + for classifier in _classifiers(root, rule.get("classifiers", [""])): + matches = list(root.glob(_format(rule["source"], modules[0][2], classifier))) + if len(matches) != 1: + raise ValueError("missing/ambiguous Central input: " + rule["source"]) + names.add(_format(rule["name"], modules[0][2], classifier) if "name" in rule else matches[0].name) + return names + + +def prepare(root, assets, destination): + """Gate both publishers on exact membership, valid signatures and frozen checksums.""" + root, assets, destination = Path(root).resolve(), Path(assets).resolve(), Path(destination).resolve() + if destination.exists(): + raise ValueError("publication destination must not exist") + missing, errors = signature_gaps(assets) + if missing: + raise ValueError("incomplete publication: " + "; ".join(errors)) + config = configuration(root) + publication = config.get("publication", {}) + modules = projects(root, config.get("maven", {}).get("standalone", [])) + version = modules[0][2] + channel = "snapshot" if version.endswith("-SNAPSHOT") else "release" + staging = "central-deferred" if channel == "snapshot" else "central-staging" + roots = publication.get("central_roots", ["target/{staging}"]) + central = {} + for item in roots: + folder = _inside(root, item.format(staging=staging)) + if not folder.is_dir(): + raise ValueError("missing Central staging: " + str(folder)) + for path in folder.rglob("*"): + if not path.is_file(): + continue + relative = path.relative_to(folder).as_posix() + safe_path(folder, relative) + if not re.search(r"\.(?:pom|jar|aar|json)(?:\.asc)?$", path.name): + continue # Maven metadata/checksums are regenerated, never mistaken for payload. + parts = PurePosixPath(relative).parts + if len(parts) < 4 or parts[-2] != version: + raise ValueError("unexpected Central coordinates: " + relative) + if relative in central and digest(central[relative]) != digest(path): + raise ValueError("conflicting Central input: " + relative) + central[relative] = path + expected = expected_central(root, config, modules) + actual = {path.name for path in central.values() if not path.name.endswith(".asc")} + if actual != expected or len(central) != 2 * len(expected): + raise ValueError("Central membership mismatch; missing=" + str(sorted(expected - actual)) + + "; unexpected=" + str(sorted(actual - expected))) + github_expected = expected_central(root, config, modules, only_central=False) + github_actual = {p.name for p in assets.iterdir() if not p.name.endswith(".asc")} + if github_actual != github_expected: + raise ValueError("GitHub membership does not match the release contract") + # Require the Central pair to match GitHub; verify signatures on the frozen copy. + for path in sorted(assets.iterdir()): + if path.is_symlink() or not path.is_file(): + raise ValueError("non-flat publication input: " + str(path)) + for name, path in central.items(): + if not (assets / path.name).is_file() or digest(assets / path.name) != digest(path): + raise ValueError("Central/GitHub bytes differ: " + name) + # Build a temporary package, then promote it locally only after all checks succeed. + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="publication-", dir=destination.parent) as tmp: + package = Path(tmp) / "package" + (package / "github").mkdir(parents=True) + for path in assets.iterdir(): + shutil.copyfile(path, package / "github" / path.name) + for relative, path in central.items(): + target = safe_path(package / "central", relative) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, target) + for path in sorted((package / "github").iterdir()): + if not path.name.endswith(".asc"): + subprocess.run(["gpg", "--batch", "--verify", str(path) + ".asc", str(path)], check=True) + for path in (package / "central").rglob("*"): + if path.is_file() and digest(path) != digest(package / "github" / path.name): + raise ValueError("frozen Central/GitHub files differ: " + path.name) + manifest = {"schema": 1, "version": version, "channel": channel, + "commit": os.environ.get("GITHUB_SHA", ""), "files": {}} + for path in sorted(package.rglob("*")): + if path.is_file(): + manifest["files"][path.relative_to(package).as_posix()] = { + "sha256": digest(path), "size": path.stat().st_size} + (package / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + shutil.move(str(package), destination) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write("complete=true\n") + print(f"Publication ready: {version}; {len(expected)} Central artifacts; all signatures verified") + + +def verify(package): + """Transport integrity check; never regenerate or re-sign publication files.""" + package = Path(package).resolve() + manifest = json.loads((package / "manifest.json").read_text()) + if manifest.get("schema") != 1 or manifest.get("channel") not in ("snapshot", "release"): + raise ValueError("unsupported publication manifest") + if os.environ.get("GITHUB_SHA") and manifest.get("commit") != os.environ["GITHUB_SHA"]: + raise ValueError("publication belongs to a different commit") + actual = {p.relative_to(package).as_posix() for p in package.rglob("*") if p.is_file()} + if actual != set(manifest["files"]) | {"manifest.json"}: + raise ValueError("publication file membership changed") + for relative, info in manifest["files"].items(): + path = safe_path(package, relative) + if not path.is_file() or path.stat().st_size != info["size"] or digest(path) != info["sha256"]: + raise ValueError("publication bytes changed: " + relative) + return manifest + + +def snapshot_pom(package, destination, url=SNAPSHOTS): + """Generate transfer-only deploy-file executions, including all signature extensions. + + Maven handles timestamped snapshot names and metadata in one request per component. + There is no project build, dependency resolution, compilation or signing phase. + """ + manifest = verify(package) + if manifest["channel"] != "snapshot": + raise ValueError("snapshot transfer requires a SNAPSHOT version") + root = ET.Element("project", xmlns="http://maven.apache.org/POM/4.0.0") + def child(parent, name, text): + node = ET.SubElement(parent, name) + node.text = str(text) + return node + for name, value in [("modelVersion", "4.0.0"), ("groupId", "local.publication"), + ("artifactId", "snapshot-transfer"), ("version", "1"), ("packaging", "pom")]: + child(root, name, value) + plugin = ET.SubElement(ET.SubElement(ET.SubElement(root, "build"), "plugins"), "plugin") + child(plugin, "groupId", "org.apache.maven.plugins") + child(plugin, "artifactId", "maven-deploy-plugin") + child(plugin, "version", DEPLOY_PLUGIN) + executions = ET.SubElement(plugin, "executions") + central = Path(package).resolve() / "central" + for i, pom in enumerate(sorted(central.rglob("*.pom"))): + parts = pom.relative_to(central).parts + group, artifact, version = ".".join(parts[:-3]), parts[-3], parts[-2] + xml = ET.parse(pom).getroot() + packaging = xml.findtext("{*}packaging", "jar") + extension = "jar" if packaging == "maven-plugin" else packaging + base = f"{artifact}-{version}" + # A POM-only component has a single unclassified signature. Deploy-file + # treats an empty classifiers parameter as absent, so use that signature + # as the primary artifact and let pomFile attach its signed POM alongside. + transfer_packaging = "pom.asc" if packaging == "pom" else packaging + main = pom.parent / (f"{base}.pom.asc" if packaging == "pom" else f"{base}.{extension}") + if not main.is_file(): + raise ValueError("missing primary Central artifact: " + str(main)) + execution = ET.SubElement(executions, "execution") + child(execution, "id", f"transfer-{i}") + child(execution, "phase", "validate") + child(ET.SubElement(execution, "goals"), "goal", "deploy-file") + config = ET.SubElement(execution, "configuration") + for name, value in [("repositoryId", "central"), ("url", url), ("file", main), + ("pomFile", pom), ("groupId", group), ("artifactId", artifact), + ("version", version), ("packaging", transfer_packaging), ("generatePom", "false")]: + child(config, name, value) + attached = [] + for path in sorted(pom.parent.iterdir()): + if path in (pom, main): + continue + suffix = path.name[len(base):] if path.name.startswith(base) else "" + match = re.fullmatch(r"(?:-(.+))?\.(pom|jar|aar|json)(\.asc)?", suffix) + if not match: + raise ValueError("invalid attached artifact: " + path.name) + classifier = match[1] or "" + kind = match[2] + (match[3] or "") + if "," in str(path): + raise ValueError("comma in Maven transfer path") + attached.append((str(path), classifier, kind)) + if attached: + for name, column in [("files", 0), ("classifiers", 1), ("types", 2)]: + child(config, name, ",".join(item[column] for item in attached)) + ET.ElementTree(root).write(destination, encoding="utf-8", xml_declaration=True) + + +def upload(package): + """Transfer frozen files, accepting Portal upload without validation polling.""" + package = Path(package).resolve() + manifest = verify(package) + if manifest["channel"] == "snapshot": + with tempfile.TemporaryDirectory(prefix="snapshot-transfer-") as tmp: + pom = Path(tmp) / "pom.xml" + snapshot_pom(package, pom) + subprocess.run(["mvn", "-B", "--no-transfer-progress", "-Dmaven.repository.credentialScope=id", + "-f", str(pom), "validate"], check=True) + return + with tempfile.TemporaryDirectory(prefix="central-transfer-") as tmp: + bundle = Path(tmp) / "central-bundle.zip" + with zipfile.ZipFile(bundle, "w", zipfile.ZIP_DEFLATED) as archive: + for relative in manifest["files"]: + if not relative.startswith("central/"): + continue + path = package / relative + name = relative[len("central/"):] + hashes = {a: hashlib.new(a) for a in ("md5", "sha1", "sha256", "sha512")} + with path.open("rb") as source, archive.open(name, "w", force_zip64=True) as target: + for block in iter(lambda: source.read(1024 * 1024), b""): + target.write(block) + for h in hashes.values(): + h.update(block) + if not name.endswith(".asc"): + for algorithm, h in hashes.items(): + archive.writestr(name + "." + algorithm, h.hexdigest()) + token = base64.b64encode((os.environ["MAVEN_USERNAME"] + ":" + + os.environ["MAVEN_PASSWORD"]).encode()).decode() + config = Path(tmp) / "curl.conf" + config.touch(mode=0o600) + config.write_text('header = "Authorization: Bearer ' + token + '"\n') + response = subprocess.run(["curl", "--fail-with-body", "--silent", "--show-error", + "--connect-timeout", "30", "--max-time", "1800", "--config", str(config), + "-F", "bundle=@" + str(bundle), PORTAL + "?publishingType=AUTOMATIC"], + text=True, capture_output=True) + if response.returncode: + raise RuntimeError("Central upload failed: " + response.stderr + response.stdout) + deployment = response.stdout.strip().strip('"') + if not re.fullmatch(r"[A-Za-z0-9_-]+", deployment): + raise RuntimeError("Central returned no valid deployment receipt") + print("Central accepted deployment: " + deployment) + print("Validation and publication continue asynchronously at Central; no polling.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + prepare_parser = commands.add_parser("prepare") + prepare_parser.add_argument("assets", type=Path) + prepare_parser.add_argument("package", type=Path) + verify_parser = commands.add_parser("verify") + verify_parser.add_argument("package", type=Path) + verify_parser.add_argument("--export-github", type=Path) + upload_parser = commands.add_parser("upload") + upload_parser.add_argument("package", type=Path) + args = parser.parse_args() + if args.command == "prepare": + prepare(Path(__file__).resolve().parents[2], args.assets, args.package) + elif args.command == "verify": + verify(args.package) + if args.export_github: + if args.export_github.exists(): + raise ValueError("GitHub export destination must not exist") + shutil.copytree(args.package / "github", args.export_github) + else: + upload(args.package) diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py index 36d36fe..de10377 100644 --- a/.github/buildcheck/releaseassets.py +++ b/.github/buildcheck/releaseassets.py @@ -33,9 +33,15 @@ def configuration(root, path=None): """Read the explicit contract; unknown keys fail instead of silently dropping assets.""" path = Path(path) if path is not None else Path(root) / ".github/release-assets.json" config = json.loads(path.read_text()) - _keys(config, ("schema", "maven", "files"), "release asset configuration") + _keys(config, ("schema", "maven", "files", "publication"), "release asset configuration") if type(config.get("schema")) is not int or config["schema"] != 1: raise ValueError("release asset configuration requires schema: 1") + publication = config.get("publication", {}) + _keys(publication, ("central_roots",), "publication configuration") + if "central_roots" in publication and (not isinstance(publication["central_roots"], list) + or not publication["central_roots"] + or any(not isinstance(path, str) or not path for path in publication["central_roots"])): + raise ValueError("publication.central_roots must be a nonempty list of paths") maven = config.get("maven", {}) _keys(maven, ("standalone", "cyclonedx"), "maven configuration") if not isinstance(maven.get("standalone", []), list): @@ -44,6 +50,9 @@ def configuration(root, path=None): raise ValueError("maven.cyclonedx must be a boolean") if not isinstance(config.get("files", []), list): raise ValueError("files must be a list") + for rule in config.get("files", []): + if isinstance(rule, dict) and "central" in rule and type(rule["central"]) is not bool: + raise ValueError("file rule central must be a boolean") return config @@ -111,7 +120,7 @@ def file_classifiers(root, index, config_path=None): item = config.get("files", [])[index] except IndexError as error: raise ValueError(f"no file rule at index {index}") from error - _keys(item, ("source", "name", "classifiers"), "file rule") + _keys(item, ("source", "name", "classifiers", "central"), "file rule") return _classifiers(Path(root).resolve(), item.get("classifiers", [""])) @@ -138,7 +147,7 @@ def collect(root, destination, config_path=None, *, maven_only=False, maven_asse expected.append((Path(maven_assets) / name if maven_assets is not None else source, name)) for item in [] if maven_only else config.get("files", []): - _keys(item, ("source", "name", "classifiers"), "file rule") + _keys(item, ("source", "name", "classifiers", "central"), "file rule") for classifier in _classifiers(root, item.get("classifiers", [""])): pattern = _format(item.get("source"), version, classifier) _inside(root, pattern) diff --git a/.github/buildcheck/tests/test_publication.py b/.github/buildcheck/tests/test_publication.py new file mode 100644 index 0000000..9fd5f3b --- /dev/null +++ b/.github/buildcheck/tests/test_publication.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Publication gate and transport invariants; no signing key or remote writes.""" +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + +from buildcheck.publication import prepare, verify, snapshot_pom +from buildcheck import workflow +from buildcheck.tests.helpers import REPO + + +class PublicationTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + (self.root / '.github').mkdir() + (self.root / '.github/release-assets.json').write_text('{"schema":1}') + self.pom = '4.0.0local.testdemo1-SNAPSHOT' + (self.root / 'pom.xml').write_text(self.pom) + self.assets = self.root / 'assets' + self.assets.mkdir() + self.stage = self.root / 'target/central-deferred/local/test/demo/1-SNAPSHOT' + self.stage.mkdir(parents=True) + for suffix in ['.pom', '.jar', '-sources.jar', '-javadoc.jar']: + name = 'demo-1-SNAPSHOT' + suffix + for extra in ['', '.asc']: + data = self.pom if suffix == '.pom' and not extra else name + extra + (self.assets / (name + extra)).write_text(data) + (self.stage / (name + extra)).write_text(data) + self.package = self.root / 'package' + self.signatures = patch('buildcheck.publication.subprocess.run') + self.run = self.signatures.start() + self.addCleanup(self.signatures.stop) + quiet = patch('builtins.print') + quiet.start() + self.addCleanup(quiet.stop) + + def freeze(self): + prepare(self.root, self.assets, self.package) + + def test_freezes_once_and_verifies_every_signature(self): + self.freeze() + self.assertEqual(self.run.call_count, 4) + self.assertEqual(verify(self.package)['channel'], 'snapshot') + (self.assets / 'demo-1-SNAPSHOT.jar').write_text('later rebuild') + self.assertNotEqual((self.package / 'github/demo-1-SNAPSHOT.jar').read_text(), 'later rebuild') + verify(self.package) + + def test_missing_signature_or_wrong_central_bytes_block_both_publishers(self): + sig = self.assets / 'demo-1-SNAPSHOT.jar.asc' + sig.unlink() + with self.assertRaisesRegex(ValueError, 'incomplete'): + self.freeze() + self.assertFalse(self.package.exists()) + sig.write_text('signature') + with self.assertRaisesRegex(ValueError, 'bytes differ'): + self.freeze() + self.assertFalse(self.package.exists()) + + def test_invalid_signature_blocks_package_creation(self): + import subprocess + self.run.side_effect = subprocess.CalledProcessError(1, 'gpg') + with self.assertRaises(subprocess.CalledProcessError): + self.freeze() + self.assertFalse(self.package.exists()) + + def test_transport_tampering_and_extra_files_are_rejected(self): + self.freeze() + unexpected = self.package / 'unexpected.txt' + unexpected.write_text('extra') + with self.assertRaisesRegex(ValueError, 'membership'): + verify(self.package) + unexpected.unlink() + (self.package / 'github/demo-1-SNAPSHOT.jar').write_text('tampered') + with self.assertRaisesRegex(ValueError, 'bytes changed'): + verify(self.package) + + def test_snapshot_transfer_retains_signature_extensions(self): + self.freeze() + target = self.root / 'transfer.xml' + snapshot_pom(self.package, target, 'file:///unused-local-diagnostic') + xml = ET.parse(target).getroot() + config = xml.find('.//{*}configuration') + self.assertIn('jar.asc', config.findtext('{*}types')) + self.assertIn('pom.asc', config.findtext('{*}types')) + self.assertEqual(xml.findtext('.//{*}phase'), 'validate') + self.assertEqual(config.findtext('{*}generatePom'), 'false') + + def test_pom_only_component_transfers_signature_and_pom_together(self): + pom = self.pom.replace('', 'pom') + (self.root / 'pom.xml').write_text(pom) + for directory in (self.assets, self.stage): + for path in directory.iterdir(): + if '.jar' in path.name: + path.unlink() + (directory / 'demo-1-SNAPSHOT.pom').write_text(pom) + self.freeze() + target = self.root / 'pom-transfer.xml' + snapshot_pom(self.package, target) + config = ET.parse(target).getroot().find('.//{*}configuration') + self.assertEqual(config.findtext('{*}packaging'), 'pom.asc') + self.assertTrue(config.findtext('{*}file').endswith('.pom.asc')) + self.assertTrue(config.findtext('{*}pomFile').endswith('.pom')) + self.assertIsNone(config.find('{*}files')) + + def test_publishers_depend_only_on_the_same_checked_handoff(self): + jobs = workflow.parse((Path(REPO) / '.github/workflows/publish.yml').read_text()) + for channel in ['snapshot', 'release']: + prep = jobs['publish-' + channel] + central = jobs['central-' + channel] + github_name = 'github-snapshot' if channel == 'snapshot' else ('github-release' if 'github-release' in jobs else 'github-release-signed') + github = jobs[github_name] + self.assertEqual(central.needs, ['publish-' + channel]) + self.assertEqual(github.needs, ['publish-' + channel]) + self.assertIn('steps.preflight.outputs.complete', prep.text) + self.assertLess(prep.text.index('publication.py prepare'), prep.text.index('uses: actions/upload-artifact@')) + self.assertIn('-DskipUpload=true deploy', prep.text) + self.assertNotIn('publishAllPublicationsToCentralSnapshotsRepository', prep.text) + self.assertIn('publication.py upload publication-package', central.text) + self.assertIn('publication.py verify publication-package', github.text) + self.assertNotIn('Fail if anything', github.text) diff --git a/.github/buildcheck/tests/test_releaseassets.py b/.github/buildcheck/tests/test_releaseassets.py index a4ed72a..8908fbc 100644 --- a/.github/buildcheck/tests/test_releaseassets.py +++ b/.github/buildcheck/tests/test_releaseassets.py @@ -188,12 +188,12 @@ def test_current_repository_workflow_passes_collection_result_and_checks_both_up for kind in ("snapshot", "release"): job = jobs[f"publish-{kind}"].text self.assertIn(f"python3 .github/collect-release-assets.py signed-{kind}-assets", job) - self.assertIn("assets_complete: $" + "{{ steps.collect.outputs.complete }}", job) + self.assertIn("assets_complete: $" + "{{ steps.preflight.outputs.complete }}", job) self.assertIn(" id: collect", job) attach = jobs["github-snapshot" if kind == "snapshot" else "github-release" if "github-release" in jobs else "github-release-signed"].text - self.assertIn(f"python3 .github/report-asset-signatures.py {'snapshot' if kind == 'snapshot' else 'release'}-assets", attach) - self.assertLess(attach.index("id: signatures"), attach.index(" - name: Fail if anything")) + self.assertIn(f"python3 .github/publication.py verify publication-package --export-github {kind}-assets", attach) + self.assertNotIn("Fail if anything", attach) if configuration(REPO).get("maven", {}).get("cyclonedx", False): for kind in ("snapshot", "release"): self.assertIn(f"signed-{kind}-assets/*.json", jobs[f"publish-{kind}"].text) diff --git a/.github/publication.py b/.github/publication.py new file mode 100644 index 0000000..a1dd8cf --- /dev/null +++ b/.github/publication.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""Prepare, verify or upload a signed publication without rebuilding its files.""" +import subprocess +import sys +from buildcheck.publication import main + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, RuntimeError, KeyError, subprocess.CalledProcessError) as error: + print(f"::error::{error}") + sys.exit(1) diff --git a/.github/release-gate-exemptions.txt b/.github/release-gate-exemptions.txt index 5663e04..bfbdfa7 100644 --- a/.github/release-gate-exemptions.txt +++ b/.github/release-gate-exemptions.txt @@ -14,3 +14,5 @@ github-snapshot: runs after publishing; attaches the assets to the rolling snaps github-release: runs after publishing; attaches the signed assets to the tag release verify-signing-key: red by design where the secret is withheld (forks, other contributors' branches); the publish jobs sign with the same key and fail on it themselves verify-signing-key-gradle: same as verify-signing-key, for the Gradle/BouncyCastle signing path +central-snapshot: Independent post-preflight Central transfer; must not gate the GitHub publication path. +central-release: Independent post-preflight Central transfer; must not gate the GitHub publication path. diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 9ca8fa5..284c318 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -68,11 +68,11 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl 09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot 51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -f28843b1fc88b8876dfc6cc0cd451e5003087e904f4f20cefcf7c645e7ebea44 .github/workflows/publish.yml#github-snapshot -eaa2911c2c754ae0b1c9acff9aaba347e5d7fdfb841b4b5f2d12b271cb71d7fe .github/workflows/publish.yml#github-release -b9bbf8762e5898709d69a2509c64750e3acf41dc0a69426578773714d263d365 .github/buildcheck/releaseassets.py +9e7609a8280585c13baa5418afc52e09d12753fea2c1f7d2d7b392e8a74e67b6 .github/workflows/publish.yml#github-snapshot +52f1f7c99a604f40fb3f3685909c060b2c270e3f3dd5d28f3119be4b3f5d4052 .github/workflows/publish.yml#github-release +ed1ba202b9d81a00893931a090112bfb28770a514030a5c08d5441c83369f8b7 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py -8e9f5870bae1d9e903ef35db9366cd680fcbb459076d1fdb0390248ab6b1cc30 .github/buildcheck/tests/test_releaseassets.py +8bc93d2ca737d7641abdd6311444233d9603c188dfd9d89d785b70cb92fb0f93 .github/buildcheck/tests/test_releaseassets.py 8b6ae70f43560a0095b6f16012b80ed694b07633c1cbde94dbe8c21c529461ea .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py @@ -81,3 +81,9 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py +4d16ef3f5f4fa5fa39d191b030e1cbff3c943c67b33ae9b392537e4b08f35004 .github/buildcheck/publication.py +9180fac217c75b3e11f67e244b9171f447d2c21665b1802574b57b97019f7173 .github/publication.py +c8d911dfd539016ad404460a3879d0b1d9635ba5c4e0843e9ce98acd5edb3f17 .github/buildcheck/tests/test_publication.py +b6a8429621c3aa5af71c4d4a63bb729cb220c858b74ba9f3d1ba14813602fddb docs/publication-preflight.md +b3cb68727cc2b0a4e6831b3b9301b6e54420dc6c52aa068dcd8b2b38a6c2308b .github/workflows/publish.yml#central-snapshot +5bf3977c27f1a5ae7f9894cc6e1b26b014ea3ee6bb854fe89daa9cbb26785a23 .github/workflows/publish.yml#central-release diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5416c0f..74efadb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -398,16 +398,21 @@ jobs: run: echo "Confirmed on tag ${{ github.ref }}" publish-snapshot: - name: Publish Snapshot to Central - needs: [check-snapshot, code-style, smoke-jar, shared-files, vmlens] + name: Prepare and verify signed snapshot artifacts + needs: + - check-snapshot + - code-style + - smoke-jar + - shared-files + - vmlens if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central outputs: - assets_complete: ${{ steps.collect.outputs.complete }} + assets_complete: ${{ steps.preflight.outputs.complete }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad with: java-version-file: .java-version distribution: temurin @@ -423,96 +428,74 @@ jobs: VERSION=$(mvn -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n1) echo "Resolved project version: $VERSION" case "$VERSION" in - *-SNAPSHOT) echo "OK: -SNAPSHOT version, continuing snapshot deploy." ;; + *-SNAPSHOT) echo "OK: -SNAPSHOT version, continuing local snapshot staging." ;; *) echo "::error::Refusing to publish non-SNAPSHOT version '$VERSION' from the snapshot job. Snapshot publishing requires a -SNAPSHOT version; releases go through the v* tag path."; exit 1 ;; esac - # Informational only (nothing depends on it): logs the effective POM with the same - # profile as the deploy below, so the resolved central-publishing configuration - # (waitUntil/waitMaxTime etc.) is visible for debugging. - name: Show effective POM (debug) - run: mvn --batch-mode --no-transfer-progress -P release help:effective-pom - # -Dmaven.repository.credentialScope=id: Maven 3.10 (on the runners since 2026-10) sends a server's - # credentials only to its declared origins, and for the id `central` that is repo.maven.apache.org -- - # an upload to central.sonatype.com then goes out anonymously and fails with HTTP 401. - # Restores id-only matching; replace with setup-java's `mvn-server-repository-origins: - # central:https://central.sonatype.com` once a release of setup-java ships that input. - - name: Deploy snapshot - run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release deploy -DskipTests + run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true help:effective-pom + - name: Stage snapshot + run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - # Runs even when the deploy step failed: a Central publish-poll timeout reds the - # job *after* the bundle was uploaded (and typically published server-side), while - # the signed jars and poms + .asc files already exist in target/ (signing happens at - # verify). Collecting on failure lets the github-snapshot job still attach them. - name: Collect signed artifacts id: collect - if: ${{ !cancelled() }} run: python3 .github/collect-release-assets.py signed-snapshot-assets - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: ${{ !cancelled() }} + - name: Verify and freeze the complete signed publication + id: preflight + run: python3 .github/publication.py prepare signed-snapshot-assets publication-package + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 with: name: signed-snapshot-assets - path: signed-snapshot-assets/ + path: publication-package/ + if-no-files-found: error + if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} github-snapshot: name: Update Snapshot Pre-release on GitHub - needs: [publish-snapshot] - # Also runs when publish-snapshot FAILED (not when skipped/cancelled): a Central - # publish-poll timeout reds that job after the artifacts were already uploaded — - # the GitHub pre-release assets must not be lost in that case. - if: ${{ !cancelled() && (needs.publish-snapshot.result == 'success' || needs.publish-snapshot.result == 'failure') }} + needs: + - publish-snapshot + if: ${{ !cancelled() && needs.publish-snapshot.outputs.assets_complete == 'true' }} runs-on: ubuntu-latest - # Serialize the complete upload, tag/notes update and pruning operation. concurrency: group: snapshot-release-${{ github.repository }} cancel-in-progress: false permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 with: name: signed-snapshot-assets - path: snapshot-assets/ - - name: Report unsigned assets (does not block the upload) - # Deliberately NON-blocking, and deliberately BEFORE the upload. Both attach jobs run even - # when their publish job failed, because a Central publish-poll timeout must not cost the - # GitHub assets: if Central is unreachable these are the ONLY way to get the artifacts at - # all. Refusing to attach on a signing failure would defeat exactly that. So annotate here, - # upload regardless, and fail the job afterwards. Assets always land; an unsigned release is - # still loudly red rather than quietly wrong. - # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". - id: signatures - run: python3 .github/report-asset-signatures.py snapshot-assets + path: publication-package/ + digest-mismatch: error + - name: Verify transport integrity and export checked GitHub files + run: python3 .github/publication.py verify publication-package --export-github snapshot-assets - name: Update snapshot assets, source tag and notes env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ASSETS_COMPLETE: ${{ needs.publish-snapshot.outputs.assets_complete }} run: python3 .github/update-snapshot.py snapshot-assets - - name: Fail if anything was attached unsigned - # After the upload on purpose: the assets must exist even when the signature does not. - if: ${{ always() && steps.signatures.outputs.missing != '0' }} - run: | - echo "::error::${{ steps.signatures.outputs.missing }} asset(s) attached without a signature (-1 means none were collected at all)" - exit 1 publish-release: - name: Publish Release to Central - needs: [check-tag, code-style, smoke-jar, shared-files, vmlens] + name: Prepare and verify signed release artifacts + needs: + - check-tag + - code-style + - smoke-jar + - shared-files + - vmlens if: needs.check-tag.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central outputs: - assets_complete: ${{ steps.collect.outputs.complete }} + assets_complete: ${{ steps.preflight.outputs.complete }} permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad with: java-version-file: .java-version distribution: temurin @@ -522,67 +505,108 @@ jobs: server-password: MAVEN_PASSWORD gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - # Informational only (nothing depends on it): logs the effective POM with the same - # profile as the deploy below, so the resolved central-publishing configuration - # (waitUntil/waitMaxTime etc.) is visible for debugging. - name: Show effective POM (debug) - run: mvn --batch-mode --no-transfer-progress -P release help:effective-pom - # -Dmaven.repository.credentialScope=id: see the first deploy step of this file (Maven 3.10). - - name: Deploy release - run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release deploy -DskipTests + run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true help:effective-pom + - name: Stage release + run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - # Runs even when the deploy step failed: a Central publish-poll timeout reds the - # job *after* the bundle was uploaded (and typically published server-side), while - # the signed jars and poms + .asc files already exist in target/ (signing happens at - # verify). Collecting on failure lets the github-release job still attach them. - name: Collect signed artifacts id: collect - if: ${{ !cancelled() }} run: python3 .github/collect-release-assets.py signed-release-assets - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: ${{ !cancelled() }} + - name: Verify and freeze the complete signed publication + id: preflight + run: python3 .github/publication.py prepare signed-release-assets publication-package + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 with: name: signed-release-assets - path: signed-release-assets/ + path: publication-package/ + if-no-files-found: error + if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} github-release: name: Attach Binaries to GitHub Release - needs: [publish-release] - # Also runs when publish-release FAILED (not when skipped/cancelled): a Central - # publish-poll timeout reds that job after the artifacts were already uploaded — - # the GitHub release assets must not be lost in that case. - if: ${{ !cancelled() && (needs.publish-release.result == 'success' || needs.publish-release.result == 'failure') }} + needs: + - publish-release + if: ${{ !cancelled() && needs.publish-release.outputs.assets_complete == 'true' }} runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 with: name: signed-release-assets - path: release-assets/ - - name: Report unsigned assets (does not block the upload) - # Deliberately NON-blocking, and deliberately BEFORE the upload. Both attach jobs run even - # when their publish job failed, because a Central publish-poll timeout must not cost the - # GitHub assets: if Central is unreachable these are the ONLY way to get the artifacts at - # all. Refusing to attach on a signing failure would defeat exactly that. So annotate here, - # upload regardless, and fail the job afterwards. Assets always land; an unsigned release is - # still loudly red rather than quietly wrong. - # See workspace/policies/fat-jar-release-assets.md, "Attach first, then go red". - id: signatures - run: python3 .github/report-asset-signatures.py release-assets + path: publication-package/ + digest-mismatch: error + - name: Verify transport integrity and export checked GitHub files + run: python3 .github/publication.py verify publication-package --export-github release-assets - name: Upload release assets - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 with: files: release-assets/* - - name: Fail if anything was attached unsigned - # After the upload on purpose: the assets must exist even when the signature does not. - if: ${{ always() && steps.signatures.outputs.missing != '0' }} - run: | - echo "::error::${{ steps.signatures.outputs.missing }} asset(s) attached without a signature (-1 means none were collected at all)" - exit 1 + + + central-snapshot: + name: Upload checked snapshot artifacts to Maven Central + needs: + - publish-snapshot + if: ${{ !cancelled() && needs.publish-snapshot.outputs.assets_complete == 'true' }} + runs-on: ubuntu-latest + environment: maven-central + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + with: + java-version-file: .java-version + distribution: temurin + server-id: central + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: signed-snapshot-assets + path: publication-package/ + digest-mismatch: error + - name: Transfer frozen artifacts (no rebuild, signing or validation polling) + run: python3 .github/publication.py upload publication-package + env: + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} + + central-release: + name: Upload checked release artifacts to Maven Central + needs: + - publish-release + if: ${{ !cancelled() && needs.publish-release.outputs.assets_complete == 'true' }} + runs-on: ubuntu-latest + environment: maven-central + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + with: + java-version-file: .java-version + distribution: temurin + server-id: central + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: signed-release-assets + path: publication-package/ + digest-mismatch: error + - name: Transfer frozen artifacts (no rebuild, signing or validation polling) + run: python3 .github/publication.py upload publication-package + env: + MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} diff --git a/docs/publication-preflight.md b/docs/publication-preflight.md new file mode 100644 index 0000000..58094c5 --- /dev/null +++ b/docs/publication-preflight.md @@ -0,0 +1,84 @@ + + +# Signed publication preflight and independent uploads + +## Invariants + +Build and sign every intended output locally. Complete the native, classifier and Android +checks applicable to those outputs. Before any publication-package upload, collect the exact +repository contract and run one complete local gate. Missing files, invalid signatures, +version mismatches or differing Central/GitHub bytes must fail before either publisher runs. + +After the gate, neither publisher compiles, packages nor signs artifacts again. Both use the +same frozen package from GitHub Actions. The Central and GitHub Release jobs depend only on +that handoff, never on each other's success. A Central outage must not hide the GitHub files. +Internal compiler caches, test reports and native-build intermediates are not publication packages. + +## Preparation + +Maven's Central publishing plugin 0.12.0 supports `-DskipUpload=true deploy`: it stages signed +release files under `target/central-staging` or snapshots under `target/central-deferred`, +without a remote upload. This is different from `skipPublishing`, which omits artifacts. +Gradle publications are similarly written to their local staging repository first. + +Preserve the signed Maven files before any later fat-jar rebuild can overwrite `target/`: + +```bash +python3 .github/collect-release-assets.py central-assets --maven-only +# Build and sign additional GitHub-only variants. +python3 .github/collect-release-assets.py signed-release-assets --maven-assets central-assets +python3 .github/publication.py prepare signed-release-assets publication-package +``` + +Use `signed-snapshot-assets` on the snapshot path. A repository without a subsequent rebuild +can collect directly without the intermediate `central-assets` directory. + +The gate checks exact file membership, nonempty artifacts/signatures, cryptographic GPG +verification, and byte equality between Central staging and the corresponding GitHub files. +It freezes `central/` in Maven layout, `github/` as flat release files, and a `manifest.json` +with commit, version, channel, file sizes and SHA-256. `complete=true` is emitted only after +success. The handoff upload is allowed only after that gate, even if an optional provenance +attestation subsequently fails. No incomplete publication package is attached to a release. + +## Repository configuration + +`.github/release-assets.json` remains the source of truth. File rules can set `central: true` +for additional Central publications/classifiers. The default is false, so runnable fat jars +and source archives do not accidentally become Central artifacts. Maven reactor outputs, +configured standalone Maven projects and enabled reactor BOMs are always Central outputs. + +Optional `publication.central_roots` lists repository-relative staging directories. The default +is `["target/{staging}"]`; `{staging}` becomes `central-staging` or `central-deferred` according +to the project version. This supports separate Maven or Gradle staging without repository names +in the shared implementation. Unexpected or missing Central members fail the gate. + +## Independent consumers + +```bash +python3 .github/publication.py verify publication-package --export-github release-assets +python3 .github/publication.py upload publication-package +``` + +Each consumer verifies the complete handoff manifest after download. This is a transport +integrity check, not a second build/signing pass. GitHub publishes only the exported `github/` +files; transport manifests and Central repository directories are not release attachments. + +For releases, the Central consumer forms a Portal bundle from the frozen Central files with +checksums and requests automatic publication. It waits only for HTTP upload acceptance and +records the deployment ID. There is no validation/publication polling. The Maven profiles also +use `autoPublish=true` and `waitUntil=uploaded` for manual deployments. + +For snapshots, a transfer-only Maven POM invokes pinned `maven-deploy-plugin:3.1.4:deploy-file` +executions during `validate`. Maven maintains snapshot timestamps and metadata; signatures are +transferred in the same component request with their proper extensions. There are no source +build phases. For a POM-only component its signature is the primary transfer artifact and the +signed POM is supplied through `pomFile`, avoiding an empty-classifier parameter edge case. + +Local gate failures stop both publication paths. Authentication or transfer failures remain +visible in the affected upload job, but never gate the other publisher. Slow Portal processing +does not fail the build. Upload acceptance is not a claim of successful Central validation; +inspect the recorded deployment in Portal if later processing fails. Re-running a transfer +uses the same frozen package. A network-interrupted GitHub release update is not atomic. diff --git a/pom.xml b/pom.xml index 2b119e4..21e9bc1 100644 --- a/pom.xml +++ b/pom.xml @@ -816,10 +816,8 @@ SPDX-License-Identifier: Apache-2.0 central true - - validated - - 21600 + + uploaded From 0bb3b153cfd07da1f58c89372b4004f1bdacf061 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 19:11:32 +0000 Subject: [PATCH 10/19] Use JDK 25 for builds while preserving all artifact and test targets --- .github/shared-files.sha256 | 6 +++--- .github/workflows/codeql.yml | 8 ++------ .github/workflows/formal-verification.yml | 8 ++++---- .github/workflows/sonarqube.yml | 2 +- .java-version | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 284c318..5c17ee0 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -51,16 +51,16 @@ e8f444ef851bd134330f1505f52b4b0970ef9a62dd76e4c25829241814ba5c73 .github/ISSUE_ 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .mvn/jvm.config.license 46fc025df0fd7b1bdbd84b875652ce9981aec87c2bb49b9263ae8c383cf7ce5a .mvn/settings.xml 8b8abf79508c4f53b2e765b78d6142655631d5ed69b59e95f234416a62e057b6 CITATION.cff?repo -49eda3d2e846e1b5ec1d0f646a1adafb45ef42efbc3a50931df207bf9759ef13 .github/workflows/sonarqube.yml?repo +c69246385983fab707a808b8198f593fef33d3d331a242ddd3265276e65fa257 .github/workflows/sonarqube.yml?repo 9a24b2a67ab8e6f52016093e9b6a89796adea29887b7820accede06cc0c0d170 .github/workflows/publish.yml#code-style -6e2ae11dad0616f66bbb2b6e6556f580bb987fd911d7132aa6bee2bfc7cc7b52 .java-version +64aeb9975f234becd55bb4635e6e2f2da7a6b7bf0a896f0c07763bdfbfb31420 .java-version 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .java-version.license 48cf852de627f1b11f402e408af907ce3e9ec926f6ba11720467645117e92e3a .github/workflows/claude.yml 362d1d169821b8954dc305dac36b8e2437acdbe2921ac42e9887a215663fa666 .github/workflows/claude-code-review.yml d4371cbcd6c5511e9b712599bbadc7b4c2437f3107f5d706a9adccdda24c2c44 .github/workflows/scorecard.yml 480afc0c1e41f3997b195d7297750b43599ba71a7471657aa32bfc9662b6fd30 .github/workflows/reuse.yml 93aedaf67af1c4d63b2502107a51f2f454f2b3b4bc5f2fe892561e12363e57a9 .github/workflows/osv-scanner.yml -ed4231dc5b4ba9357a06974cb61e92b60e3eaac9a293810087efb2933e0cac82 .github/workflows/codeql.yml +2dc51e7f4873bc923d0149c12f1eb1e2752805147f89b359aa7a68e632d48c9c .github/workflows/codeql.yml c9e757f2d0a2f456a08791697a94161c776d9b008a8aa75890da4133c81a99e9 .github/dependabot.yml f26d05f25a154b84d8f281bb92857811749e2225cdbc167362de930c48b532d1 .github/workflows/publish.yml#startgate e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workflows/publish.yml#shared-files diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index da8c6ad..ee8ef70 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,12 +19,8 @@ jobs: security-events: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - # Two JDKs, as in every job that builds a whole reactor: a module with a Java floor above - # .java-version (java-llama.cpp's llama-ffm, Java 22) builds through maven-toolchains-plugin, - # and setup-java writes a toolchains.xml entry per JDK it installs; the second step keeps - # JAVA_HOME on .java-version. Harmless in a repository without such a module. Shared file. - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 - with: { java-version: '25', distribution: temurin } + # The build JDK comes from the shared .java-version. Production and test + # bytecode releases remain explicitly configured in the Maven POMs. - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: { java-version-file: .java-version, distribution: temurin } - uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4 diff --git a/.github/workflows/formal-verification.yml b/.github/workflows/formal-verification.yml index 639dba6..1428c9a 100644 --- a/.github/workflows/formal-verification.yml +++ b/.github/workflows/formal-verification.yml @@ -232,17 +232,17 @@ jobs: run: bash .github/print-host-info.sh - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: - # 21 builds the repo's Java-8 production classes (matching the shipped bytecode); + # 25 builds the repo's Java-8 production classes (matching the shipped bytecode); # 11 builds and runs jpf-core. java-version: | 11 - 21 + 25 distribution: temurin cache: maven - - name: Build Java-8 production classes + collect annotation jars (JDK 21) + - name: Build Java-8 production classes + collect annotation jars (JDK 25) run: | set -euo pipefail - export JAVA_HOME="$JAVA_HOME_21_X64" + export JAVA_HOME="$JAVA_HOME_25_X64" mvn -B --no-transfer-progress -DskipTests -Denforcer.skip=true compile # Copy the compile-scope dependencies (incl. the optional checker-qual/jspecify/ # error-prone annotation jars the production bytecode references) into one directory, diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index f8cc69c..e5fa817 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Set up JDK 21 + - name: Set up build JDK from .java-version uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: java-version-file: .java-version diff --git a/.java-version b/.java-version index aabe6ec..7273c0f 100644 --- a/.java-version +++ b/.java-version @@ -1 +1 @@ -21 +25 From 8a55fbf6a5cdf26ea7f14d696c5e052e8e7a2f65 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 19:14:33 +0000 Subject: [PATCH 11/19] Run dependency submission with the repository build JDK --- .github/shared-files.sha256 | 1 + .github/workflows/dependency-submission.yml | 30 +++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .github/workflows/dependency-submission.yml diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 5c17ee0..b21facb 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -87,3 +87,4 @@ c8d911dfd539016ad404460a3879d0b1d9635ba5c4e0843e9ce98acd5edb3f17 .github/buildc b6a8429621c3aa5af71c4d4a63bb729cb220c858b74ba9f3d1ba14813602fddb docs/publication-preflight.md b3cb68727cc2b0a4e6831b3b9301b6e54420dc6c52aa068dcd8b2b38a6c2308b .github/workflows/publish.yml#central-snapshot 5bf3977c27f1a5ae7f9894cc6e1b26b014ea3ee6bb854fe89daa9cbb26785a23 .github/workflows/publish.yml#central-release +ae21a7de07e232c32ab5a370fc1c153ba1fb8534ae722ed61770a15903553229 .github/workflows/dependency-submission.yml diff --git a/.github/workflows/dependency-submission.yml b/.github/workflows/dependency-submission.yml new file mode 100644 index 0000000..3b73dd5 --- /dev/null +++ b/.github/workflows/dependency-submission.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 + +name: Maven Dependency Submission +on: + push: + workflow_dispatch: +permissions: + contents: read +jobs: + submit: + name: Submit Maven dependencies with the build JDK + runs-on: ubuntu-latest + # The dependency-submission API requires contents:write. This workflow runs on + # repository pushes, never on pull_request or pull_request_target events. + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 + with: + distribution: temurin + java-version-file: .java-version + cache: maven + # Same action revision as GitHub's automatic Maven submission, with explicit JDK setup. + - uses: actions/maven-dependency-submission-action@1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db + with: + maven-args: -Denforcer.skip=true -DskipTests From f05fff3ea6cf06d714085b0aa6fd8036343d2dd4 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sat, 24 Oct 2026 21:00:17 +0000 Subject: [PATCH 12/19] Unify publication finalization and independent provenance attestation --- .../actions/finalize-publication/action.yml | 49 ++++++++++ .github/buildcheck/tests/test_publication.py | 35 +++++-- .../buildcheck/tests/test_releaseassets.py | 14 ++- .github/release-gate-exemptions.txt | 2 + .github/shared-files.sha256 | 9 +- .github/workflows/publish.yml | 91 ++++++++++++++----- docs/publication-preflight.md | 39 ++++++-- 7 files changed, 184 insertions(+), 55 deletions(-) create mode 100644 .github/actions/finalize-publication/action.yml diff --git a/.github/actions/finalize-publication/action.yml b/.github/actions/finalize-publication/action.yml new file mode 100644 index 0000000..410a550 --- /dev/null +++ b/.github/actions/finalize-publication/action.yml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# SPDX-License-Identifier: MIT OR Apache-2.0 + +name: Finalize signed publication +description: Collect, verify and secure the complete signed package before any publication consumer runs. +inputs: + channel: + description: Publication channel (snapshot or release). + required: true + maven-assets: + description: Optional directory containing Maven outputs preserved before a rebuild. + required: false + default: '' +outputs: + complete: + description: True only after the local gate AND the Actions handoff upload succeed. + value: ${{ steps.ready.outputs.complete }} +runs: + using: composite + steps: + - name: Collect signed artifacts + id: collect + shell: bash + env: + CHANNEL: ${{ inputs.channel }} + MAVEN_ASSETS: ${{ inputs.maven-assets }} + run: | + set -euo pipefail + case "$CHANNEL" in snapshot|release) ;; *) echo "::error::Invalid publication channel"; exit 1 ;; esac + args=("signed-${CHANNEL}-assets") + if [ -n "$MAVEN_ASSETS" ]; then args+=(--maven-assets "$MAVEN_ASSETS"); fi + python3 .github/collect-release-assets.py "${args[@]}" + - name: Verify and freeze the complete signed publication + id: preflight + shell: bash + env: + CHANNEL: ${{ inputs.channel }} + run: python3 .github/publication.py prepare "signed-${CHANNEL}-assets" publication-package + - name: Secure the checked publication package + id: handoff + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: signed-${{ inputs.channel }}-assets + path: publication-package/ + if-no-files-found: error + - name: Release the secured package to independent consumers + id: ready + shell: bash + run: echo 'complete=true' >> "$GITHUB_OUTPUT" diff --git a/.github/buildcheck/tests/test_publication.py b/.github/buildcheck/tests/test_publication.py index 9fd5f3b..b036c0d 100644 --- a/.github/buildcheck/tests/test_publication.py +++ b/.github/buildcheck/tests/test_publication.py @@ -110,17 +110,32 @@ def test_pom_only_component_transfers_signature_and_pom_together(self): def test_publishers_depend_only_on_the_same_checked_handoff(self): jobs = workflow.parse((Path(REPO) / '.github/workflows/publish.yml').read_text()) + action = (Path(REPO) / '.github/actions/finalize-publication/action.yml').read_text() + self.assertIn('steps.ready.outputs.complete', action) + order = ['id: collect', 'id: preflight', 'id: handoff', 'id: ready'] + self.assertEqual([action.index(item) for item in order], sorted(action.index(item) for item in order)) + self.assertIn('if-no-files-found: error', action) + self.assertNotIn('always()', action) + self.assertNotIn('continue-on-error', action) for channel in ['snapshot', 'release']: prep = jobs['publish-' + channel] - central = jobs['central-' + channel] - github_name = 'github-snapshot' if channel == 'snapshot' else ('github-release' if 'github-release' in jobs else 'github-release-signed') - github = jobs[github_name] - self.assertEqual(central.needs, ['publish-' + channel]) - self.assertEqual(github.needs, ['publish-' + channel]) - self.assertIn('steps.preflight.outputs.complete', prep.text) - self.assertLess(prep.text.index('publication.py prepare'), prep.text.index('uses: actions/upload-artifact@')) + self.assertIn('steps.finalize.outputs.complete', prep.text) + self.assertIn('uses: ./.github/actions/finalize-publication', prep.text) + self.assertIn('channel: ' + channel, prep.text) + self.assertNotIn('publication.py prepare', prep.text) + self.assertNotIn('attestations: write', prep.text) + self.assertNotIn('id-token: write', prep.text) self.assertIn('-DskipUpload=true deploy', prep.text) self.assertNotIn('publishAllPublicationsToCentralSnapshotsRepository', prep.text) - self.assertIn('publication.py upload publication-package', central.text) - self.assertIn('publication.py verify publication-package', github.text) - self.assertNotIn('Fail if anything', github.text) + for consumer in ['central', 'github', 'attest']: + job = jobs[consumer + '-' + channel] + self.assertEqual(job.needs, ['publish-' + channel]) + self.assertIn("needs.publish-" + channel + ".outputs.assets_complete == 'true'", job.text) + self.assertIn('digest-mismatch: error', job.text) + self.assertIn('publication.py upload publication-package', jobs['central-' + channel].text) + self.assertIn('publication.py verify publication-package', jobs['github-' + channel].text) + attest = jobs['attest-' + channel].text + self.assertLess(attest.index('publication.py verify'), attest.index('uses: actions/attest-build-provenance@')) + self.assertIn('subject-path: publication-package/github/*', attest) + self.assertIn('id-token: write', attest) + self.assertIn('attestations: write', attest) diff --git a/.github/buildcheck/tests/test_releaseassets.py b/.github/buildcheck/tests/test_releaseassets.py index 8908fbc..5ec9bae 100644 --- a/.github/buildcheck/tests/test_releaseassets.py +++ b/.github/buildcheck/tests/test_releaseassets.py @@ -187,16 +187,14 @@ def test_current_repository_workflow_passes_collection_result_and_checks_both_up jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) for kind in ("snapshot", "release"): job = jobs[f"publish-{kind}"].text - self.assertIn(f"python3 .github/collect-release-assets.py signed-{kind}-assets", job) - self.assertIn("assets_complete: $" + "{{ steps.preflight.outputs.complete }}", job) - self.assertIn(" id: collect", job) - attach = jobs["github-snapshot" if kind == "snapshot" else - "github-release" if "github-release" in jobs else "github-release-signed"].text + self.assertIn("uses: ./.github/actions/finalize-publication", job) + self.assertIn(f"channel: {kind}", job) + self.assertIn("assets_complete: $" + "{{ steps.finalize.outputs.complete }}", job) + self.assertIn(" id: finalize", job) + attach = jobs[f"github-{kind}"].text self.assertIn(f"python3 .github/publication.py verify publication-package --export-github {kind}-assets", attach) self.assertNotIn("Fail if anything", attach) - if configuration(REPO).get("maven", {}).get("cyclonedx", False): - for kind in ("snapshot", "release"): - self.assertIn(f"signed-{kind}-assets/*.json", jobs[f"publish-{kind}"].text) + self.assertIn("subject-path: publication-package/github/*", jobs[f"attest-{kind}"].text) def test_current_repository_contract_collects_exact_complete_and_partial_fixture(self): # Exercise this checkout's real POMs and JSON, not just a hand-written generic fixture. diff --git a/.github/release-gate-exemptions.txt b/.github/release-gate-exemptions.txt index bfbdfa7..f428b2b 100644 --- a/.github/release-gate-exemptions.txt +++ b/.github/release-gate-exemptions.txt @@ -16,3 +16,5 @@ verify-signing-key: red by design where the secret is withheld (forks, other con verify-signing-key-gradle: same as verify-signing-key, for the Gradle/BouncyCastle signing path central-snapshot: Independent post-preflight Central transfer; must not gate the GitHub publication path. central-release: Independent post-preflight Central transfer; must not gate the GitHub publication path. +attest-snapshot: Independent provenance after the secured handoff; cannot gate either publication consumer. +attest-release: Independent provenance after the secured handoff; cannot gate either publication consumer. diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index b21facb..165f326 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -72,7 +72,7 @@ a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workfl 52f1f7c99a604f40fb3f3685909c060b2c270e3f3dd5d28f3119be4b3f5d4052 .github/workflows/publish.yml#github-release ed1ba202b9d81a00893931a090112bfb28770a514030a5c08d5441c83369f8b7 .github/buildcheck/releaseassets.py bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py -8bc93d2ca737d7641abdd6311444233d9603c188dfd9d89d785b70cb92fb0f93 .github/buildcheck/tests/test_releaseassets.py +08828c6a69f498adae4bf4d9bc8b4d70ccfe21b9185e29512e3807dd6613e7a5 .github/buildcheck/tests/test_releaseassets.py 8b6ae70f43560a0095b6f16012b80ed694b07633c1cbde94dbe8c21c529461ea .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py @@ -83,8 +83,11 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py 4d16ef3f5f4fa5fa39d191b030e1cbff3c943c67b33ae9b392537e4b08f35004 .github/buildcheck/publication.py 9180fac217c75b3e11f67e244b9171f447d2c21665b1802574b57b97019f7173 .github/publication.py -c8d911dfd539016ad404460a3879d0b1d9635ba5c4e0843e9ce98acd5edb3f17 .github/buildcheck/tests/test_publication.py -b6a8429621c3aa5af71c4d4a63bb729cb220c858b74ba9f3d1ba14813602fddb docs/publication-preflight.md +6e142c05ddc3c1d4cc31789be5e2040393318c6ae79d99d01eec72e72dc94232 .github/buildcheck/tests/test_publication.py +7b96c3db888769146900b123c81d4edf5e8ef4061fa9f547abcbd02744a3a4d5 docs/publication-preflight.md b3cb68727cc2b0a4e6831b3b9301b6e54420dc6c52aa068dcd8b2b38a6c2308b .github/workflows/publish.yml#central-snapshot 5bf3977c27f1a5ae7f9894cc6e1b26b014ea3ee6bb854fe89daa9cbb26785a23 .github/workflows/publish.yml#central-release ae21a7de07e232c32ab5a370fc1c153ba1fb8534ae722ed61770a15903553229 .github/workflows/dependency-submission.yml +68b7bc3d8deb1696e1c69a2c7200c662737c4be74dce09fb1aae4e12fb55c284 .github/actions/finalize-publication/action.yml +3779cc3b48f923b3b63ea265bbfccc5e9c3709b9db230d8c13534143784c1264 .github/workflows/publish.yml#attest-snapshot +16c54b3c6c516e300974e4fe2fd5264af373e45f9bac3ecba553ac8a33b9e5cd .github/workflows/publish.yml#attest-release diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 74efadb..743f093 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -409,7 +409,9 @@ jobs: runs-on: ubuntu-latest environment: maven-central outputs: - assets_complete: ${{ steps.preflight.outputs.complete }} + assets_complete: ${{ steps.finalize.outputs.complete }} + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad @@ -437,18 +439,11 @@ jobs: run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests env: MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - - name: Collect signed artifacts - id: collect - run: python3 .github/collect-release-assets.py signed-snapshot-assets - - name: Verify and freeze the complete signed publication - id: preflight - run: python3 .github/publication.py prepare signed-snapshot-assets publication-package - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + - name: Finalize and secure the signed publication + id: finalize + uses: ./.github/actions/finalize-publication with: - name: signed-snapshot-assets - path: publication-package/ - if-no-files-found: error - if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} + channel: snapshot github-snapshot: name: Update Snapshot Pre-release on GitHub @@ -490,9 +485,9 @@ jobs: runs-on: ubuntu-latest environment: maven-central outputs: - assets_complete: ${{ steps.preflight.outputs.complete }} + assets_complete: ${{ steps.finalize.outputs.complete }} permissions: - contents: write + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad @@ -511,18 +506,11 @@ jobs: run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests env: MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - - name: Collect signed artifacts - id: collect - run: python3 .github/collect-release-assets.py signed-release-assets - - name: Verify and freeze the complete signed publication - id: preflight - run: python3 .github/publication.py prepare signed-release-assets publication-package - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + - name: Finalize and secure the signed publication + id: finalize + uses: ./.github/actions/finalize-publication with: - name: signed-release-assets - path: publication-package/ - if-no-files-found: error - if: ${{ !cancelled() && steps.preflight.outcome == 'success' }} + channel: release github-release: name: Attach Binaries to GitHub Release @@ -610,3 +598,56 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} + + attest-snapshot: + name: Attest the checked snapshot publication + needs: + - publish-snapshot + if: ${{ !cancelled() && needs.publish-snapshot.outputs.assets_complete == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: signed-snapshot-assets + path: publication-package/ + digest-mismatch: error + - name: Verify the secured package before attesting + run: python3 .github/publication.py verify publication-package + - name: Attest the frozen artifacts and detached signatures + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 + with: + subject-path: publication-package/github/* + + + attest-release: + name: Attest the checked release publication + needs: + - publish-release + if: ${{ !cancelled() && needs.publish-release.outputs.assets_complete == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + attestations: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + name: signed-release-assets + path: publication-package/ + digest-mismatch: error + - name: Verify the secured package before attesting + run: python3 .github/publication.py verify publication-package + - name: Attest the frozen artifacts and detached signatures + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 + with: + subject-path: publication-package/github/* diff --git a/docs/publication-preflight.md b/docs/publication-preflight.md index 58094c5..71085dc 100644 --- a/docs/publication-preflight.md +++ b/docs/publication-preflight.md @@ -13,8 +13,8 @@ repository contract and run one complete local gate. Missing files, invalid sign version mismatches or differing Central/GitHub bytes must fail before either publisher runs. After the gate, neither publisher compiles, packages nor signs artifacts again. Both use the -same frozen package from GitHub Actions. The Central and GitHub Release jobs depend only on -that handoff, never on each other's success. A Central outage must not hide the GitHub files. +same frozen package from GitHub Actions. The Central, GitHub Release and provenance attestation jobs depend only on +that secured handoff, never on each other's success. A Central outage must not hide the GitHub files. Internal compiler caches, test reports and native-build intermediates are not publication packages. ## Preparation @@ -29,19 +29,30 @@ Preserve the signed Maven files before any later fat-jar rebuild can overwrite ` ```bash python3 .github/collect-release-assets.py central-assets --maven-only # Build and sign additional GitHub-only variants. -python3 .github/collect-release-assets.py signed-release-assets --maven-assets central-assets -python3 .github/publication.py prepare signed-release-assets publication-package ``` -Use `signed-snapshot-assets` on the snapshot path. A repository without a subsequent rebuild -can collect directly without the intermediate `central-assets` directory. +Finish both publication channels with the shared, checksummed composite action: + +```yaml +- name: Finalize and secure the signed publication + id: finalize + uses: ./.github/actions/finalize-publication + with: + channel: release # snapshot for the rolling snapshot + maven-assets: central-assets # omit when no outputs need preserving +``` + +The action collects the exact contract, calls `publication.py prepare`, and uploads the frozen +`publication-package/` as `signed-release-assets` or `signed-snapshot-assets`. +The preparation job exports `assets_complete: ${{ steps.finalize.outputs.complete }}`. The gate checks exact file membership, nonempty artifacts/signatures, cryptographic GPG verification, and byte equality between Central staging and the corresponding GitHub files. It freezes `central/` in Maven layout, `github/` as flat release files, and a `manifest.json` -with commit, version, channel, file sizes and SHA-256. `complete=true` is emitted only after -success. The handoff upload is allowed only after that gate, even if an optional provenance -attestation subsequently fails. No incomplete publication package is attached to a release. +with commit, version, channel, file sizes and SHA-256. The composite's `complete=true` output +is emitted only after both the local gate and the Actions handoff upload succeed. A failed +collection, gate or handoff upload cannot release a consumer. No incomplete publication package +is attached to a release. ## Repository configuration @@ -66,6 +77,16 @@ Each consumer verifies the complete handoff manifest after download. This is a t integrity check, not a second build/signing pass. GitHub publishes only the exported `github/` files; transport manifests and Central repository directories are not release attachments. +All four repositories use identical `central-snapshot`, `central-release`, `github-snapshot`, +`github-release`, `attest-snapshot` and `attest-release` jobs, tracked as shared workflow fragments. +Each needs only the matching preparation job and requires its successful handoff output. + +The attestation job verifies the downloaded manifest first, then attests every frozen file in +`publication-package/github/*`, including detached signatures and configured BOMs. It alone among +the finalization/attestation jobs requests `id-token: write` and `attestations: write`; it needs no +signing key. Attestation failure remains visible but cannot prevent either publisher from running. +Existing producer-specific attestations for native build outputs remain applicable independently. + For releases, the Central consumer forms a Portal bundle from the frozen Central files with checksums and requests automatic publication. It waits only for HTTP upload acceptance and records the deployment ID. There is no validation/publication polling. The Maven profiles also From 43282d4f15bfa6b5458e49fdfc8024ac9b881268 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sun, 25 Oct 2026 11:31:31 +0100 Subject: [PATCH 13/19] Check publications against their tag and align the docs Publication gate (shared byte for byte with java-llama.cpp and the other siblings): - check-tag / check-snapshot run `publication.py check-ref`: a release publishes only from the tag v of a non-SNAPSHOT version, a snapshot only a -SNAPSHOT version and never from a tag; `prepare --channel` repeats both. The docs already claimed this check; nothing did it. - Unknown files in a Central staging directory fail instead of being dropped; Gradle timestamped snapshot names map back to -SNAPSHOT; Central signatures pair by name; `prepare` honours --config; names GitHub would rename are refused. - update-snapshot.py refuses an incomplete collection and a versioned tag before any change. - central-snapshot is serialized and skips a commit another run already superseded. - report-asset-signatures.py is gone; the report job no longer submits a second dependency graph and runs with contents: read; dependency-submission.yml runs on pushes to main only. - Version comments on every SHA-pinned action; tests for the transfers, the Central rules, several roots and the ref check. streambuffer: CLAUDE.md describes the publication flow and the shared jobs as they are; THIRD_PARTY.md names the optional annotation dependencies instead of "no runtime dependencies" and drops the fat-jar collector text that does not apply here. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VszvDmRRPP1UcwozsUMY1L --- .../actions/finalize-publication/action.yml | 2 +- .github/buildcheck/__init__.py | 9 +- .github/buildcheck/publication.py | 133 ++++++-- .github/buildcheck/releaseassets.py | 5 +- .github/buildcheck/snapshot.py | 21 +- .github/buildcheck/tests/test_publication.py | 288 +++++++++++++++++- .../buildcheck/tests/test_releaseassets.py | 16 +- .github/buildcheck/tests/test_snapshot.py | 35 ++- .github/report-asset-signatures.py | 33 -- .github/shared-files.sha256 | 37 ++- .github/workflows/dependency-submission.yml | 3 + .github/workflows/publish.yml | 80 +++-- CLAUDE.md | 33 +- THIRD_PARTY.md | 14 +- docs/publication-preflight.md | 29 +- 15 files changed, 571 insertions(+), 167 deletions(-) delete mode 100644 .github/report-asset-signatures.py diff --git a/.github/actions/finalize-publication/action.yml b/.github/actions/finalize-publication/action.yml index 410a550..b6399c9 100644 --- a/.github/actions/finalize-publication/action.yml +++ b/.github/actions/finalize-publication/action.yml @@ -35,7 +35,7 @@ runs: shell: bash env: CHANNEL: ${{ inputs.channel }} - run: python3 .github/publication.py prepare "signed-${CHANNEL}-assets" publication-package + run: python3 .github/publication.py prepare "signed-${CHANNEL}-assets" publication-package --channel "$CHANNEL" - name: Secure the checked publication package id: handoff uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/.github/buildcheck/__init__.py b/.github/buildcheck/__init__.py index add4f0f..5beaff1 100644 --- a/.github/buildcheck/__init__.py +++ b/.github/buildcheck/__init__.py @@ -3,10 +3,11 @@ # SPDX-License-Identifier: MIT OR Apache-2.0 """Build checks as a library, so buildcheck/tests can test them. Standard library only. -Two kinds of module live here. workflow.py, releasegate.py, sharedfiles.py, versions.py and -runscripts.py (with their tests and the check-*.py entry points next to this package) are kept -BYTE-IDENTICAL in java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer: each repository -lists them in .github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). +Two kinds of module live here. workflow.py, releasegate.py, sharedfiles.py, versions.py, +runscripts.py, releaseassets.py, publication.py and snapshot.py (with their tests and the *.py +entry points next to this package) are kept BYTE-IDENTICAL in java-llama.cpp, +BitcoinAddressFinder, srcmorph and streambuffer: each repository lists them in +.github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). Every other module is the repository's own. Run the tests from the repository root: diff --git a/.github/buildcheck/publication.py b/.github/buildcheck/publication.py index a8d0db5..3880f8e 100644 --- a/.github/buildcheck/publication.py +++ b/.github/buildcheck/publication.py @@ -15,10 +15,20 @@ import zipfile from .releaseassets import configuration, projects, signature_gaps, _classifiers, _format, _inside +from .snapshot import GitHub PORTAL = "https://central.sonatype.com/api/v1/publisher/upload" SNAPSHOTS = "https://central.sonatype.com/repository/maven-snapshots/" DEPLOY_PLUGIN = "3.1.4" +# The name of the shared central-snapshot job; superseded() looks for it in other runs. +CENTRAL_SNAPSHOT_JOB = "Upload checked snapshot artifacts to Maven Central" +# Publication payload in a Central staging directory. Repository metadata and checksums are +# regenerated by the transfer; anything else in a staging directory fails the gate. +PAYLOAD = re.compile(r".+\.(?:pom|jar|aar|json)(?:\.asc)?") +REGENERATED = re.compile(r"maven-metadata[\w.-]*\.xml|.+\.(?:md5|sha1|sha256|sha512)|\.index") +# GitHub renames release assets with other characters; a renamed asset would no longer match +# the publication manifest, and update-snapshot.py would prune it as obsolete. +ASSET_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") def digest(path): @@ -39,6 +49,30 @@ def safe_path(root, name): return target +def check_ref(version, ref, channel=None): + """A release publishes only from the tag v; a snapshot never from a tag.""" + actual = "snapshot" if version.endswith("-SNAPSHOT") else "release" + if channel is not None and channel != actual: + raise ValueError(f"{channel} publication of {actual} version {version}") + if ref is None: + return actual + tag = ref[len("refs/tags/"):] if ref.startswith("refs/tags/") else None + if actual == "snapshot" and tag is not None: + raise ValueError(f"tag {tag} names SNAPSHOT version {version}; a release tag needs a release version") + if actual == "release" and tag != "v" + version: + raise ValueError(f"release version {version} publishes only from tag v{version}, not {ref}") + return actual + + +def central_name(name, artifact, version): + """Undo a timestamped snapshot name (Gradle writes one even to a local file repository).""" + if not version.endswith("-SNAPSHOT"): + return name + prefix = f"{artifact}-{version[:-len('-SNAPSHOT')]}-" + match = re.fullmatch(re.escape(prefix) + r"\d{8}\.\d{6}-\d+(.*)", name) + return f"{artifact}-{version}{match[1]}" if match else name + + def expected_central(root, config, modules, *, only_central=True): names = set() for _, artifact, version, packaging, reactor in modules: @@ -59,19 +93,23 @@ def expected_central(root, config, modules, *, only_central=True): return names -def prepare(root, assets, destination): - """Gate both publishers on exact membership, valid signatures and frozen checksums.""" +def prepare(root, assets, destination, *, config_path=None, channel=None, ref=None): + """Gate both publishers on exact membership, valid signatures and frozen checksums. + + channel is the one the caller publishes to; ref defaults to GITHUB_REF ("": no ref check). + """ root, assets, destination = Path(root).resolve(), Path(assets).resolve(), Path(destination).resolve() if destination.exists(): raise ValueError("publication destination must not exist") missing, errors = signature_gaps(assets) if missing: raise ValueError("incomplete publication: " + "; ".join(errors)) - config = configuration(root) + config = configuration(root, config_path) publication = config.get("publication", {}) modules = projects(root, config.get("maven", {}).get("standalone", [])) version = modules[0][2] - channel = "snapshot" if version.endswith("-SNAPSHOT") else "release" + ref = os.environ.get("GITHUB_REF") if ref is None else ref + channel = check_ref(version, ref or None, channel) staging = "central-deferred" if channel == "snapshot" else "central-staging" roots = publication.get("central_roots", ["target/{staging}"]) central = {} @@ -84,30 +122,43 @@ def prepare(root, assets, destination): continue relative = path.relative_to(folder).as_posix() safe_path(folder, relative) - if not re.search(r"\.(?:pom|jar|aar|json)(?:\.asc)?$", path.name): - continue # Maven metadata/checksums are regenerated, never mistaken for payload. + if not PAYLOAD.fullmatch(path.name): + if REGENERATED.fullmatch(path.name): + continue # Maven metadata/checksums are regenerated, never mistaken for payload. + raise ValueError("unexpected file in Central staging: " + relative) parts = PurePosixPath(relative).parts if len(parts) < 4 or parts[-2] != version: raise ValueError("unexpected Central coordinates: " + relative) + relative = PurePosixPath(*parts[:-1], central_name(parts[-1], parts[-3], version)).as_posix() if relative in central and digest(central[relative]) != digest(path): raise ValueError("conflicting Central input: " + relative) central[relative] = path + names = [PurePosixPath(relative).name for relative in central] + if len(set(names)) != len(names): + raise ValueError("one file name under two Central coordinates: " + + str(sorted({name for name in names if names.count(name) > 1}))) expected = expected_central(root, config, modules) - actual = {path.name for path in central.values() if not path.name.endswith(".asc")} - if actual != expected or len(central) != 2 * len(expected): - raise ValueError("Central membership mismatch; missing=" + str(sorted(expected - actual)) - + "; unexpected=" + str(sorted(actual - expected))) + # Each artifact with its own signature, and no signature without its artifact. + wanted = expected | {name + ".asc" for name in expected} + if set(names) != wanted: + raise ValueError("Central membership mismatch; missing=" + str(sorted(wanted - set(names))) + + "; unexpected=" + str(sorted(set(names) - wanted))) github_expected = expected_central(root, config, modules, only_central=False) github_actual = {p.name for p in assets.iterdir() if not p.name.endswith(".asc")} if github_actual != github_expected: - raise ValueError("GitHub membership does not match the release contract") + raise ValueError("GitHub membership does not match the release contract; missing=" + + str(sorted(github_expected - github_actual)) + + "; unexpected=" + str(sorted(github_actual - github_expected))) # Require the Central pair to match GitHub; verify signatures on the frozen copy. for path in sorted(assets.iterdir()): if path.is_symlink() or not path.is_file(): raise ValueError("non-flat publication input: " + str(path)) - for name, path in central.items(): - if not (assets / path.name).is_file() or digest(assets / path.name) != digest(path): - raise ValueError("Central/GitHub bytes differ: " + name) + if not ASSET_NAME.fullmatch(path.name): + raise ValueError("GitHub would rename release asset: " + path.name) + for relative, path in central.items(): + name = PurePosixPath(relative).name + if not (assets / name).is_file() or digest(assets / name) != digest(path): + raise ValueError("Central/GitHub bytes differ: " + relative) # Build a temporary package, then promote it locally only after all checks succeed. destination.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix="publication-", dir=destination.parent) as tmp: @@ -222,11 +273,44 @@ def child(parent, name, text): ET.ElementTree(root).write(destination, encoding="utf-8", xml_declaration=True) -def upload(package): +def superseded(repository, sha, run_id, github=None): + """True when another publish run already transferred the snapshot of a newer commit. + + The central-snapshot jobs of all runs share one concurrency group, so a newer run's transfer + either finished before this check or starts after this transfer: Maven Central serves the + snapshot transferred last as the latest one. + """ + gh = github or GitHub() + repo = f"repos/{repository}" + workflow = gh.api("GET", f"{repo}/actions/runs/{run_id}")["workflow_id"] + runs = gh.api("GET", f"{repo}/actions/workflows/{workflow}/runs?event=workflow_dispatch&per_page=30") + for run in runs["workflow_runs"]: + if run["id"] == int(run_id) or run["head_sha"] == sha: + continue + jobs, page = [], 1 + while True: + batch = gh.api("GET", f"{repo}/actions/runs/{run['id']}/jobs?filter=latest&per_page=100&page={page}") + jobs.extend(batch["jobs"]) + if len(batch["jobs"]) < 100: + break + page += 1 + if not any(job["name"] == CENTRAL_SNAPSHOT_JOB and job["conclusion"] == "success" for job in jobs): + continue + if gh.api("GET", f"{repo}/compare/{sha}...{run['head_sha']}")["status"] == "ahead": + print(f"Run {run['id']} already transferred the snapshot of newer commit {run['head_sha']}.") + return True + return False + + +def upload(package, *, skip_superseded=False, github=None): """Transfer frozen files, accepting Portal upload without validation polling.""" package = Path(package).resolve() manifest = verify(package) if manifest["channel"] == "snapshot": + if skip_superseded and superseded(os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_SHA"], + os.environ["GITHUB_RUN_ID"], github): + print("Skipping the transfer of this older snapshot.") + return with tempfile.TemporaryDirectory(prefix="snapshot-transfer-") as tmp: pom = Path(tmp) / "pom.xml" snapshot_pom(package, pom) @@ -271,17 +355,30 @@ def upload(package): def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) + check_parser = commands.add_parser("check-ref", help="check the project version against GITHUB_REF") + check_parser.add_argument("--channel", choices=("snapshot", "release"), required=True) + check_parser.add_argument("--config", type=Path, help="JSON contract (default: .github/release-assets.json)") prepare_parser = commands.add_parser("prepare") prepare_parser.add_argument("assets", type=Path) prepare_parser.add_argument("package", type=Path) + prepare_parser.add_argument("--channel", choices=("snapshot", "release")) + prepare_parser.add_argument("--config", type=Path, help="JSON contract (default: .github/release-assets.json)") verify_parser = commands.add_parser("verify") verify_parser.add_argument("package", type=Path) verify_parser.add_argument("--export-github", type=Path) upload_parser = commands.add_parser("upload") upload_parser.add_argument("package", type=Path) + upload_parser.add_argument("--skip-superseded", action="store_true", + help="skip a snapshot when another run transferred a newer commit") args = parser.parse_args() - if args.command == "prepare": - prepare(Path(__file__).resolve().parents[2], args.assets, args.package) + root = Path(__file__).resolve().parents[2] + if args.command == "check-ref": + config = configuration(root, args.config) + version = projects(root, config.get("maven", {}).get("standalone", []))[0][2] + check_ref(version, os.environ["GITHUB_REF"], args.channel) + print(f"Version {version} publishes as {args.channel} from {os.environ['GITHUB_REF']}") + elif args.command == "prepare": + prepare(root, args.assets, args.package, config_path=args.config, channel=args.channel) elif args.command == "verify": verify(args.package) if args.export_github: @@ -289,4 +386,4 @@ def main(): raise ValueError("GitHub export destination must not exist") shutil.copytree(args.package / "github", args.export_github) else: - upload(args.package) + upload(args.package, skip_superseded=args.skip_superseded) diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py index de10377..63f2501 100644 --- a/.github/buildcheck/releaseassets.py +++ b/.github/buildcheck/releaseassets.py @@ -173,9 +173,10 @@ def collect(root, destination, config_path=None, *, maven_only=False, maven_asse raise ValueError(f"asset destination must be empty: {destination}") for source, name in expected: for suffix in ("", ".asc"): - path = Path(str(source) + suffix) + path = Path(str(source) + suffix).resolve() if not path.is_file() or path.stat().st_size == 0: - errors.append(f"missing or empty release output: {path}") + shown = path.relative_to(root) if path.is_relative_to(root) else path + errors.append(f"missing or empty release output: {shown.as_posix()}") continue shutil.copyfile(path, destination / (name + suffix)) return errors diff --git a/.github/buildcheck/snapshot.py b/.github/buildcheck/snapshot.py index 0c13dbb..4374cb8 100644 --- a/.github/buildcheck/snapshot.py +++ b/.github/buildcheck/snapshot.py @@ -5,8 +5,7 @@ The workflow serializes this entire operation with a job concurrency group. The ancestry/run checks also reject an older build which finishes after a newer build has already published. -Available artifacts are uploaded even after an incomplete collection; only complete uploads -advance the source tag/notes and prune obsolete assets. +An incomplete collection changes nothing: no upload, no tag or notes update, no prune. """ import json @@ -16,6 +15,9 @@ from .releaseassets import signature_gaps +# GitHub renames release assets with other characters; a renamed asset would be pruned as obsolete. +ASSET_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + class GitHub: def command(self, *args): @@ -41,6 +43,8 @@ def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, raise ValueError("invalid repository or published commit") if not re.fullmatch(r"[\w][\w.-]*", tag) or ".." in tag or tag.endswith((".", ".lock")): raise ValueError("invalid rolling release tag") + if re.match(r"v\d", tag): + raise ValueError("a versioned release is never a rolling snapshot: " + tag) run_id, attempt = int(run_id), int(attempt) if run_id < 1 or attempt < 1: raise ValueError("run ID and attempt must be positive") @@ -51,7 +55,13 @@ def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, if not artifacts: raise ValueError("refusing to update snapshot with signatures but no artifacts") names = {path.name for path in files} - complete = complete and signature_gaps(assets_dir)[0] == 0 + renamed = sorted(name for name in names if not ASSET_NAME.fullmatch(name)) + if renamed: + raise ValueError("GitHub would rename release assets: " + ", ".join(renamed)) + missing, errors = signature_gaps(assets_dir) + if not complete or missing: + raise ValueError("refusing to update snapshot from an incomplete collection; existing " + "snapshot metadata and assets were retained" + "".join("; " + e for e in errors)) gh = github or GitHub() repo = f"repos/{repository}" ref = gh.api("GET", f"{repo}/git/ref/tags/{tag}", optional=True) @@ -87,12 +97,9 @@ def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, ref = gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) release = gh.api("POST", f"{repo}/releases", { "tag_name": tag, "target_commitish": sha, "name": title, - "prerelease": True, "body": body if complete else body + "\n\nIncomplete artifact collection." + "prerelease": True, "body": body }) gh.command("release", "upload", tag, *files, "--repo", repository, "--clobber") - if not complete: - raise ValueError("available files uploaded, but collection is incomplete; " - "existing snapshot metadata and assets were retained") if ref: gh.api("PATCH", f"{repo}/git/refs/tags/{tag}", {"sha": sha, "force": True}) diff --git a/.github/buildcheck/tests/test_publication.py b/.github/buildcheck/tests/test_publication.py index b036c0d..23438ba 100644 --- a/.github/buildcheck/tests/test_publication.py +++ b/.github/buildcheck/tests/test_publication.py @@ -2,18 +2,22 @@ # SPDX-License-Identifier: MIT OR Apache-2.0 """Publication gate and transport invariants; no signing key or remote writes.""" import json +import os from pathlib import Path +import subprocess import tempfile import unittest from unittest.mock import patch import xml.etree.ElementTree as ET +import zipfile -from buildcheck.publication import prepare, verify, snapshot_pom +from buildcheck.publication import (CENTRAL_SNAPSHOT_JOB, central_name, check_ref, prepare, snapshot_pom, + superseded, upload, verify) from buildcheck import workflow from buildcheck.tests.helpers import REPO -class PublicationTest(unittest.TestCase): +class PublicationFixture(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) @@ -40,9 +44,28 @@ def setUp(self): quiet.start() self.addCleanup(quiet.stop) - def freeze(self): - prepare(self.root, self.assets, self.package) + def freeze(self, **kwargs): + # ref="" keeps the runner's GITHUB_REF (a tag on a release run) out of the snapshot fixture. + prepare(self.root, self.assets, self.package, **{"ref": "", **kwargs}) + def add(self, directory, name, data=None): + for extra in ("", ".asc"): + (directory / (name + extra)).write_text((data or name) + extra) + + def release(self): + """Turn the fixture into version 1.2.3, staged as Maven stages a release.""" + pom = self.pom.replace("1-SNAPSHOT", "1.2.3") + (self.root / "pom.xml").write_text(pom) + stage = self.root / "target/central-staging/local/test/demo/1.2.3" + stage.mkdir(parents=True) + for path in list(self.assets.iterdir()): + path.unlink() + for suffix in [".pom", ".jar", "-sources.jar", "-javadoc.jar"]: + for directory in (self.assets, stage): + self.add(directory, "demo-1.2.3" + suffix, pom if suffix == ".pom" else None) + + +class PublicationTest(PublicationFixture): def test_freezes_once_and_verifies_every_signature(self): self.freeze() self.assertEqual(self.run.call_count, 4) @@ -139,3 +162,260 @@ def test_publishers_depend_only_on_the_same_checked_handoff(self): self.assertIn('subject-path: publication-package/github/*', attest) self.assertIn('id-token: write', attest) self.assertIn('attestations: write', attest) + + def test_snapshot_channel_and_ref_must_match_the_version(self): + with self.assertRaisesRegex(ValueError, "release publication of snapshot version"): + self.freeze(channel="release") + with self.assertRaisesRegex(ValueError, "tag v1 names SNAPSHOT version"): + self.freeze(ref="refs/tags/v1") + self.assertFalse(self.package.exists()) + self.freeze(channel="snapshot", ref="refs/heads/main") + self.assertEqual(verify(self.package)["channel"], "snapshot") + + def test_release_publishes_only_from_the_tag_of_its_version(self): + self.release() + for ref in ("refs/heads/main", "refs/tags/v1.2.4", "refs/tags/1.2.3"): + with self.subTest(ref=ref): + with self.assertRaisesRegex(ValueError, "publishes only from tag v1.2.3"): + self.freeze(ref=ref) + with self.assertRaisesRegex(ValueError, "snapshot publication of release version"): + self.freeze(channel="snapshot", ref="refs/tags/v1.2.3") + self.freeze(channel="release", ref="refs/tags/v1.2.3") + self.assertEqual(verify(self.package)["channel"], "release") + + def test_check_ref_without_a_ref_checks_only_the_channel(self): + self.assertEqual(check_ref("1-SNAPSHOT", None), "snapshot") + self.assertEqual(check_ref("1.0", None, "release"), "release") + self.assertEqual(check_ref("1-SNAPSHOT", "refs/pull/7/merge", "snapshot"), "snapshot") + with self.assertRaises(ValueError): + check_ref("1.0", None, "snapshot") + + def test_gradle_timestamped_snapshot_in_a_second_root_matches_the_github_names(self): + (self.root / ".github/release-assets.json").write_text(json.dumps({ + "schema": 1, + "files": [{"source": "mobile/build/mobile-{version}.aar", "central": True}, + {"source": "mobile/build/pom-default.xml", "name": "mobile-{version}.pom", "central": True}, + {"source": "mobile/build/notes-{version}.txt"}], + "publication": {"central_roots": ["target/{staging}", "mobile/build/staging-repo"]}})) + build = self.root / "mobile/build" + build.mkdir(parents=True) + self.add(build, "mobile-1-SNAPSHOT.aar") + self.add(build, "pom-default.xml", "mobile pom") + self.add(build, "notes-1-SNAPSHOT.txt") + for name in ("mobile-1-SNAPSHOT.aar", "mobile-1-SNAPSHOT.pom", "notes-1-SNAPSHOT.txt"): + source = build / ("pom-default.xml" if name.endswith(".pom") else name) + for extra in ("", ".asc"): + (self.assets / (name + extra)).write_bytes(Path(str(source) + extra).read_bytes()) + # What Gradle's maven-publish writes for a SNAPSHOT, even into a file repository. + gradle = self.root / "mobile/build/staging-repo/local/test/mobile/1-SNAPSHOT" + gradle.mkdir(parents=True) + stamp = "mobile-1-20261025.085646-1" + for name, source in ((stamp + ".aar", "mobile-1-SNAPSHOT.aar"), (stamp + ".pom", "pom-default.xml")): + for extra in ("", ".asc"): + (gradle / (name + extra)).write_bytes((build / (source + extra)).read_bytes()) + for checksum in (".md5", ".sha1", ".sha256", ".sha512"): + (gradle / (name + checksum)).write_text("regenerated") + (gradle / "maven-metadata.xml").write_text("") + (gradle.parent / "maven-metadata.xml.sha1").write_text("regenerated") + (self.root / "target/central-deferred/.index").write_text("plugin bookkeeping") + self.freeze() + files = verify(self.package)["files"] + self.assertIn("central/local/test/mobile/1-SNAPSHOT/mobile-1-SNAPSHOT.aar.asc", files) + self.assertIn("central/local/test/mobile/1-SNAPSHOT/mobile-1-SNAPSHOT.pom", files) + self.assertIn("github/notes-1-SNAPSHOT.txt.asc", files) + self.assertFalse(any("notes" in name for name in files if name.startswith("central/"))) + self.assertFalse(any("2026" in name or "metadata" in name or name.endswith(".md5") for name in files)) + + def test_github_only_file_in_central_staging_and_unknown_staging_files_fail(self): + self.add(self.stage, "demo-1-SNAPSHOT-cyclonedx.json") + self.add(self.assets, "demo-1-SNAPSHOT-cyclonedx.json") + with self.assertRaisesRegex(ValueError, "unexpected=.*cyclonedx"): + self.freeze() + for name in ("demo-1-SNAPSHOT-cyclonedx.json", "demo-1-SNAPSHOT-cyclonedx.json.asc"): + (self.stage / name).unlink() + (self.assets / name).unlink() + for name in ("demo-1-SNAPSHOT.module", "demo-1-SNAPSHOT-bundle.zip"): + with self.subTest(name=name): + (self.stage / name).write_text("payload the transfer would not carry") + with self.assertRaisesRegex(ValueError, "unexpected file in Central staging: .*" + name): + self.freeze() + (self.stage / name).unlink() + self.assertFalse(self.package.exists()) + + def test_central_signatures_pair_by_name_not_by_count(self): + # One signature missing and one orphan keep the count; the gate names both. + (self.stage / "demo-1-SNAPSHOT.jar.asc").unlink() + (self.stage / "demo-1-SNAPSHOT-tests.jar.asc").write_text("orphan") + with self.assertRaisesRegex(ValueError, r"missing=\['demo-1-SNAPSHOT.jar.asc'\]; " + r"unexpected=\['demo-1-SNAPSHOT-tests.jar.asc'\]"): + self.freeze() + + def test_github_asset_names_github_would_rename_fail(self): + (self.root / ".github/release-assets.json").write_text(json.dumps( + {"schema": 1, "files": [{"source": "dist/demo {version}.zip"}]})) + (self.root / "dist").mkdir() + self.add(self.root / "dist", "demo 1-SNAPSHOT.zip") + self.add(self.assets, "demo 1-SNAPSHOT.zip") + with self.assertRaisesRegex(ValueError, "GitHub would rename release asset: demo 1-SNAPSHOT.zip"): + self.freeze() + + def test_prepare_reads_the_given_contract(self): + other = self.root / "other.json" + other.write_text(json.dumps({"schema": 1, "files": [{"source": "extra-{version}.txt"}]})) + self.add(self.root, "extra-1-SNAPSHOT.txt") + with self.assertRaisesRegex(ValueError, r"GitHub membership.*missing=\['extra-1-SNAPSHOT.txt'\]"): + self.freeze(config_path=other) + self.add(self.assets, "extra-1-SNAPSHOT.txt") + self.freeze(config_path=other) + self.assertIn("github/extra-1-SNAPSHOT.txt", verify(self.package)["files"]) + + def test_timestamp_names_are_normalized_only_for_their_own_artifact_and_snapshot(self): + self.assertEqual(central_name("app-1.0-20261025.085646-12-sources.jar", "app", "1.0-SNAPSHOT"), + "app-1.0-SNAPSHOT-sources.jar") + self.assertEqual(central_name("app-1.0-SNAPSHOT.jar", "app", "1.0-SNAPSHOT"), "app-1.0-SNAPSHOT.jar") + self.assertEqual(central_name("app-tool-1.0-20261025.085646-1.jar", "app", "1.0-SNAPSHOT"), + "app-tool-1.0-20261025.085646-1.jar") + self.assertEqual(central_name("app-1.0-20261025.085646-1.jar", "app", "1.0"), + "app-1.0-20261025.085646-1.jar") + + +class UploadTest(PublicationFixture): + """The two transfers with the network and Maven replaced; the bytes are the frozen ones.""" + + def setUp(self): + super().setUp() + self.signatures.stop() # upload() runs curl/mvn, each test patches subprocess.run itself + env = patch.dict(os.environ, {"MAVEN_USERNAME": "user", "MAVEN_PASSWORD": "secret", + "GITHUB_REPOSITORY": "owner/repo", "GITHUB_SHA": "", + "GITHUB_RUN_ID": "10"}) + env.start() + self.addCleanup(env.stop) + + def frozen(self, release=False): + with patch("buildcheck.publication.subprocess.run"): + if release: + self.release() + self.freeze(ref="refs/tags/v1.2.3") + else: + self.freeze() + return self.package + + def test_release_bundle_holds_the_frozen_central_files_with_checksums_and_no_credentials_in_argv(self): + package = self.frozen(release=True) + seen = {} + + def curl(args, **kwargs): + seen["config"] = Path(args[args.index("--config") + 1]).read_text() + bundle = next(a for a in args if a.startswith("bundle=@"))[len("bundle=@"):] + with zipfile.ZipFile(bundle) as archive: + seen["names"] = set(archive.namelist()) + seen["jar"] = archive.read("local/test/demo/1.2.3/demo-1.2.3.jar") + seen["args"] = args + return subprocess.CompletedProcess(args, 0, '"deployment-id_1"', "") + + with patch("buildcheck.publication.subprocess.run", side_effect=curl): + upload(package) + self.assertIn("local/test/demo/1.2.3/demo-1.2.3.jar.asc", seen["names"]) + self.assertIn("local/test/demo/1.2.3/demo-1.2.3.jar.sha512", seen["names"]) + self.assertNotIn("local/test/demo/1.2.3/demo-1.2.3.jar.asc.md5", seen["names"]) + self.assertFalse(any(name.startswith("github/") or name == "manifest.json" for name in seen["names"])) + self.assertEqual(seen["jar"], (package / "github/demo-1.2.3.jar").read_bytes()) + self.assertIn("Authorization: Bearer dXNlcjpzZWNyZXQ=", seen["config"]) + self.assertFalse(any("secret" in arg or "dXNlcjpzZWNyZXQ" in arg for arg in seen["args"])) + self.assertTrue(seen["args"][-1].endswith("?publishingType=AUTOMATIC")) + + def test_release_upload_without_a_valid_receipt_fails(self): + package = self.frozen(release=True) + for code, out in ((22, "rejected"), (0, "not a receipt!")): + with self.subTest(code=code): + result = subprocess.CompletedProcess([], code, out, "") + with patch("buildcheck.publication.subprocess.run", return_value=result): + with self.assertRaises(RuntimeError): + upload(package) + + def test_snapshot_transfer_runs_maven_on_the_generated_pom(self): + package = self.frozen() + with patch("buildcheck.publication.subprocess.run") as run: + upload(package) + args = run.call_args.args[0] + self.assertEqual((args[0], args[-1]), ("mvn", "validate")) + self.assertTrue(args[args.index("-f") + 1].endswith("pom.xml")) + + def test_snapshot_transfer_skips_only_when_asked_and_superseded(self): + os.environ["GITHUB_SHA"] = "a" * 40 + package = self.frozen() + gh = FakeRuns(newer_published=True) + with patch("buildcheck.publication.subprocess.run") as run: + upload(package, skip_superseded=True, github=gh) + self.assertFalse(run.called) + upload(package, github=gh) + self.assertTrue(run.called) + + +class FakeRuns: + def __init__(self, newer_published=False, status="ahead", job=CENTRAL_SNAPSHOT_JOB, conclusion="success"): + self.newer_published, self.status, self.job, self.conclusion = newer_published, status, job, conclusion + self.calls = [] + + def api(self, method, endpoint, payload=None, optional=False): + self.calls.append(endpoint) + if endpoint.endswith("/actions/runs/10"): + return {"workflow_id": 99} + if "/actions/workflows/99/runs?" in endpoint: + return {"workflow_runs": [{"id": 10, "head_sha": "a" * 40}, {"id": 11, "head_sha": "a" * 40}, + {"id": 12, "head_sha": "b" * 40}]} + if "/actions/runs/12/jobs?" in endpoint: + if endpoint.endswith("page=1"): + return {"jobs": [{"name": f"job {i}", "conclusion": "success"} for i in range(100)]} + return {"jobs": [{"name": self.job, "conclusion": self.conclusion}] if self.newer_published else []} + if "/compare/" in endpoint: + return {"status": self.status} + raise AssertionError(endpoint) + + +class SupersededTest(unittest.TestCase): + def setUp(self): + quiet = patch("builtins.print") + quiet.start() + self.addCleanup(quiet.stop) + + def test_a_newer_commit_transferred_by_another_run_supersedes_this_one(self): + gh = FakeRuns(newer_published=True) + self.assertTrue(superseded("owner/repo", "a" * 40, "10", gh)) + # Its own run and other runs of the same commit are never asked about their jobs. + self.assertFalse(any("/runs/10/jobs" in c or "/runs/11/jobs" in c for c in gh.calls)) + self.assertTrue(any(c.endswith("page=2") for c in gh.calls)) + self.assertTrue(any(c.endswith("/compare/" + "a" * 40 + "..." + "b" * 40) for c in gh.calls)) + + def test_older_divergent_unfinished_or_failed_runs_do_not_supersede(self): + for kwargs in ({"newer_published": True, "status": "behind"}, + {"newer_published": True, "status": "diverged"}, + {"newer_published": False}, + {"newer_published": True, "conclusion": "failure"}, + {"newer_published": True, "job": "Some other job"}): + with self.subTest(**kwargs): + self.assertFalse(superseded("owner/repo", "a" * 40, "10", FakeRuns(**kwargs))) + + +class WorkflowContractTest(unittest.TestCase): + """The shared jobs call the gate the way the code above expects.""" + + def setUp(self): + self.jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) + + def test_ref_checks_run_before_anything_is_prepared(self): + for job, channel in (("check-snapshot", "snapshot"), ("check-tag", "release")): + self.assertIn("python3 .github/publication.py check-ref --channel " + channel, self.jobs[job].text) + self.assertIn("persist-credentials: false", self.jobs[job].text) + action = (Path(REPO) / ".github/actions/finalize-publication/action.yml").read_text() + self.assertIn('publication.py prepare "signed-${CHANNEL}-assets" publication-package --channel "$CHANNEL"', + action) + + def test_snapshot_transfers_are_serialized_and_skip_superseded_commits(self): + job = self.jobs["central-snapshot"].text + self.assertIn("name: " + CENTRAL_SNAPSHOT_JOB, job) + self.assertIn("group: central-snapshot-${{ github.repository }}", job) + self.assertIn("cancel-in-progress: false", job) + self.assertIn("actions: read", job) + self.assertIn("python3 .github/publication.py upload publication-package --skip-superseded", job) + self.assertNotIn("--skip-superseded", self.jobs["central-release"].text) diff --git a/.github/buildcheck/tests/test_releaseassets.py b/.github/buildcheck/tests/test_releaseassets.py index 5ec9bae..2e5d04a 100644 --- a/.github/buildcheck/tests/test_releaseassets.py +++ b/.github/buildcheck/tests/test_releaseassets.py @@ -269,7 +269,7 @@ def test_cli_reports_completeness_even_on_failure_and_accepts_parameters(self): self.assertTrue((destination / f"app-{VERSION}.jar").is_file()) -class SignatureReportTest(unittest.TestCase): +class SignatureGapsTest(unittest.TestCase): def test_all_file_types_spaces_empty_orphan_and_missing_signatures(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -284,20 +284,6 @@ def test_all_file_types_spaces_empty_orphan_and_missing_signatures(self): (root / "file.jar").write_text("") self.assertEqual(signature_gaps(root)[0], 4) - def test_cli_always_returns_zero_and_writes_missing_for_post_upload_assertion(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - assets = root / "assets" - assets.mkdir() - (assets / "unsigned.json").write_text("bom") - output = root / "github-output" - result = subprocess.run([sys.executable, str(Path(REPO) / ".github/report-asset-signatures.py"), - str(assets)], env={**os.environ, "GITHUB_OUTPUT": str(output)}, - capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("::error::unsigned", result.stdout) - self.assertEqual(output.read_text(), "missing=1\n") - if __name__ == "__main__": unittest.main() diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py index 4b74f8c..7593ea2 100644 --- a/.github/buildcheck/tests/test_snapshot.py +++ b/.github/buildcheck/tests/test_snapshot.py @@ -83,17 +83,31 @@ def test_upload_precedes_tag_notes_and_prune_and_release_itself_survives(self): deletes = [c[1] for c in calls if c[0] == "DELETE"] self.assertEqual(deletes, ["repos/owner/repo/releases/assets/1"]) - def test_incomplete_collection_uploads_available_files_without_promoting_or_pruning(self): - with self.assertRaisesRegex(ValueError, "collection is incomplete"): + def test_incomplete_collection_changes_nothing(self): + with self.assertRaisesRegex(ValueError, "incomplete collection"): self.publish(complete=False) - self.assertTrue(any(c[0] == "upload" for c in self.gh.calls)) - self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + self.assertEqual(self.gh.calls, []) - def test_missing_signature_also_prevents_promotion(self): + def test_missing_signature_also_changes_nothing(self): (self.assets / "new.json.asc").unlink() - with self.assertRaisesRegex(ValueError, "collection is incomplete"): + with self.assertRaisesRegex(ValueError, "incomplete collection.*unsigned: new.json"): + self.publish() + self.assertEqual(self.gh.calls, []) + + def test_versioned_release_tag_is_never_a_rolling_snapshot(self): + for tag in ("v1.2.3", "v2"): + with self.subTest(tag=tag): + with self.assertRaisesRegex(ValueError, "versioned release"): + self.publish(tag=tag) + self.assertEqual(self.gh.calls, []) + + def test_names_github_would_rename_are_refused_before_any_change(self): + # GitHub would upload "a b.jar" as "a.b.jar", and the prune would then delete it. + for name in ("a b.jar", "a b.jar.asc"): + (self.assets / name).write_text("artifact") + with self.assertRaisesRegex(ValueError, "rename release assets: a b.jar"): self.publish() - self.assertFalse(any(c[0] == "PATCH" for c in self.gh.calls)) + self.assertEqual(self.gh.calls, []) def test_late_finishing_older_commit_cannot_overwrite_newer_snapshot(self): self.gh.comparison = "behind" @@ -106,9 +120,9 @@ def test_empty_signature_or_orphan_also_prevents_promotion(self): (self.assets / "new.json.asc").write_text("signature") (self.assets / name).write_text(content) self.gh.calls.clear() - with self.assertRaisesRegex(ValueError, "collection is incomplete"): + with self.assertRaisesRegex(ValueError, "incomplete collection"): self.publish() - self.assertFalse(any(c[0] in ("PATCH", "DELETE") for c in self.gh.calls)) + self.assertEqual(self.gh.calls, []) def test_rolling_tag_and_title_are_parameters(self): self.gh.tag = "nightly" @@ -204,8 +218,7 @@ def test_optional_api_failure_accepts_only_404_not_authentication_failure(self): def test_release_upload_jobs_do_not_persist_checkout_credentials(self): jobs = workflow.parse((Path(REPO) / ".github/workflows/publish.yml").read_text()) - versioned = "github-release" if "github-release" in jobs else "github-release-signed" - for name in ("github-snapshot", versioned): + for name in ("github-snapshot", "github-release"): checkouts = [step for step in jobs[name].steps() if any("uses: actions/checkout@" in line for line in step)] self.assertTrue(checkouts, name) diff --git a/.github/report-asset-signatures.py b/.github/report-asset-signatures.py deleted file mode 100644 index a8fbcef..0000000 --- a/.github/report-asset-signatures.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# -# SPDX-License-Identifier: MIT OR Apache-2.0 -"""Report signature gaps without blocking uploads; assert the missing output afterwards.""" - -import argparse -import os - -from buildcheck.releaseassets import signature_gaps - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("directory") - args = parser.parse_args() - try: - missing, errors = signature_gaps(args.directory) - except OSError as error: - missing, errors = -1, [str(error)] - for error in errors: - print(f"::error::{error}") - if not missing: - print("All collected artifacts have nonempty detached signatures.") - if os.environ.get("GITHUB_OUTPUT"): - with open(os.environ["GITHUB_OUTPUT"], "a") as output: - output.write(f"missing={missing}\n") - # The workflow uploads first and fails on the output afterwards. - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 165f326..caad31a 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -11,7 +11,7 @@ # job of a workflow (kept identical inside otherwise different workflows); an entry ending in `?repo` # for a file or job identical up to the repository's name (hashed with that name replaced by # `{repo}`); `sha256sum -c` reads the plain file entries only. -9ad21ea42bced35498c6fcaa533d9ba448af042497f923e60646a77cfbe0fc5f .github/buildcheck/__init__.py +59b253ed6d8813ed64b304089d04c5ced106669a38b37bbc643a3751f1fc4a1c .github/buildcheck/__init__.py 3d1a9f2e93709941261a9655bc19bf50d5becefd8f6a6c2f5564096a757996bb .github/buildcheck/workflow.py 5a70eee1bb8af180edfd2a4bae61be61fac15b409f5df6a8bb9869425d7fc381 .github/buildcheck/releasegate.py ff2aaec95e53916ea340a0a291bf0f57e0011522aedb486622b1fb1cac136865 .github/buildcheck/sharedfiles.py @@ -65,29 +65,28 @@ c9e757f2d0a2f456a08791697a94161c776d9b008a8aa75890da4133c81a99e9 .github/depend f26d05f25a154b84d8f281bb92857811749e2225cdbc167362de930c48b532d1 .github/workflows/publish.yml#startgate e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workflows/publish.yml#shared-files 3ef45f189f4419e0d4087f572b3c1b6085c0fe39ac7979b07deb2e882f7ffe8c .github/workflows/publish.yml#verify-signing-key -09230b04c5f14bdda50a0c862152ff197a656cbc3fd0d110899c5f8150f8aa9a .github/workflows/publish.yml#check-snapshot -51e987d59efd9887b43e345edc0e0f3db21686879e3ca6ea9f0fdbc493d37121 .github/workflows/publish.yml#check-tag +216fe05041f385c2d60f168302cd7a11d29069eae6ef51c855c82771f750ea92 .github/workflows/publish.yml#check-snapshot +6fc44f69d3549f4f248b8612d596a877c491a0d497916e5d9ffeb67c51769804 .github/workflows/publish.yml#check-tag a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle -9e7609a8280585c13baa5418afc52e09d12753fea2c1f7d2d7b392e8a74e67b6 .github/workflows/publish.yml#github-snapshot -52f1f7c99a604f40fb3f3685909c060b2c270e3f3dd5d28f3119be4b3f5d4052 .github/workflows/publish.yml#github-release -ed1ba202b9d81a00893931a090112bfb28770a514030a5c08d5441c83369f8b7 .github/buildcheck/releaseassets.py -bdf9390419deba3ffd9ce883895bde8331abcbae472972d2962812601da735d0 .github/buildcheck/snapshot.py -08828c6a69f498adae4bf4d9bc8b4d70ccfe21b9185e29512e3807dd6613e7a5 .github/buildcheck/tests/test_releaseassets.py -8b6ae70f43560a0095b6f16012b80ed694b07633c1cbde94dbe8c21c529461ea .github/buildcheck/tests/test_snapshot.py +408aec023edac2aa8769d70166ffffa54233ec825862abfc70cec05b39834fbf .github/workflows/publish.yml#github-snapshot +f90bda93c5ce115e571d61028808121dbbe65e75771eb853027402486fabd4e1 .github/workflows/publish.yml#github-release +9c1c523f8131ee9129305831125fd90b7f34e8f9aa386d05509f75c720f7ef5d .github/buildcheck/releaseassets.py +0de7abac98ead2f22c46ea5f48051bf0965811fab674df1c8d55207cba12a1fc .github/buildcheck/snapshot.py +894465632ecdb87c587cd7e18d6fc14f6ac4fa77490154b77188cb4dfc425700 .github/buildcheck/tests/test_releaseassets.py +d6eee9187eeb42b8112c6b0a665723e4ce409b4bddb7f52151177a3d593574f5 .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py -6febe6ae3c2e1a4e0bbb0e0d979726c580fc1d33db0f9da6d186cdd5e1f72e3e .github/report-asset-signatures.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license 0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py -4d16ef3f5f4fa5fa39d191b030e1cbff3c943c67b33ae9b392537e4b08f35004 .github/buildcheck/publication.py +7b2a353968eeafb96b9fe0d61b380bebbba9f88dcbf05489c6d3ab3fa80a6ff9 .github/buildcheck/publication.py 9180fac217c75b3e11f67e244b9171f447d2c21665b1802574b57b97019f7173 .github/publication.py -6e142c05ddc3c1d4cc31789be5e2040393318c6ae79d99d01eec72e72dc94232 .github/buildcheck/tests/test_publication.py -7b96c3db888769146900b123c81d4edf5e8ef4061fa9f547abcbd02744a3a4d5 docs/publication-preflight.md -b3cb68727cc2b0a4e6831b3b9301b6e54420dc6c52aa068dcd8b2b38a6c2308b .github/workflows/publish.yml#central-snapshot -5bf3977c27f1a5ae7f9894cc6e1b26b014ea3ee6bb854fe89daa9cbb26785a23 .github/workflows/publish.yml#central-release -ae21a7de07e232c32ab5a370fc1c153ba1fb8534ae722ed61770a15903553229 .github/workflows/dependency-submission.yml -68b7bc3d8deb1696e1c69a2c7200c662737c4be74dce09fb1aae4e12fb55c284 .github/actions/finalize-publication/action.yml -3779cc3b48f923b3b63ea265bbfccc5e9c3709b9db230d8c13534143784c1264 .github/workflows/publish.yml#attest-snapshot -16c54b3c6c516e300974e4fe2fd5264af373e45f9bac3ecba553ac8a33b9e5cd .github/workflows/publish.yml#attest-release +862934ffa7265b3ef5612e287aca6f8f073dd44873c873f7b47affe8b89391b5 .github/buildcheck/tests/test_publication.py +6a18808dc0c672811309a6e34864f5e2fd02699b4f820ea7b5e841b6e1fb627a docs/publication-preflight.md +aa9989fd22438c3f7cb427ae6bcd6f3b87a9ef014e7d24c2ee9bdc179a137928 .github/workflows/publish.yml#central-snapshot +2180d438fb840a1ca7015cdb95f28c25641c0b151679d95767c4b4393794e4f1 .github/workflows/publish.yml#central-release +8f96d48502f487a311d5cc01dadedc92fe4a88f565e01f05ddbd1ba839bfe70e .github/workflows/dependency-submission.yml +38f493414ec975a0c89d8836aa3825dd2d77677220bce5fa1c041a19ab85e4cc .github/actions/finalize-publication/action.yml +ad9631c57b868851a6acbc08f47bc790d3b69a587c51cb7429d657e63c5fd23b .github/workflows/publish.yml#attest-snapshot +481c8112f38f5aede30905702033e2dccf9c807a6eb167cefd167f15738e9a84 .github/workflows/publish.yml#attest-release diff --git a/.github/workflows/dependency-submission.yml b/.github/workflows/dependency-submission.yml index 3b73dd5..eac4807 100644 --- a/.github/workflows/dependency-submission.yml +++ b/.github/workflows/dependency-submission.yml @@ -3,7 +3,10 @@ name: Maven Dependency Submission on: + # The graph GitHub shows is the default branch's. Other branches (Dependabot's among them, whose + # pushes get a read-only token) would only fail or add snapshots nobody reads. push: + branches: [ main ] workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 743f093..74f87e1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -324,19 +324,18 @@ jobs: needs: [test] runs-on: ubuntu-latest permissions: - contents: write + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: { java-version-file: .java-version, distribution: temurin, cache: maven } - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: { name: jacoco-report, path: target/site/jacoco/ } continue-on-error: true - # Submits the dependency graph to GitHub. Informational: it says nothing about whether the - # artifacts are correct, but it sits in the `report` job, which the release path needs -- so - # without this flag a third-party action having a bad day can block a publish. - - uses: advanced-security/maven-dependency-submission-action@a64327a7329c9939cf675e458452febe1894a70c # v6 - continue-on-error: true + # The dependency graph is submitted by dependency-submission.yml (on pushes to main, with the + # JDK of .java-version), not here: this job gates the release path and needs no write access. - name: Coveralls uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2 with: @@ -385,8 +384,15 @@ jobs: (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && !startsWith(github.ref, 'refs/tags/v')) steps: - - name: Confirm snapshot ref - run: echo "Confirmed on snapshot ref ${{ github.ref }}" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + # Every project version ends in -SNAPSHOT before anything is published to the snapshot channel. + # Only then: a push publishes nothing, and main carries the release version for the commit + # a release is tagged on. + - name: Require a SNAPSHOT version + if: inputs.publish_to_central + run: python3 .github/publication.py check-ref --channel snapshot check-tag: name: "Check: v* tag" @@ -394,8 +400,12 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') steps: - - name: Confirm tag ref - run: echo "Confirmed on tag ${{ github.ref }}" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + # The tag names exactly the project version (v1.2.3 for 1.2.3), which is no SNAPSHOT. + - name: Require the release version the tag names + run: python3 .github/publication.py check-ref --channel release publish-snapshot: name: Prepare and verify signed snapshot artifacts @@ -413,8 +423,8 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: java-version-file: .java-version distribution: temurin @@ -457,10 +467,10 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets path: publication-package/ @@ -489,8 +499,8 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: java-version-file: .java-version distribution: temurin @@ -521,10 +531,10 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets path: publication-package/ @@ -532,7 +542,7 @@ jobs: - name: Verify transport integrity and export checked GitHub files run: python3 .github/publication.py verify publication-package --export-github release-assets - name: Upload release assets - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3 with: files: release-assets/* @@ -544,27 +554,33 @@ jobs: if: ${{ !cancelled() && needs.publish-snapshot.outputs.assets_complete == 'true' }} runs-on: ubuntu-latest environment: maven-central + # One transfer at a time; an older commit finishing late is skipped (publication.py superseded). + concurrency: + group: central-snapshot-${{ github.repository }} + cancel-in-progress: false permissions: contents: read + actions: read steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: java-version-file: .java-version distribution: temurin server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets path: publication-package/ digest-mismatch: error - name: Transfer frozen artifacts (no rebuild, signing or validation polling) - run: python3 .github/publication.py upload publication-package + run: python3 .github/publication.py upload publication-package --skip-superseded env: + GH_TOKEN: ${{ github.token }} MAVEN_USERNAME: ${{ secrets.CENTRAL_USERNAME }} MAVEN_PASSWORD: ${{ secrets.CENTRAL_TOKEN }} @@ -578,17 +594,17 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: java-version-file: .java-version distribution: temurin server-id: central server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets path: publication-package/ @@ -610,10 +626,10 @@ jobs: id-token: write attestations: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-snapshot-assets path: publication-package/ @@ -621,7 +637,7 @@ jobs: - name: Verify the secured package before attesting run: python3 .github/publication.py verify publication-package - name: Attest the frozen artifacts and detached signatures - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 with: subject-path: publication-package/github/* @@ -637,10 +653,10 @@ jobs: id-token: write attestations: write steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: signed-release-assets path: publication-package/ @@ -648,6 +664,6 @@ jobs: - name: Verify the secured package before attesting run: python3 .github/publication.py verify publication-package - name: Attest the frozen artifacts and detached signatures - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 with: subject-path: publication-package/github/* diff --git a/CLAUDE.md b/CLAUDE.md index 9b138ea..413676a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -263,23 +263,34 @@ enforcer default gotcha are in The thin jar, its sources and javadoc jars and the pom are attached to the GitHub `snapshot` pre-release and to every versioned release with the `.asc` maven-gpg wrote at deploy; no fat jar -exists here. Release collection, signature reporting and rolling-snapshot updates use the byte-identical -`collect-release-assets.py`, `report-asset-signatures.py`, `update-snapshot.py` and their buildcheck -library/tests, all listed in `.github/shared-files.sha256`. Repository-specific extras live in -`.github/release-assets.json`; Maven modules are discovered from POMs. Missing artifacts/signatures -fail collection while preserving available files for upload. The whole `github-snapshot` job is -shared across all four Java repos and serializes upload, source-tag/notes updates and pruning; -only complete signed uploads advance/prune the snapshot. Versioned releases are never pruned. -The `github-release` job remains shared across the three Maven-only repos. Interface, staging -options and synchronization rules: [`../workspace/policies/release-asset-scripts.md`](../workspace/policies/release-asset-scripts.md). +exists here. Every publication goes through one gate, shared byte for byte by the four Java repositories +(listed in `.github/shared-files.sha256`; the full text is +[`docs/publication-preflight.md`](docs/publication-preflight.md)). `check-snapshot` / `check-tag` +run `publication.py check-ref`: a release publishes only from the tag `v` of a +non-SNAPSHOT version, a snapshot only a `-SNAPSHOT` version and never from a tag. The publish jobs +stage and sign the Central files locally (`-DskipUpload=true deploy`) and end in the composite +action `finalize-publication`: `collect-release-assets.py` collects what `.github/release-assets.json` +names (Maven modules are discovered from the POMs), and `publication.py prepare` checks it -- exact +membership, every signature with `gpg --verify`, each Central file byte-equal to its GitHub twin -- +and freezes it into one package. Only after that package is uploaded does the job output +`assets_complete=true`; a missing file or signature fails there, and nothing is published. Three +pairs of consumers each need only that output, never each other: `github-snapshot` / +`github-release` attach the frozen GitHub files (`update-snapshot.py` serializes the rolling +`snapshot` pre-release, never lets an older commit or run replace a newer one and prunes what the +upload did not contain; a versioned tag is refused), `central-snapshot` / `central-release` +transfer the frozen Central files (`publication.py upload`, no rebuild, no re-signing), and +`attest-snapshot` / `attest-release` attest them. Interface, staging options and synchronization +rules: [`../workspace/policies/release-asset-scripts.md`](../workspace/policies/release-asset-scripts.md). ## Shared files and the release gate (`shared-files` job) Files kept byte-identical with java-llama.cpp, BitcoinAddressFinder, srcmorph and streambuffer are listed with their SHA-256 in **`.github/shared-files.sha256`** — the reference for what must stay equal. An entry `.github/workflows/publish.yml#` stands for one job of the workflow: the jobs kept -identical across the repositories (`startgate`, `shared-files`, `verify-signing-key`, `check-snapshot`, -`check-tag`, `verify-signing-key-gradle`, `github-snapshot`, `github-release`) are checked like files. An entry ending in `?repo` covers a file or job identical up to the +identical across the four repositories (`startgate`, `shared-files`, `verify-signing-key`, +`verify-signing-key-gradle`, `check-snapshot`, `check-tag`, and the publication consumers +`github-snapshot`, `github-release`, `central-snapshot`, `central-release`, `attest-snapshot`, +`attest-release`) are checked like files. An entry ending in `?repo` covers a file or job identical up to the repository's name (hashed with the name replaced by `{repo}`), e.g. `SUPPORT.md?repo`. The `shared-files` job of `publish.yml` (gating both publish jobs) fails when a listed file changed here alone and warns when another repository's default branch lists it with a different hash. To change a shared file, change every copy, then run diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index 711e990..692eb3e 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -6,7 +6,10 @@ SPDX-License-Identifier: MIT OR Apache-2.0 # Third-party license and attribution notes -The project is Apache-2.0. The library has no declared runtime dependencies. +The project is Apache-2.0. Its only compile dependencies are optional annotation libraries +(Error Prone annotations and JSpecify, Apache-2.0; Checker Framework qualifiers, MIT): consumers +do not receive them transitively, and the JVM never loads them, so the library has no runtime +dependency. JMH 1.37, JCStress 0.16, Lincheck 3.7, JUnit and jqwik are test-only tools. Lincheck includes LGPL-3.0-or-later source files as well as its declared MPL license. A FOSSA scan of the old master revision does not cover this current build. @@ -22,10 +25,7 @@ For unchanged Maven components, source is available from as `--sources.jar`; POMs and upstream source repositories provide provenance. Preserve copyright, license and NOTICE files when redistributing dependencies. -Runnable BAF/srcmorph CLI packaging uses Python 3.8+ to collect original dependency -legal files before the project JAR and fat JAR are assembled. Override `-Dlicense.python=python` -if that is the Python command on the build host. Legal files are under -`META-INF/third-party/dependencies/-/`. -`inventory.json` maps exact JAR hashes/classifiers to preserved files and also lists JARs -without embedded legal material, which require review or supplemental notices. +streambuffer ships no fat JAR, so `.github/collect-third-party-licenses.py`, which collects +dependency legal files into the runnable fat JARs of BitcoinAddressFinder and srcmorph, is kept +here byte-identical with them but not run. This source-location statement does not clear unknown proprietary or historical Git findings. diff --git a/docs/publication-preflight.md b/docs/publication-preflight.md index 71085dc..61d1256 100644 --- a/docs/publication-preflight.md +++ b/docs/publication-preflight.md @@ -11,6 +11,8 @@ Build and sign every intended output locally. Complete the native, classifier an checks applicable to those outputs. Before any publication-package upload, collect the exact repository contract and run one complete local gate. Missing files, invalid signatures, version mismatches or differing Central/GitHub bytes must fail before either publisher runs. +A release publishes only from the tag `v` of a non-SNAPSHOT version; a snapshot only a +`-SNAPSHOT` version, and never from a tag. After the gate, neither publisher compiles, packages nor signs artifacts again. Both use the same frozen package from GitHub Actions. The Central, GitHub Release and provenance attestation jobs depend only on @@ -42,12 +44,20 @@ Finish both publication channels with the shared, checksummed composite action: maven-assets: central-assets # omit when no outputs need preserving ``` -The action collects the exact contract, calls `publication.py prepare`, and uploads the frozen -`publication-package/` as `signed-release-assets` or `signed-snapshot-assets`. +The action collects the exact contract, calls `publication.py prepare --channel `, and +uploads the frozen `publication-package/` as `signed-release-assets` or `signed-snapshot-assets`. The preparation job exports `assets_complete: ${{ steps.finalize.outputs.complete }}`. +The ref is checked twice. `check-snapshot` and `check-tag` run `publication.py check-ref --channel +` before anything is built (`check-snapshot` only on a run that publishes: `main` carries +the release version for the commit a release is tagged on). `prepare` repeats the check against +`GITHUB_REF` and refuses a channel the version does not have. + The gate checks exact file membership, nonempty artifacts/signatures, cryptographic GPG verification, and byte equality between Central staging and the corresponding GitHub files. +Central signatures pair by name: every artifact has its own `.asc`, and no `.asc` lacks its +artifact. GitHub renames release assets with characters outside `[A-Za-z0-9._-]` (or a leading +dot), so such a name fails the gate rather than reaching a release under another name. It freezes `central/` in Maven layout, `github/` as flat release files, and a `manifest.json` with commit, version, channel, file sizes and SHA-256. The composite's `complete=true` output is emitted only after both the local gate and the Actions handoff upload succeed. A failed @@ -66,6 +76,16 @@ is `["target/{staging}"]`; `{staging}` becomes `central-staging` or `central-def to the project version. This supports separate Maven or Gradle staging without repository names in the shared implementation. Unexpected or missing Central members fail the gate. +In a staging directory the gate reads `.pom`, `.jar`, `.aar` and `.json` files and their `.asc`. +Repository metadata (`maven-metadata*.xml`), checksums (`.md5`, `.sha1`, `.sha256`, `.sha512`) and +the Central plugin's `.index` are regenerated by the transfer and ignored; any other file fails the +gate instead of silently staying behind. Gradle writes a SNAPSHOT with a timestamp even into a file +repository (`app-1.0-20261025.085646-1.aar` under `1.0-SNAPSHOT/`); the gate maps such a name back +to `app-1.0-SNAPSHOT.aar`, the name the GitHub file and the Maven transfer use. + +`python3 .github/publication.py prepare --config ` reads another contract, +as `collect-release-assets.py --config` does. + ## Independent consumers ```bash @@ -93,7 +113,10 @@ records the deployment ID. There is no validation/publication polling. The Maven use `autoPublish=true` and `waitUntil=uploaded` for manual deployments. For snapshots, a transfer-only Maven POM invokes pinned `maven-deploy-plugin:3.1.4:deploy-file` -executions during `validate`. Maven maintains snapshot timestamps and metadata; signatures are +executions during `validate`. The `central-snapshot` jobs of all runs share one concurrency group, +and `upload --skip-superseded` skips the transfer when another dispatched run already transferred +the snapshot of a newer commit (one that descends from this run's); Central serves the snapshot +transferred last as the latest. The job reads the runs with `actions: read` and nothing else. Maven maintains snapshot timestamps and metadata; signatures are transferred in the same component request with their proper extensions. There are no source build phases. For a POM-only component its signature is the primary transfer artifact and the signed POM is supplied through `pomFile`, avoiding an empty-classifier parameter edge case. From c00d4ee5ad65bbe411503df1a12d53142368a00f Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sun, 25 Oct 2026 12:21:43 +0100 Subject: [PATCH 14/19] Correct what the delta review found Shared (byte-identical in the four Java repos): a --config outside the repository is refused (SonarCloud S8707), the upload test no longer carries a literal Bearer token (S8217), the 30-run window of superseded() is documented, and publication-preflight.md says the ref check runs after the build and test jobs, before anything is staged. streambuffer: the Python build checks are out of Sonar's coverage measurement (tested in the shared-files job, no report reaches Sonar), and CHANGELOG.md records the publication gate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VszvDmRRPP1UcwozsUMY1L --- .github/buildcheck/publication.py | 5 +++-- .github/buildcheck/releaseassets.py | 11 +++++++++-- .github/buildcheck/tests/test_publication.py | 15 +++++++++++++-- .github/shared-files.sha256 | 8 ++++---- CHANGELOG.md | 10 ++++++++++ docs/publication-preflight.md | 16 +++++++++------- pom.xml | 4 ++++ 7 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.github/buildcheck/publication.py b/.github/buildcheck/publication.py index 3880f8e..e579518 100644 --- a/.github/buildcheck/publication.py +++ b/.github/buildcheck/publication.py @@ -274,11 +274,12 @@ def child(parent, name, text): def superseded(repository, sha, run_id, github=None): - """True when another publish run already transferred the snapshot of a newer commit. + """True when one of the 30 most recent dispatched runs transferred the snapshot of a newer commit. The central-snapshot jobs of all runs share one concurrency group, so a newer run's transfer either finished before this check or starts after this transfer: Maven Central serves the - snapshot transferred last as the latest one. + snapshot transferred last as the latest one. Snapshots are published by dispatch only, so the + window of 30 runs reaches back far beyond any transfer still waiting in that group. """ gh = github or GitHub() repo = f"repos/{repository}" diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py index 63f2501..19c972d 100644 --- a/.github/buildcheck/releaseassets.py +++ b/.github/buildcheck/releaseassets.py @@ -30,8 +30,15 @@ def _inside(root, relative): def configuration(root, path=None): - """Read the explicit contract; unknown keys fail instead of silently dropping assets.""" - path = Path(path) if path is not None else Path(root) / ".github/release-assets.json" + """Read the explicit contract; unknown keys fail instead of silently dropping assets. + + The contract is part of the repository: a relative --config is relative to the repository root, + and a path outside the repository is refused. + """ + root = Path(root).resolve() + path = (root / (path if path is not None else ".github/release-assets.json")).resolve() + if not path.is_relative_to(root): + raise ValueError(f"release asset configuration outside the repository: {path}") config = json.loads(path.read_text()) _keys(config, ("schema", "maven", "files", "publication"), "release asset configuration") if type(config.get("schema")) is not int or config["schema"] != 1: diff --git a/.github/buildcheck/tests/test_publication.py b/.github/buildcheck/tests/test_publication.py index 23438ba..1916269 100644 --- a/.github/buildcheck/tests/test_publication.py +++ b/.github/buildcheck/tests/test_publication.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2026 Bernard Ladenthin # SPDX-License-Identifier: MIT OR Apache-2.0 """Publication gate and transport invariants; no signing key or remote writes.""" +import base64 import json import os from pathlib import Path @@ -259,6 +260,15 @@ def test_github_asset_names_github_would_rename_fail(self): with self.assertRaisesRegex(ValueError, "GitHub would rename release asset: demo 1-SNAPSHOT.zip"): self.freeze() + def test_a_contract_outside_the_repository_is_refused(self): + outside = Path(self.tmp.name).parent / (Path(self.tmp.name).name + "-outside.json") + outside.write_text('{"schema":1}') + self.addCleanup(outside.unlink) + with self.assertRaisesRegex(ValueError, "outside the repository"): + self.freeze(config_path=outside) + with self.assertRaisesRegex(ValueError, "outside the repository"): + self.freeze(config_path="../" + outside.name) + def test_prepare_reads_the_given_contract(self): other = self.root / "other.json" other.write_text(json.dumps({"schema": 1, "files": [{"source": "extra-{version}.txt"}]})) @@ -320,8 +330,9 @@ def curl(args, **kwargs): self.assertNotIn("local/test/demo/1.2.3/demo-1.2.3.jar.asc.md5", seen["names"]) self.assertFalse(any(name.startswith("github/") or name == "manifest.json" for name in seen["names"])) self.assertEqual(seen["jar"], (package / "github/demo-1.2.3.jar").read_bytes()) - self.assertIn("Authorization: Bearer dXNlcjpzZWNyZXQ=", seen["config"]) - self.assertFalse(any("secret" in arg or "dXNlcjpzZWNyZXQ" in arg for arg in seen["args"])) + token = base64.b64encode(b"user:secret").decode() # the fixture's MAVEN_USERNAME/PASSWORD + self.assertIn("Authorization: Bearer " + token, seen["config"]) + self.assertFalse(any("secret" in arg or token in arg for arg in seen["args"])) self.assertTrue(seen["args"][-1].endswith("?publishingType=AUTOMATIC")) def test_release_upload_without_a_valid_receipt_fails(self): diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index caad31a..7e4a4dc 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -70,7 +70,7 @@ e3dde66089fd568f03caeefee86ea36a9f6eb7f47e5dc6d053e0e5784fd7a208 .github/workfl a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workflows/publish.yml#verify-signing-key-gradle 408aec023edac2aa8769d70166ffffa54233ec825862abfc70cec05b39834fbf .github/workflows/publish.yml#github-snapshot f90bda93c5ce115e571d61028808121dbbe65e75771eb853027402486fabd4e1 .github/workflows/publish.yml#github-release -9c1c523f8131ee9129305831125fd90b7f34e8f9aa386d05509f75c720f7ef5d .github/buildcheck/releaseassets.py +e8d3ef1a4d034abc82fc416a66a1a0765239161beaae7b508c373059e79e5606 .github/buildcheck/releaseassets.py 0de7abac98ead2f22c46ea5f48051bf0965811fab674df1c8d55207cba12a1fc .github/buildcheck/snapshot.py 894465632ecdb87c587cd7e18d6fc14f6ac4fa77490154b77188cb4dfc425700 .github/buildcheck/tests/test_releaseassets.py d6eee9187eeb42b8112c6b0a665723e4ce409b4bddb7f52151177a3d593574f5 .github/buildcheck/tests/test_snapshot.py @@ -80,10 +80,10 @@ bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collec 0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo 31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py -7b2a353968eeafb96b9fe0d61b380bebbba9f88dcbf05489c6d3ab3fa80a6ff9 .github/buildcheck/publication.py +3c950825638ced6152192a1adcef9a0eb4f7dd545610cea3508f62a048b309e7 .github/buildcheck/publication.py 9180fac217c75b3e11f67e244b9171f447d2c21665b1802574b57b97019f7173 .github/publication.py -862934ffa7265b3ef5612e287aca6f8f073dd44873c873f7b47affe8b89391b5 .github/buildcheck/tests/test_publication.py -6a18808dc0c672811309a6e34864f5e2fd02699b4f820ea7b5e841b6e1fb627a docs/publication-preflight.md +792b1259417593f198260a469e518cce7faeafeb20cab037c9be00590ce83133 .github/buildcheck/tests/test_publication.py +a6f9fadc79d16eff8ab56b76f313ac9433326e360ba7969d8607c49cf4bafc49 docs/publication-preflight.md aa9989fd22438c3f7cb427ae6bcd6f3b87a9ef014e7d24c2ee9bdc179a137928 .github/workflows/publish.yml#central-snapshot 2180d438fb840a1ca7015cdb95f28c25641c0b151679d95767c4b4393794e4f1 .github/workflows/publish.yml#central-release 8f96d48502f487a311d5cc01dadedc92fe4a88f565e01f05ddbd1ba839bfe70e .github/workflows/dependency-submission.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 40857c2..6c9a1a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **Releases and snapshots are published through one local gate.** The publish jobs stage and sign + everything, `publication.py prepare` checks the exact file set, every signature and the version + against the tag (`v`, no `-SNAPSHOT`; `check-tag` / `check-snapshot` check it first), and + freezes the result; the GitHub release, the Maven Central transfer and the provenance attestation + are independent jobs on that frozen package, so a Central outage no longer hides the GitHub files and + nothing is attached before the gate has passed. Builds run on JDK 25 (`.java-version`); the + published bytecode target is unchanged. Every published jar carries `LICENSE` and `THIRD_PARTY.md` + under `META-INF/third-party/`. + ### Added - OpenSSF Best Practices badge added to README (project 12861). - `CONTRIBUTING.md` with build instructions, test policy, and PR workflow. diff --git a/docs/publication-preflight.md b/docs/publication-preflight.md index 61d1256..0d378b8 100644 --- a/docs/publication-preflight.md +++ b/docs/publication-preflight.md @@ -49,8 +49,9 @@ uploads the frozen `publication-package/` as `signed-release-assets` or `signed- The preparation job exports `assets_complete: ${{ steps.finalize.outputs.complete }}`. The ref is checked twice. `check-snapshot` and `check-tag` run `publication.py check-ref --channel -` before anything is built (`check-snapshot` only on a run that publishes: `main` carries -the release version for the commit a release is tagged on). `prepare` repeats the check against +` after the build and test jobs (they need `report`) and before anything is staged, +signed or published (`check-snapshot` only on a run that publishes: `main` carries the release +version for the commit a release is tagged on). `prepare` repeats the check against `GITHUB_REF` and refuses a channel the version does not have. The gate checks exact file membership, nonempty artifacts/signatures, cryptographic GPG @@ -83,8 +84,8 @@ gate instead of silently staying behind. Gradle writes a SNAPSHOT with a timesta repository (`app-1.0-20261025.085646-1.aar` under `1.0-SNAPSHOT/`); the gate maps such a name back to `app-1.0-SNAPSHOT.aar`, the name the GitHub file and the Maven transfer use. -`python3 .github/publication.py prepare --config ` reads another contract, -as `collect-release-assets.py --config` does. +`python3 .github/publication.py prepare --config ` reads another contract +inside the repository (a path outside it is refused), as `collect-release-assets.py --config` does. ## Independent consumers @@ -114,9 +115,10 @@ use `autoPublish=true` and `waitUntil=uploaded` for manual deployments. For snapshots, a transfer-only Maven POM invokes pinned `maven-deploy-plugin:3.1.4:deploy-file` executions during `validate`. The `central-snapshot` jobs of all runs share one concurrency group, -and `upload --skip-superseded` skips the transfer when another dispatched run already transferred -the snapshot of a newer commit (one that descends from this run's); Central serves the snapshot -transferred last as the latest. The job reads the runs with `actions: read` and nothing else. Maven maintains snapshot timestamps and metadata; signatures are +and `upload --skip-superseded` skips the transfer when one of the 30 most recent dispatched runs +already transferred the snapshot of a newer commit (one that descends from this run's); Central serves the snapshot +transferred last as the latest. The job reads the runs with `actions: read` and nothing else. +Maven maintains snapshot timestamps and metadata; signatures are transferred in the same component request with their proper extensions. There are no source build phases. For a POM-only component its signature is the primary transfer artifact and the signed POM is supplied through `pomFile`, avoiding an empty-classifier parameter edge case. diff --git a/pom.xml b/pom.xml index 21e9bc1..b23f7f3 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,10 @@ SPDX-License-Identifier: Apache-2.0 1.2.28 ${git.commit.time} bernardladenthin + + .github/** 2014 From 529e02f39decc5c71702da0419e88b1dbe488901 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sun, 25 Oct 2026 13:57:41 +0100 Subject: [PATCH 15/19] Sync the shared license collector and publication script Shared (byte-identical in the four Java repos): collect-third-party-licenses.py takes --supplemental (a folder with notices.json mapping JAR base names to committed upstream texts) and --require-complete (fails on a runtime JAR with neither legal files nor a supplement), with tests; dependency-license-review.md says so; publication.py documents why only the snapshot transfer needs -Dmaven.repository.credentialScope=id, which the staging deploys (-DskipUpload=true) no longer pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VszvDmRRPP1UcwozsUMY1L --- .github/buildcheck/publication.py | 4 + .../tests/test_thirdpartylicenses.py | 99 +++++++++++++++++++ .github/collect-third-party-licenses.py | 77 +++++++++++++-- .github/shared-files.sha256 | 7 +- .github/workflows/publish.yml | 4 +- docs/dependency-license-review.md | 10 +- 6 files changed, 186 insertions(+), 15 deletions(-) create mode 100644 .github/buildcheck/tests/test_thirdpartylicenses.py diff --git a/.github/buildcheck/publication.py b/.github/buildcheck/publication.py index e579518..d01bf46 100644 --- a/.github/buildcheck/publication.py +++ b/.github/buildcheck/publication.py @@ -315,6 +315,10 @@ def upload(package, *, skip_superseded=False, github=None): with tempfile.TemporaryDirectory(prefix="snapshot-transfer-") as tmp: pom = Path(tmp) / "pom.xml" snapshot_pom(package, pom) + # credentialScope=id: Maven 3.10 sends a server's credentials only to its declared + # origins, which for the id `central` is repo.maven.apache.org, so the upload to + # central.sonatype.com would go out anonymously (HTTP 401). This is the only upload by + # Maven; the staging deploys run with -DskipUpload=true and need no credentials. subprocess.run(["mvn", "-B", "--no-transfer-progress", "-Dmaven.repository.credentialScope=id", "-f", str(pom), "validate"], check=True) return diff --git a/.github/buildcheck/tests/test_thirdpartylicenses.py b/.github/buildcheck/tests/test_thirdpartylicenses.py new file mode 100644 index 0000000..a4a9d4f --- /dev/null +++ b/.github/buildcheck/tests/test_thirdpartylicenses.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT OR Apache-2.0 +"""collect-third-party-licenses.py: embedded legal files, supplemental notices, completeness.""" + +import importlib.util +import json +import os +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +from buildcheck.tests.helpers import REPO + +_SPEC = importlib.util.spec_from_file_location( + "collect_third_party_licenses", Path(REPO) / ".github/collect-third-party-licenses.py") +licenses = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(licenses) + + +class CollectTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + quiet = patch("builtins.print") + quiet.start() + self.addCleanup(quiet.stop) + + def jar(self, name, entries): + path = self.root / "repo" / name + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w") as archive: + for entry, data in entries.items(): + archive.writestr(entry, data) + return path + + def classpath(self, *jars): + path = self.root / "classpath.txt" + path.write_text(os.pathsep.join(str(jar) for jar in jars), encoding="utf-8") + return path + + def notices(self, notices): + folder = self.root / "notices" + for notice in notices: + (folder / notice["directory"]).mkdir(parents=True, exist_ok=True) + (folder / notice["directory"] / "LICENSE").write_text("license of " + notice["directory"]) + (folder / "notices.json").write_text(json.dumps({"format": 1, "notices": notices})) + return folder + + def test_embedded_files_are_kept_and_code_named_license_is_not(self): + jar = self.jar("lib-1.0.jar", {"META-INF/LICENSE": "MIT", "a/LICENSE.class": b"\xca\xfe", + "META-INF/NOTICE.txt": "notice"}) + entries = licenses.collect(self.classpath(jar), self.root / "out") + self.assertEqual(entries[0]["legal_files"], ["META-INF/LICENSE", "META-INF/NOTICE.txt"]) + self.assertNotIn("supplemental_files", entries[0]) + + def test_a_jar_without_legal_files_gets_the_supplied_notice_by_base_name(self): + bare = self.jar("lwjgl-opencl-3.3.6-natives-linux.jar", {"x.class": b"\xca\xfe"}) + folder = self.notices([{"directory": "lwjgl-3.3.6", "license": "BSD-3-Clause", + "artifacts": ["lwjgl", "lwjgl-opencl"]}]) + entries = licenses.collect(self.classpath(bare), self.root / "out", folder, require_complete=True) + self.assertEqual(entries[0]["supplemental_files"], ["supplemental/LICENSE"]) + copied = self.root / "out" / entries[0]["directory"] / "supplemental/LICENSE" + self.assertEqual(copied.read_text(), "license of lwjgl-3.3.6") + + def test_embedded_files_win_over_a_supplied_notice(self): + jar = self.jar("lib-1.0.jar", {"META-INF/LICENSE": "MIT"}) + folder = self.notices([{"directory": "lib", "license": "MIT", "artifacts": ["lib"]}]) + entries = licenses.collect(self.classpath(jar), self.root / "out", folder) + self.assertNotIn("supplemental_files", entries[0]) + + def test_require_complete_fails_on_a_jar_without_any_legal_files(self): + bare = self.jar("bare-2.0.jar", {"x.class": b"\xca\xfe"}) + with self.assertRaisesRegex(ValueError, "bare-2.0.jar"): + licenses.collect(self.classpath(bare), self.root / "out", None, require_complete=True) + entries = licenses.collect(self.classpath(bare), self.root / "out2") + self.assertEqual(entries[0]["legal_files"], []) + + def test_malformed_or_ambiguous_notices_fail(self): + folder = self.notices([{"directory": "a", "license": "MIT", "artifacts": ["x"]}, + {"directory": "b", "license": "MIT", "artifacts": ["x"]}]) + with self.assertRaisesRegex(ValueError, "listed twice"): + licenses.supplemental_notices(folder) + (folder / "notices.json").write_text(json.dumps({"format": 1, "notices": [ + {"directory": "missing", "license": "MIT", "artifacts": ["y"]}]})) + with self.assertRaisesRegex(ValueError, "missing or empty"): + licenses.supplemental_notices(folder) + + def test_base_name_strips_version_and_classifier(self): + self.assertEqual(licenses.base_name("lwjgl-opencl-3.3.6-natives-linux.jar"), "lwjgl-opencl") + self.assertEqual(licenses.base_name("jcip-annotations-1.0.jar"), "jcip-annotations") + self.assertEqual(licenses.base_name("plain.jar"), "plain") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/collect-third-party-licenses.py b/.github/collect-third-party-licenses.py index d337a08..77fb096 100644 --- a/.github/collect-third-party-licenses.py +++ b/.github/collect-third-party-licenses.py @@ -6,6 +6,12 @@ Input is maven-dependency-plugin:build-classpath output on this operating system. This performs no license approvals, downloads, source rewriting or file filtering by license type. Archives without legal files remain visible in inventory.json. + +A JAR that carries no legal files of its own gets the texts a repository supplies for it +(--supplemental: a directory with notices.json naming, per notice directory, the JAR base +names it covers -- the file name without version, classifier and extension). With +--require-complete a JAR that has neither fails the run, so a new dependency without +legal files cannot reach a fat jar unnoticed. """ import argparse import hashlib @@ -31,7 +37,34 @@ def legal_file(name): or re.fullmatch(r"(?:licen[sc]e|notice|copying|copyright)(?:[-_.].*)?", lower) is not None) -def collect(classpath, output): +def base_name(jar_name): + """`lwjgl-opencl-3.3.6-natives-linux.jar` -> `lwjgl-opencl`: everything before the version.""" + match = re.match(r"(.+?)-\d", jar_name) + return match.group(1) if match else jar_name[:-len(".jar")] + + +def supplemental_notices(directory): + """Map JAR base name -> notice directory, from /notices.json.""" + if directory is None: + return {} + config = json.loads((directory / "notices.json").read_text(encoding="utf-8")) + if set(config) - {"format", "notices"} or config.get("format") != 1: + raise ValueError("notices.json needs {\"format\": 1, \"notices\": [...]}") + result = {} + for notice in config["notices"]: + if set(notice) - {"directory", "artifacts", "license"} or not notice.get("artifacts"): + raise ValueError("a notice needs directory, artifacts and license: " + json.dumps(notice)) + folder = directory / notice["directory"] + if not folder.is_dir() or not any(path.is_file() for path in folder.rglob("*")): + raise ValueError("notice directory missing or empty: " + str(folder)) + for artifact in notice["artifacts"]: + if artifact in result: + raise ValueError("JAR base name listed twice in notices.json: " + artifact) + result[artifact] = folder + return result + + +def collect(classpath, output, supplemental=None, require_complete=False): marker = output / ".generated-by-collect-third-party-licenses" if output.is_symlink(): raise ValueError("Output must not be a symlink") @@ -39,9 +72,11 @@ def collect(classpath, output): if not marker.is_file(): raise ValueError("Refusing to replace an output directory without our marker") shutil.rmtree(output) + notices = supplemental_notices(supplemental) output.mkdir(parents=True) marker.write_text("Generated runtime dependency license inventory.\n", encoding="utf-8") entries = [] + used = set() for item in sorted(set(classpath.read_text(encoding="utf-8").strip().split(os.pathsep))): if not item: continue @@ -68,22 +103,50 @@ def collect(classpath, output): raise ValueError("Conflicting duplicate entry: " + info.filename) destination.write_bytes(data) files.append(info.filename) - entries.append({"artifact": source.name, "sha256": digest, - "directory": directory, "legal_files": files}) + entry = {"artifact": source.name, "sha256": digest, "directory": directory, "legal_files": files} + folder = notices.get(base_name(source.name)) + if not files and folder is not None: + supplied = [] + for path in sorted(p for p in folder.rglob("*") if p.is_file()): + relative = "supplemental/" + path.relative_to(folder).as_posix() + destination = output / directory / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, destination) + supplied.append(relative) + entry["supplemental_files"] = supplied + used.add(base_name(source.name)) + entries.append(entry) (output / "inventory.json").write_text( json.dumps({"format": 1, "scope": "runtime", "artifacts": entries}, indent=2) + "\n", encoding="utf-8") - missing = sum(not entry["legal_files"] for entry in entries) - print(f"Runtime legal inventory: {len(entries)} JARs, {missing} without embedded legal files") + missing = [entry["artifact"] for entry in entries + if not entry["legal_files"] and not entry.get("supplemental_files")] + supplied = sum(bool(entry.get("supplemental_files")) for entry in entries) + print(f"Runtime legal inventory: {len(entries)} JARs, {supplied} with supplemental notices, " + f"{len(missing)} without any legal files") + stale = sorted(set(notices) - used) + for name in stale: + print(f"Note: notices.json covers {name}, which no runtime JAR without its own legal files needs") + if require_complete and missing: + raise ValueError("JARs without legal files and without a supplemental notice: " + ", ".join(missing)) + return entries def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--classpath", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--supplemental", type=Path, help="directory with notices.json and the notice texts") + parser.add_argument("--require-complete", action="store_true", + help="fail when a JAR has neither legal files nor a supplemental notice") args = parser.parse_args() - collect(args.classpath, args.output) + try: + collect(args.classpath, args.output, args.supplemental, args.require_complete) + except (ValueError, OSError) as error: + print(f"::error::{error}") + return 1 + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 7e4a4dc..2bde783 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -77,10 +77,11 @@ d6eee9187eeb42b8112c6b0a665723e4ce409b4bddb7f52151177a3d593574f5 .github/buildc bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license -0ed4e1ebe554c9dabf455ef1bfb3bebc292d010138f2190abb39d3aba9ea80f5 docs/dependency-license-review.md +cda6313687d98ad4b9f10ffe9b917656310bca0286fc0349b663dcc8fe24615a docs/dependency-license-review.md 8c7e4771f4543aebecbec1013ff091196c5abe475e1c7bc90738244d9c5a02d2 .fossa.yml?repo -31089ad5513d5db2e8ab3c73a3e960c105d91423b93617092c3a32f7c43d9355 .github/collect-third-party-licenses.py -3c950825638ced6152192a1adcef9a0eb4f7dd545610cea3508f62a048b309e7 .github/buildcheck/publication.py +675abf347248393ab656ccba51d76b9172ec77fc9a2f25a17161dd733569f7d4 .github/collect-third-party-licenses.py +63a83d9e2f3da55aeae0b20ef0a8c48da0114aef34bd1e3eaba5e427ca541024 .github/buildcheck/tests/test_thirdpartylicenses.py +4e3c1b33c1960b8fc8c5bb892b8c7861eb71deb50b989edcf83202838e786a0d .github/buildcheck/publication.py 9180fac217c75b3e11f67e244b9171f447d2c21665b1802574b57b97019f7173 .github/publication.py 792b1259417593f198260a469e518cce7faeafeb20cab037c9be00590ce83133 .github/buildcheck/tests/test_publication.py a6f9fadc79d16eff8ab56b76f313ac9433326e360ba7969d8607c49cf4bafc49 docs/publication-preflight.md diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 74f87e1..03e7b50 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -446,7 +446,7 @@ jobs: - name: Show effective POM (debug) run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true help:effective-pom - name: Stage snapshot - run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests + run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true deploy -DskipTests env: MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - name: Finalize and secure the signed publication @@ -513,7 +513,7 @@ jobs: - name: Show effective POM (debug) run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true help:effective-pom - name: Stage release - run: mvn --batch-mode --no-transfer-progress -Dmaven.repository.credentialScope=id -P release -DskipUpload=true deploy -DskipTests + run: mvn --batch-mode --no-transfer-progress -P release -DskipUpload=true deploy -DskipTests env: MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - name: Finalize and secure the signed publication diff --git a/docs/dependency-license-review.md b/docs/dependency-license-review.md index 76b82d0..080a883 100644 --- a/docs/dependency-license-review.md +++ b/docs/dependency-license-review.md @@ -65,6 +65,10 @@ copied legal entries with the original JAR bytes. The collector reads the resolv classpath, preserves classifiers and distinguishes equal filenames by content hash. It never treats a `LICENSE.class` bytecode file as a text license. BAF's assembly profile and the srcmorph CLI run it before packaging; Python 3.8+ is needed on that build host. -JARs with no legal entries remain explicitly listed. No network requests or approvals are made. -Missing legal material must be supplied explicitly. This check does not replace -source-availability obligations or the review of native third-party components. +No network requests or approvals are made. A JAR with no legal entries of its own gets the +upstream texts the repository commits for it (`--supplemental third-party-notices`: `notices.json` +maps JAR base names to folders, `SOURCES.tsv` records each text's source URL and SHA-256; listed as +`supplemental_files` in the inventory), and `--require-complete` fails the build when a runtime JAR +has neither. This check does not replace source-availability obligations. java-llama.cpp's natives +jars carry the texts of the code compiled into them (`llama/legal/natives/`, `NOTICE.md` is the +index), so a fat jar bundling them needs no supplement for them. From bfc898a8d4ce0334e8e90303e457824043afa429 Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sun, 25 Oct 2026 15:11:05 +0100 Subject: [PATCH 16/19] Snapshot: create the first rolling release as a draft until its upload is complete Synced from java-llama.cpp (shared file). When a repository had no snapshot release yet, update-snapshot.py created the tag and a public pre-release naming the new build before the upload; a failed upload left an empty or partial public snapshot behind. The first release is now a draft until the upload is complete, then tagged and published with its notes; a draft a failed run left behind is reused. Existing snapshots are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VszvDmRRPP1UcwozsUMY1L --- .github/buildcheck/snapshot.py | 21 ++++++++----- .github/buildcheck/tests/test_snapshot.py | 38 +++++++++++++++++++++-- .github/shared-files.sha256 | 4 +-- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/buildcheck/snapshot.py b/.github/buildcheck/snapshot.py index 4374cb8..253535c 100644 --- a/.github/buildcheck/snapshot.py +++ b/.github/buildcheck/snapshot.py @@ -5,7 +5,8 @@ The workflow serializes this entire operation with a job concurrency group. The ancestry/run checks also reject an older build which finishes after a newer build has already published. -An incomplete collection changes nothing: no upload, no tag or notes update, no prune. +An incomplete collection changes nothing: no upload, no tag or notes update, no prune. The first +snapshot starts as a draft, so a failed first upload leaves neither a public release nor a tag. """ import json @@ -93,12 +94,16 @@ def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, f"(https://github.com/{repository}/actions/runs/{run_id}/attempts/{attempt}).\n\n" f"") if not release: - if not ref: - ref = gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) - release = gh.api("POST", f"{repo}/releases", { - "tag_name": tag, "target_commitish": sha, "name": title, - "prerelease": True, "body": body - }) + # The first snapshot is a draft until its upload is complete: no public release and no tag + # that names this build before its assets are there. A draft a failed run left behind is + # reused (the tag lookup above does not see drafts). + release = next((r for r in gh.api("GET", f"{repo}/releases?per_page=100") or [] + if r.get("draft") and r.get("tag_name") == tag), None) + if not release: + release = gh.api("POST", f"{repo}/releases", { + "tag_name": tag, "target_commitish": sha, "name": title, "prerelease": True, + "draft": True, "body": "Snapshot upload in progress." + }) gh.command("release", "upload", tag, *files, "--repo", repository, "--clobber") if ref: @@ -106,7 +111,7 @@ def publish(repository, sha, run_id, attempt, assets_dir, complete, github=None, else: gh.api("POST", f"{repo}/git/refs", {"ref": f"refs/tags/{tag}", "sha": sha}) gh.api("PATCH", f"{repo}/releases/{release['id']}", { - "target_commitish": sha, "name": title, "prerelease": True, "body": body + "target_commitish": sha, "name": title, "prerelease": True, "draft": False, "body": body }) # Read every page before deleting: deleting while paginating could skip shifted entries. diff --git a/.github/buildcheck/tests/test_snapshot.py b/.github/buildcheck/tests/test_snapshot.py index 7593ea2..b8ab8c8 100644 --- a/.github/buildcheck/tests/test_snapshot.py +++ b/.github/buildcheck/tests/test_snapshot.py @@ -26,6 +26,7 @@ def __init__(self): self.fail_upload = False self.fail_deletes = set() self.tag = "snapshot" + self.drafts = [] def api(self, method, endpoint, payload=None, optional=False): self.calls.append((method, endpoint, payload)) @@ -42,6 +43,8 @@ def api(self, method, endpoint, payload=None, optional=False): return {"object": {"type": "commit", "sha": OLD}} if "/assets?" in endpoint: return self.pages[int(endpoint.split("&page=")[1])] + if endpoint.endswith("/releases?per_page=100"): + return self.drafts if method == "POST" and endpoint.endswith("/releases"): return {"id": 7} if method == "POST" and endpoint.endswith("/git/refs"): @@ -153,8 +156,39 @@ def test_annotated_tags_are_resolved_before_comparing_commits(self): def test_creates_missing_tag_and_release_once(self): self.gh.ref, self.gh.release = None, None self.assertTrue(self.publish()) - self.assertEqual(sum(c[0] == "POST" and c[1].endswith("/git/refs") for c in self.gh.calls), 1) - self.assertTrue(any(c[0] == "POST" and c[1].endswith("/releases") for c in self.gh.calls)) + calls = self.gh.calls + self.assertEqual(sum(c[0] == "POST" and c[1].endswith("/git/refs") for c in calls), 1) + created = [c[2] for c in calls if c[0] == "POST" and c[1].endswith("/releases")] + self.assertEqual(len(created), 1) + # A draft without the build's notes until the upload is complete; the tag comes after it. + self.assertTrue(created[0]["draft"]) + self.assertNotIn(NEW, created[0]["body"]) + upload = next(i for i, c in enumerate(calls) if c[0] == "upload") + tag = next(i for i, c in enumerate(calls) if c[0] == "POST" and c[1].endswith("/git/refs")) + self.assertLess(upload, tag) + notes = next(c[2] for c in calls if c[0] == "PATCH" and c[1].endswith("/releases/7")) + self.assertFalse(notes["draft"]) + self.assertIn(NEW, notes["body"]) + + def test_failed_first_upload_leaves_no_tag_and_no_public_release(self): + self.gh.ref, self.gh.release = None, None + self.gh.fail_upload = True + with self.assertRaises(subprocess.CalledProcessError): + self.publish() + self.assertFalse(any(c[0] in ("PATCH", "DELETE") or c[1].endswith("/git/refs") + for c in self.gh.calls if c[0] != "upload")) + created = next(c[2] for c in self.gh.calls if c[0] == "POST" and c[1].endswith("/releases")) + self.assertTrue(created["draft"]) + + def test_a_draft_a_failed_first_upload_left_is_reused(self): + self.gh.ref, self.gh.release = None, None + self.gh.drafts = [{"id": 5, "draft": False, "tag_name": "snapshot"}, + {"id": 6, "draft": True, "tag_name": "other"}, + {"id": 8, "draft": True, "tag_name": "snapshot"}] + self.assertTrue(self.publish()) + self.assertFalse(any(c[0] == "POST" and c[1].endswith("/releases") for c in self.gh.calls)) + notes = next(c[2] for c in self.gh.calls if c[0] == "PATCH" and c[1].endswith("/releases/8")) + self.assertFalse(notes["draft"]) def test_reads_all_asset_pages_before_deleting(self): self.gh.pages = {1: [{"id": i, "name": f"old-{i}.jar"} for i in range(100)], diff --git a/.github/shared-files.sha256 b/.github/shared-files.sha256 index 2bde783..3c6330c 100644 --- a/.github/shared-files.sha256 +++ b/.github/shared-files.sha256 @@ -71,9 +71,9 @@ a4f42a0bfa9b14ca937675052b990e5f1dda238d762f9d0268ac282cd7030c0d .github/workfl 408aec023edac2aa8769d70166ffffa54233ec825862abfc70cec05b39834fbf .github/workflows/publish.yml#github-snapshot f90bda93c5ce115e571d61028808121dbbe65e75771eb853027402486fabd4e1 .github/workflows/publish.yml#github-release e8d3ef1a4d034abc82fc416a66a1a0765239161beaae7b508c373059e79e5606 .github/buildcheck/releaseassets.py -0de7abac98ead2f22c46ea5f48051bf0965811fab674df1c8d55207cba12a1fc .github/buildcheck/snapshot.py +7f8c715af79007e851d337b1dac600ff93d14f7f831c46e41300e5441e0f0fa5 .github/buildcheck/snapshot.py 894465632ecdb87c587cd7e18d6fc14f6ac4fa77490154b77188cb4dfc425700 .github/buildcheck/tests/test_releaseassets.py -d6eee9187eeb42b8112c6b0a665723e4ce409b4bddb7f52151177a3d593574f5 .github/buildcheck/tests/test_snapshot.py +6479634aaa6f2b18341760477c6dfb2f5bc82fdc08f1a32aa8a431f1138628b5 .github/buildcheck/tests/test_snapshot.py bf43c70a38ea1db6a92b35b8e97bb442d2fddcd5b3401919976ac139348e2a0e .github/collect-release-assets.py 930cd6f5ec26cb037acbcc65d43a01905056cbf9f51b618fd841b1373f4125f7 .github/update-snapshot.py 71ec85b7a30e80b18a9ab1fac0199cc50d68548e6f00dd116ab0f904ad63aa50 .github/release-assets.json.license From eaaf49588a4c7000508b18165f442325b70192ea Mon Sep 17 00:00:00 2001 From: Bernard Ladenthin Date: Sun, 25 Oct 2026 15:48:45 +0100 Subject: [PATCH 17/19] Move the shared CI code to bernardladenthin/actions The build-check library and its tests, the check/publication/snapshot entry points, the shared shell scripts, the signing self-test, the finalize-publication action, the shared workflows (codeql, dependency submission, reuse, osv-scanner, claude, claude-code-review, sonarqube where shared) and the two shared docs move to bernardladenthin/actions, unchanged, where they exist once instead of as four copies held together by shared-files.sha256. This commit only removes them; the next one wires this repository to the actions. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VszvDmRRPP1UcwozsUMY1L --- .../actions/finalize-publication/action.yml | 49 -- .github/buildcheck/__init__.py | 15 - .github/buildcheck/publication.py | 394 ---------------- .github/buildcheck/releaseassets.py | 209 --------- .github/buildcheck/releasegate.py | 60 --- .github/buildcheck/runscripts.py | 273 ----------- .github/buildcheck/sharedfiles.py | 178 -------- .github/buildcheck/snapshot.py | 136 ------ .github/buildcheck/tests/__init__.py | 3 - .github/buildcheck/tests/helpers.py | 31 -- .github/buildcheck/tests/test_publication.py | 432 ------------------ .../buildcheck/tests/test_releaseassets.py | 289 ------------ .github/buildcheck/tests/test_releasegate.py | 61 --- .github/buildcheck/tests/test_runscripts.py | 127 ----- .github/buildcheck/tests/test_sharedfiles.py | 209 --------- .github/buildcheck/tests/test_snapshot.py | 264 ----------- .../tests/test_thirdpartylicenses.py | 99 ---- .github/buildcheck/tests/test_versions.py | 161 ------- .github/buildcheck/tests/test_workflow.py | 70 --- .github/buildcheck/versions.py | 181 -------- .github/buildcheck/workflow.py | 120 ----- .github/check-release-gate.py | 33 -- .github/check-run-scripts.py | 16 - .github/check-shared-files.py | 20 - .github/check-versions.py | 16 - .github/collect-release-assets.py | 49 -- .github/collect-third-party-licenses.py | 152 ------ .github/print-crash-logs.sh | 48 -- .github/print-host-info.sh | 37 -- .github/publication.py | 14 - .github/signing-selftest/build.gradle.kts | 41 -- .github/signing-selftest/settings.gradle.kts | 5 - .github/update-snapshot.py | 32 -- .github/verify-bytecode-version.sh | 160 ------- .github/verify-signing-key.sh | 100 ---- .github/workflows/claude-code-review.yml | 38 -- .github/workflows/claude.yml | 53 --- .github/workflows/codeql.yml | 31 -- .github/workflows/dependency-submission.yml | 33 -- .github/workflows/osv-scanner.yml | 33 -- .github/workflows/reuse.yml | 16 - .github/workflows/sonarqube.yml | 40 -- docs/dependency-license-review.md | 74 --- docs/publication-preflight.md | 130 ------ 44 files changed, 4532 deletions(-) delete mode 100644 .github/actions/finalize-publication/action.yml delete mode 100644 .github/buildcheck/__init__.py delete mode 100644 .github/buildcheck/publication.py delete mode 100644 .github/buildcheck/releaseassets.py delete mode 100644 .github/buildcheck/releasegate.py delete mode 100644 .github/buildcheck/runscripts.py delete mode 100644 .github/buildcheck/sharedfiles.py delete mode 100644 .github/buildcheck/snapshot.py delete mode 100644 .github/buildcheck/tests/__init__.py delete mode 100644 .github/buildcheck/tests/helpers.py delete mode 100644 .github/buildcheck/tests/test_publication.py delete mode 100644 .github/buildcheck/tests/test_releaseassets.py delete mode 100644 .github/buildcheck/tests/test_releasegate.py delete mode 100644 .github/buildcheck/tests/test_runscripts.py delete mode 100644 .github/buildcheck/tests/test_sharedfiles.py delete mode 100644 .github/buildcheck/tests/test_snapshot.py delete mode 100644 .github/buildcheck/tests/test_thirdpartylicenses.py delete mode 100644 .github/buildcheck/tests/test_versions.py delete mode 100644 .github/buildcheck/tests/test_workflow.py delete mode 100644 .github/buildcheck/versions.py delete mode 100644 .github/buildcheck/workflow.py delete mode 100755 .github/check-release-gate.py delete mode 100755 .github/check-run-scripts.py delete mode 100755 .github/check-shared-files.py delete mode 100755 .github/check-versions.py delete mode 100644 .github/collect-release-assets.py delete mode 100644 .github/collect-third-party-licenses.py delete mode 100755 .github/print-crash-logs.sh delete mode 100755 .github/print-host-info.sh delete mode 100644 .github/publication.py delete mode 100644 .github/signing-selftest/build.gradle.kts delete mode 100644 .github/signing-selftest/settings.gradle.kts delete mode 100644 .github/update-snapshot.py delete mode 100755 .github/verify-bytecode-version.sh delete mode 100755 .github/verify-signing-key.sh delete mode 100644 .github/workflows/claude-code-review.yml delete mode 100644 .github/workflows/claude.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/dependency-submission.yml delete mode 100644 .github/workflows/osv-scanner.yml delete mode 100644 .github/workflows/reuse.yml delete mode 100644 .github/workflows/sonarqube.yml delete mode 100644 docs/dependency-license-review.md delete mode 100644 docs/publication-preflight.md diff --git a/.github/actions/finalize-publication/action.yml b/.github/actions/finalize-publication/action.yml deleted file mode 100644 index b6399c9..0000000 --- a/.github/actions/finalize-publication/action.yml +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# SPDX-License-Identifier: MIT OR Apache-2.0 - -name: Finalize signed publication -description: Collect, verify and secure the complete signed package before any publication consumer runs. -inputs: - channel: - description: Publication channel (snapshot or release). - required: true - maven-assets: - description: Optional directory containing Maven outputs preserved before a rebuild. - required: false - default: '' -outputs: - complete: - description: True only after the local gate AND the Actions handoff upload succeed. - value: ${{ steps.ready.outputs.complete }} -runs: - using: composite - steps: - - name: Collect signed artifacts - id: collect - shell: bash - env: - CHANNEL: ${{ inputs.channel }} - MAVEN_ASSETS: ${{ inputs.maven-assets }} - run: | - set -euo pipefail - case "$CHANNEL" in snapshot|release) ;; *) echo "::error::Invalid publication channel"; exit 1 ;; esac - args=("signed-${CHANNEL}-assets") - if [ -n "$MAVEN_ASSETS" ]; then args+=(--maven-assets "$MAVEN_ASSETS"); fi - python3 .github/collect-release-assets.py "${args[@]}" - - name: Verify and freeze the complete signed publication - id: preflight - shell: bash - env: - CHANNEL: ${{ inputs.channel }} - run: python3 .github/publication.py prepare "signed-${CHANNEL}-assets" publication-package --channel "$CHANNEL" - - name: Secure the checked publication package - id: handoff - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - with: - name: signed-${{ inputs.channel }}-assets - path: publication-package/ - if-no-files-found: error - - name: Release the secured package to independent consumers - id: ready - shell: bash - run: echo 'complete=true' >> "$GITHUB_OUTPUT" diff --git a/.github/buildcheck/__init__.py b/.github/buildcheck/__init__.py deleted file mode 100644 index 5beaff1..0000000 --- a/.github/buildcheck/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# -# SPDX-License-Identifier: MIT OR Apache-2.0 -"""Build checks as a library, so buildcheck/tests can test them. Standard library only. - -Two kinds of module live here. workflow.py, releasegate.py, sharedfiles.py, versions.py, -runscripts.py, releaseassets.py, publication.py and snapshot.py (with their tests and the *.py -entry points next to this package) are kept BYTE-IDENTICAL in java-llama.cpp, -BitcoinAddressFinder, srcmorph and streambuffer: each repository lists them in -.github/shared-files.sha256, which its `shared-files` job checks (see sharedfiles.py). -Every other module is the repository's own. - -Run the tests from the repository root: - python3 -m unittest discover -s .github/buildcheck/tests -t .github -""" diff --git a/.github/buildcheck/publication.py b/.github/buildcheck/publication.py deleted file mode 100644 index d01bf46..0000000 --- a/.github/buildcheck/publication.py +++ /dev/null @@ -1,394 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# SPDX-License-Identifier: MIT OR Apache-2.0 -"""Freeze signed publication bytes once; independent consumers never rebuild them.""" -import argparse -import base64 -import hashlib -import json -import os -from pathlib import Path, PurePosixPath -import re -import shutil -import subprocess -import tempfile -import xml.etree.ElementTree as ET -import zipfile - -from .releaseassets import configuration, projects, signature_gaps, _classifiers, _format, _inside -from .snapshot import GitHub - -PORTAL = "https://central.sonatype.com/api/v1/publisher/upload" -SNAPSHOTS = "https://central.sonatype.com/repository/maven-snapshots/" -DEPLOY_PLUGIN = "3.1.4" -# The name of the shared central-snapshot job; superseded() looks for it in other runs. -CENTRAL_SNAPSHOT_JOB = "Upload checked snapshot artifacts to Maven Central" -# Publication payload in a Central staging directory. Repository metadata and checksums are -# regenerated by the transfer; anything else in a staging directory fails the gate. -PAYLOAD = re.compile(r".+\.(?:pom|jar|aar|json)(?:\.asc)?") -REGENERATED = re.compile(r"maven-metadata[\w.-]*\.xml|.+\.(?:md5|sha1|sha256|sha512)|\.index") -# GitHub renames release assets with other characters; a renamed asset would no longer match -# the publication manifest, and update-snapshot.py would prune it as obsolete. -ASSET_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") - - -def digest(path): - h = hashlib.sha256() - with path.open("rb") as stream: - for block in iter(lambda: stream.read(1024 * 1024), b""): - h.update(block) - return h.hexdigest() - - -def safe_path(root, name): - p = PurePosixPath(name) - if not name or p.is_absolute() or ".." in p.parts or "\\" in name or ":" in name: - raise ValueError("unsafe publication path: " + name) - target = root / name - if target.is_symlink() or any(parent.is_symlink() for parent in target.parents): - raise ValueError("publication symlink: " + name) - return target - - -def check_ref(version, ref, channel=None): - """A release publishes only from the tag v; a snapshot never from a tag.""" - actual = "snapshot" if version.endswith("-SNAPSHOT") else "release" - if channel is not None and channel != actual: - raise ValueError(f"{channel} publication of {actual} version {version}") - if ref is None: - return actual - tag = ref[len("refs/tags/"):] if ref.startswith("refs/tags/") else None - if actual == "snapshot" and tag is not None: - raise ValueError(f"tag {tag} names SNAPSHOT version {version}; a release tag needs a release version") - if actual == "release" and tag != "v" + version: - raise ValueError(f"release version {version} publishes only from tag v{version}, not {ref}") - return actual - - -def central_name(name, artifact, version): - """Undo a timestamped snapshot name (Gradle writes one even to a local file repository).""" - if not version.endswith("-SNAPSHOT"): - return name - prefix = f"{artifact}-{version[:-len('-SNAPSHOT')]}-" - match = re.fullmatch(re.escape(prefix) + r"\d{8}\.\d{6}-\d+(.*)", name) - return f"{artifact}-{version}{match[1]}" if match else name - - -def expected_central(root, config, modules, *, only_central=True): - names = set() - for _, artifact, version, packaging, reactor in modules: - base = f"{artifact}-{version}" - names.add(base + ".pom") - if packaging != "pom": - names.update(base + suffix + ".jar" for suffix in ("", "-sources", "-javadoc")) - if reactor and config.get("maven", {}).get("cyclonedx", False): - names.add(base + "-cyclonedx.json") - for rule in config.get("files", []): - if only_central and not rule.get("central", False): - continue - for classifier in _classifiers(root, rule.get("classifiers", [""])): - matches = list(root.glob(_format(rule["source"], modules[0][2], classifier))) - if len(matches) != 1: - raise ValueError("missing/ambiguous Central input: " + rule["source"]) - names.add(_format(rule["name"], modules[0][2], classifier) if "name" in rule else matches[0].name) - return names - - -def prepare(root, assets, destination, *, config_path=None, channel=None, ref=None): - """Gate both publishers on exact membership, valid signatures and frozen checksums. - - channel is the one the caller publishes to; ref defaults to GITHUB_REF ("": no ref check). - """ - root, assets, destination = Path(root).resolve(), Path(assets).resolve(), Path(destination).resolve() - if destination.exists(): - raise ValueError("publication destination must not exist") - missing, errors = signature_gaps(assets) - if missing: - raise ValueError("incomplete publication: " + "; ".join(errors)) - config = configuration(root, config_path) - publication = config.get("publication", {}) - modules = projects(root, config.get("maven", {}).get("standalone", [])) - version = modules[0][2] - ref = os.environ.get("GITHUB_REF") if ref is None else ref - channel = check_ref(version, ref or None, channel) - staging = "central-deferred" if channel == "snapshot" else "central-staging" - roots = publication.get("central_roots", ["target/{staging}"]) - central = {} - for item in roots: - folder = _inside(root, item.format(staging=staging)) - if not folder.is_dir(): - raise ValueError("missing Central staging: " + str(folder)) - for path in folder.rglob("*"): - if not path.is_file(): - continue - relative = path.relative_to(folder).as_posix() - safe_path(folder, relative) - if not PAYLOAD.fullmatch(path.name): - if REGENERATED.fullmatch(path.name): - continue # Maven metadata/checksums are regenerated, never mistaken for payload. - raise ValueError("unexpected file in Central staging: " + relative) - parts = PurePosixPath(relative).parts - if len(parts) < 4 or parts[-2] != version: - raise ValueError("unexpected Central coordinates: " + relative) - relative = PurePosixPath(*parts[:-1], central_name(parts[-1], parts[-3], version)).as_posix() - if relative in central and digest(central[relative]) != digest(path): - raise ValueError("conflicting Central input: " + relative) - central[relative] = path - names = [PurePosixPath(relative).name for relative in central] - if len(set(names)) != len(names): - raise ValueError("one file name under two Central coordinates: " - + str(sorted({name for name in names if names.count(name) > 1}))) - expected = expected_central(root, config, modules) - # Each artifact with its own signature, and no signature without its artifact. - wanted = expected | {name + ".asc" for name in expected} - if set(names) != wanted: - raise ValueError("Central membership mismatch; missing=" + str(sorted(wanted - set(names))) - + "; unexpected=" + str(sorted(set(names) - wanted))) - github_expected = expected_central(root, config, modules, only_central=False) - github_actual = {p.name for p in assets.iterdir() if not p.name.endswith(".asc")} - if github_actual != github_expected: - raise ValueError("GitHub membership does not match the release contract; missing=" - + str(sorted(github_expected - github_actual)) - + "; unexpected=" + str(sorted(github_actual - github_expected))) - # Require the Central pair to match GitHub; verify signatures on the frozen copy. - for path in sorted(assets.iterdir()): - if path.is_symlink() or not path.is_file(): - raise ValueError("non-flat publication input: " + str(path)) - if not ASSET_NAME.fullmatch(path.name): - raise ValueError("GitHub would rename release asset: " + path.name) - for relative, path in central.items(): - name = PurePosixPath(relative).name - if not (assets / name).is_file() or digest(assets / name) != digest(path): - raise ValueError("Central/GitHub bytes differ: " + relative) - # Build a temporary package, then promote it locally only after all checks succeed. - destination.parent.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix="publication-", dir=destination.parent) as tmp: - package = Path(tmp) / "package" - (package / "github").mkdir(parents=True) - for path in assets.iterdir(): - shutil.copyfile(path, package / "github" / path.name) - for relative, path in central.items(): - target = safe_path(package / "central", relative) - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(path, target) - for path in sorted((package / "github").iterdir()): - if not path.name.endswith(".asc"): - subprocess.run(["gpg", "--batch", "--verify", str(path) + ".asc", str(path)], check=True) - for path in (package / "central").rglob("*"): - if path.is_file() and digest(path) != digest(package / "github" / path.name): - raise ValueError("frozen Central/GitHub files differ: " + path.name) - manifest = {"schema": 1, "version": version, "channel": channel, - "commit": os.environ.get("GITHUB_SHA", ""), "files": {}} - for path in sorted(package.rglob("*")): - if path.is_file(): - manifest["files"][path.relative_to(package).as_posix()] = { - "sha256": digest(path), "size": path.stat().st_size} - (package / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") - shutil.move(str(package), destination) - if os.environ.get("GITHUB_OUTPUT"): - with open(os.environ["GITHUB_OUTPUT"], "a") as output: - output.write("complete=true\n") - print(f"Publication ready: {version}; {len(expected)} Central artifacts; all signatures verified") - - -def verify(package): - """Transport integrity check; never regenerate or re-sign publication files.""" - package = Path(package).resolve() - manifest = json.loads((package / "manifest.json").read_text()) - if manifest.get("schema") != 1 or manifest.get("channel") not in ("snapshot", "release"): - raise ValueError("unsupported publication manifest") - if os.environ.get("GITHUB_SHA") and manifest.get("commit") != os.environ["GITHUB_SHA"]: - raise ValueError("publication belongs to a different commit") - actual = {p.relative_to(package).as_posix() for p in package.rglob("*") if p.is_file()} - if actual != set(manifest["files"]) | {"manifest.json"}: - raise ValueError("publication file membership changed") - for relative, info in manifest["files"].items(): - path = safe_path(package, relative) - if not path.is_file() or path.stat().st_size != info["size"] or digest(path) != info["sha256"]: - raise ValueError("publication bytes changed: " + relative) - return manifest - - -def snapshot_pom(package, destination, url=SNAPSHOTS): - """Generate transfer-only deploy-file executions, including all signature extensions. - - Maven handles timestamped snapshot names and metadata in one request per component. - There is no project build, dependency resolution, compilation or signing phase. - """ - manifest = verify(package) - if manifest["channel"] != "snapshot": - raise ValueError("snapshot transfer requires a SNAPSHOT version") - root = ET.Element("project", xmlns="http://maven.apache.org/POM/4.0.0") - def child(parent, name, text): - node = ET.SubElement(parent, name) - node.text = str(text) - return node - for name, value in [("modelVersion", "4.0.0"), ("groupId", "local.publication"), - ("artifactId", "snapshot-transfer"), ("version", "1"), ("packaging", "pom")]: - child(root, name, value) - plugin = ET.SubElement(ET.SubElement(ET.SubElement(root, "build"), "plugins"), "plugin") - child(plugin, "groupId", "org.apache.maven.plugins") - child(plugin, "artifactId", "maven-deploy-plugin") - child(plugin, "version", DEPLOY_PLUGIN) - executions = ET.SubElement(plugin, "executions") - central = Path(package).resolve() / "central" - for i, pom in enumerate(sorted(central.rglob("*.pom"))): - parts = pom.relative_to(central).parts - group, artifact, version = ".".join(parts[:-3]), parts[-3], parts[-2] - xml = ET.parse(pom).getroot() - packaging = xml.findtext("{*}packaging", "jar") - extension = "jar" if packaging == "maven-plugin" else packaging - base = f"{artifact}-{version}" - # A POM-only component has a single unclassified signature. Deploy-file - # treats an empty classifiers parameter as absent, so use that signature - # as the primary artifact and let pomFile attach its signed POM alongside. - transfer_packaging = "pom.asc" if packaging == "pom" else packaging - main = pom.parent / (f"{base}.pom.asc" if packaging == "pom" else f"{base}.{extension}") - if not main.is_file(): - raise ValueError("missing primary Central artifact: " + str(main)) - execution = ET.SubElement(executions, "execution") - child(execution, "id", f"transfer-{i}") - child(execution, "phase", "validate") - child(ET.SubElement(execution, "goals"), "goal", "deploy-file") - config = ET.SubElement(execution, "configuration") - for name, value in [("repositoryId", "central"), ("url", url), ("file", main), - ("pomFile", pom), ("groupId", group), ("artifactId", artifact), - ("version", version), ("packaging", transfer_packaging), ("generatePom", "false")]: - child(config, name, value) - attached = [] - for path in sorted(pom.parent.iterdir()): - if path in (pom, main): - continue - suffix = path.name[len(base):] if path.name.startswith(base) else "" - match = re.fullmatch(r"(?:-(.+))?\.(pom|jar|aar|json)(\.asc)?", suffix) - if not match: - raise ValueError("invalid attached artifact: " + path.name) - classifier = match[1] or "" - kind = match[2] + (match[3] or "") - if "," in str(path): - raise ValueError("comma in Maven transfer path") - attached.append((str(path), classifier, kind)) - if attached: - for name, column in [("files", 0), ("classifiers", 1), ("types", 2)]: - child(config, name, ",".join(item[column] for item in attached)) - ET.ElementTree(root).write(destination, encoding="utf-8", xml_declaration=True) - - -def superseded(repository, sha, run_id, github=None): - """True when one of the 30 most recent dispatched runs transferred the snapshot of a newer commit. - - The central-snapshot jobs of all runs share one concurrency group, so a newer run's transfer - either finished before this check or starts after this transfer: Maven Central serves the - snapshot transferred last as the latest one. Snapshots are published by dispatch only, so the - window of 30 runs reaches back far beyond any transfer still waiting in that group. - """ - gh = github or GitHub() - repo = f"repos/{repository}" - workflow = gh.api("GET", f"{repo}/actions/runs/{run_id}")["workflow_id"] - runs = gh.api("GET", f"{repo}/actions/workflows/{workflow}/runs?event=workflow_dispatch&per_page=30") - for run in runs["workflow_runs"]: - if run["id"] == int(run_id) or run["head_sha"] == sha: - continue - jobs, page = [], 1 - while True: - batch = gh.api("GET", f"{repo}/actions/runs/{run['id']}/jobs?filter=latest&per_page=100&page={page}") - jobs.extend(batch["jobs"]) - if len(batch["jobs"]) < 100: - break - page += 1 - if not any(job["name"] == CENTRAL_SNAPSHOT_JOB and job["conclusion"] == "success" for job in jobs): - continue - if gh.api("GET", f"{repo}/compare/{sha}...{run['head_sha']}")["status"] == "ahead": - print(f"Run {run['id']} already transferred the snapshot of newer commit {run['head_sha']}.") - return True - return False - - -def upload(package, *, skip_superseded=False, github=None): - """Transfer frozen files, accepting Portal upload without validation polling.""" - package = Path(package).resolve() - manifest = verify(package) - if manifest["channel"] == "snapshot": - if skip_superseded and superseded(os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_SHA"], - os.environ["GITHUB_RUN_ID"], github): - print("Skipping the transfer of this older snapshot.") - return - with tempfile.TemporaryDirectory(prefix="snapshot-transfer-") as tmp: - pom = Path(tmp) / "pom.xml" - snapshot_pom(package, pom) - # credentialScope=id: Maven 3.10 sends a server's credentials only to its declared - # origins, which for the id `central` is repo.maven.apache.org, so the upload to - # central.sonatype.com would go out anonymously (HTTP 401). This is the only upload by - # Maven; the staging deploys run with -DskipUpload=true and need no credentials. - subprocess.run(["mvn", "-B", "--no-transfer-progress", "-Dmaven.repository.credentialScope=id", - "-f", str(pom), "validate"], check=True) - return - with tempfile.TemporaryDirectory(prefix="central-transfer-") as tmp: - bundle = Path(tmp) / "central-bundle.zip" - with zipfile.ZipFile(bundle, "w", zipfile.ZIP_DEFLATED) as archive: - for relative in manifest["files"]: - if not relative.startswith("central/"): - continue - path = package / relative - name = relative[len("central/"):] - hashes = {a: hashlib.new(a) for a in ("md5", "sha1", "sha256", "sha512")} - with path.open("rb") as source, archive.open(name, "w", force_zip64=True) as target: - for block in iter(lambda: source.read(1024 * 1024), b""): - target.write(block) - for h in hashes.values(): - h.update(block) - if not name.endswith(".asc"): - for algorithm, h in hashes.items(): - archive.writestr(name + "." + algorithm, h.hexdigest()) - token = base64.b64encode((os.environ["MAVEN_USERNAME"] + ":" + - os.environ["MAVEN_PASSWORD"]).encode()).decode() - config = Path(tmp) / "curl.conf" - config.touch(mode=0o600) - config.write_text('header = "Authorization: Bearer ' + token + '"\n') - response = subprocess.run(["curl", "--fail-with-body", "--silent", "--show-error", - "--connect-timeout", "30", "--max-time", "1800", "--config", str(config), - "-F", "bundle=@" + str(bundle), PORTAL + "?publishingType=AUTOMATIC"], - text=True, capture_output=True) - if response.returncode: - raise RuntimeError("Central upload failed: " + response.stderr + response.stdout) - deployment = response.stdout.strip().strip('"') - if not re.fullmatch(r"[A-Za-z0-9_-]+", deployment): - raise RuntimeError("Central returned no valid deployment receipt") - print("Central accepted deployment: " + deployment) - print("Validation and publication continue asynchronously at Central; no polling.") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - check_parser = commands.add_parser("check-ref", help="check the project version against GITHUB_REF") - check_parser.add_argument("--channel", choices=("snapshot", "release"), required=True) - check_parser.add_argument("--config", type=Path, help="JSON contract (default: .github/release-assets.json)") - prepare_parser = commands.add_parser("prepare") - prepare_parser.add_argument("assets", type=Path) - prepare_parser.add_argument("package", type=Path) - prepare_parser.add_argument("--channel", choices=("snapshot", "release")) - prepare_parser.add_argument("--config", type=Path, help="JSON contract (default: .github/release-assets.json)") - verify_parser = commands.add_parser("verify") - verify_parser.add_argument("package", type=Path) - verify_parser.add_argument("--export-github", type=Path) - upload_parser = commands.add_parser("upload") - upload_parser.add_argument("package", type=Path) - upload_parser.add_argument("--skip-superseded", action="store_true", - help="skip a snapshot when another run transferred a newer commit") - args = parser.parse_args() - root = Path(__file__).resolve().parents[2] - if args.command == "check-ref": - config = configuration(root, args.config) - version = projects(root, config.get("maven", {}).get("standalone", []))[0][2] - check_ref(version, os.environ["GITHUB_REF"], args.channel) - print(f"Version {version} publishes as {args.channel} from {os.environ['GITHUB_REF']}") - elif args.command == "prepare": - prepare(root, args.assets, args.package, config_path=args.config, channel=args.channel) - elif args.command == "verify": - verify(args.package) - if args.export_github: - if args.export_github.exists(): - raise ValueError("GitHub export destination must not exist") - shutil.copytree(args.package / "github", args.export_github) - else: - upload(args.package, skip_superseded=args.skip_superseded) diff --git a/.github/buildcheck/releaseassets.py b/.github/buildcheck/releaseassets.py deleted file mode 100644 index 19c972d..0000000 --- a/.github/buildcheck/releaseassets.py +++ /dev/null @@ -1,209 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# -# SPDX-License-Identifier: MIT OR Apache-2.0 -"""Collect signed Maven release assets and explicitly configured extra files. - -No repository names or native/Android layouts live here. See the workspace policy -policies/release-asset-scripts.md for the JSON contract and staging options. -""" - -import csv -import json -import re -import shutil -import xml.etree.ElementTree as ET -from pathlib import Path - - -def _keys(value, allowed, context): - if not isinstance(value, dict) or set(value) - set(allowed): - raise ValueError(f"invalid {context}; allowed keys: {', '.join(allowed)}") - - -def _inside(root, relative): - if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): - raise ValueError(f"expected a repository-relative path: {relative!r}") - path = (root / relative).resolve() - if not path.is_relative_to(root): - raise ValueError(f"path escapes repository: {relative}") - return path - - -def configuration(root, path=None): - """Read the explicit contract; unknown keys fail instead of silently dropping assets. - - The contract is part of the repository: a relative --config is relative to the repository root, - and a path outside the repository is refused. - """ - root = Path(root).resolve() - path = (root / (path if path is not None else ".github/release-assets.json")).resolve() - if not path.is_relative_to(root): - raise ValueError(f"release asset configuration outside the repository: {path}") - config = json.loads(path.read_text()) - _keys(config, ("schema", "maven", "files", "publication"), "release asset configuration") - if type(config.get("schema")) is not int or config["schema"] != 1: - raise ValueError("release asset configuration requires schema: 1") - publication = config.get("publication", {}) - _keys(publication, ("central_roots",), "publication configuration") - if "central_roots" in publication and (not isinstance(publication["central_roots"], list) - or not publication["central_roots"] - or any(not isinstance(path, str) or not path for path in publication["central_roots"])): - raise ValueError("publication.central_roots must be a nonempty list of paths") - maven = config.get("maven", {}) - _keys(maven, ("standalone", "cyclonedx"), "maven configuration") - if not isinstance(maven.get("standalone", []), list): - raise ValueError("maven.standalone must be a list of module paths") - if type(maven.get("cyclonedx", False)) is not bool: - raise ValueError("maven.cyclonedx must be a boolean") - if not isinstance(config.get("files", []), list): - raise ValueError("files must be a list") - for rule in config.get("files", []): - if isinstance(rule, dict) and "central" in rule and type(rule["central"]) is not bool: - raise ValueError("file rule central must be a boolean") - return config - - -def projects(root, standalone=()): - """Discover the reactor recursively, then optional separately published Maven projects.""" - root = Path(root).resolve() - seen, result = set(), [] - - def visit(folder, reactor=True): - folder = folder.resolve() - if not folder.is_relative_to(root) or folder in seen: - raise ValueError(f"invalid or repeated Maven module: {folder}") - seen.add(folder) - pom = ET.parse(folder / "pom.xml").getroot() - artifact = pom.findtext("{*}artifactId") - version = pom.findtext("{*}version") or pom.findtext("{*}parent/{*}version") - packaging = pom.findtext("{*}packaging", "jar") - if (not artifact or not version or not re.fullmatch(r"[\w.-]+", artifact) - or not re.fullmatch(r"[\w.+-]+", version) - or packaging not in ("pom", "jar", "maven-plugin")): - raise ValueError(f"unsupported release coordinates in {folder / 'pom.xml'}") - if result and version != result[0][2]: - raise ValueError(f"release version mismatch in {folder / 'pom.xml'}: {version}") - result.append((folder, artifact, version, packaging, reactor)) - for module in pom.findall("{*}modules/{*}module"): - visit(_inside(root, str(folder.relative_to(root) / (module.text or "").strip())), reactor) - - visit(root) - for module in standalone: - visit(_inside(root, module), reactor=False) - return result - - -def _classifiers(root, spec): - if isinstance(spec, dict): - _keys(spec, ("csv", "column"), "classifier CSV") - if not isinstance(spec.get("column"), str) or not spec["column"]: - raise ValueError("classifier CSV requires a column") - with _inside(root, spec.get("csv")).open() as stream: - rows = csv.DictReader(line for line in stream if line.strip() and not line.lstrip().startswith("#")) - if spec["column"] not in (rows.fieldnames or []): - raise ValueError(f"missing classifier CSV column: {spec['column']}") - spec = [row[spec["column"]] for row in rows] - if (not isinstance(spec, list) or not spec or any(not isinstance(c, str) - or not re.fullmatch(r"[\w.-]*", c) for c in spec) or len(set(spec)) != len(spec)): - raise ValueError("classifiers must be a nonempty list of unique classifier names") - return spec - - -def _format(template, version, classifier): - if not isinstance(template, str) or not template: - raise ValueError("file source/name must be a nonempty string") - try: - return template.format(version=version, classifier=classifier) - except (KeyError, ValueError, IndexError, AttributeError) as error: - raise ValueError(f"invalid release filename template: {template}") from error - - -def file_classifiers(root, index, config_path=None): - """Let packaging loops consume the same classifier list the collector checks.""" - if index < 0: - raise ValueError("file rule index must not be negative") - config = configuration(root, config_path) - try: - item = config.get("files", [])[index] - except IndexError as error: - raise ValueError(f"no file rule at index {index}") from error - _keys(item, ("source", "name", "classifiers", "central"), "file rule") - return _classifiers(Path(root).resolve(), item.get("classifiers", [""])) - - -def collect(root, destination, config_path=None, *, maven_only=False, maven_assets=None): - """Copy available outputs, returning all missing files/signatures. Never hide partial output. - - maven_only stages deploy-signed files before later packaging overwrites target/. - maven_assets reads those staged files by their final names during the final collection. - """ - root, destination = Path(root).resolve(), Path(destination).resolve() - config = configuration(root, config_path) - modules = projects(root, config.get("maven", {}).get("standalone", [])) - version, expected, errors = modules[0][2], [], [] - for folder, artifact, _, packaging, reactor in modules: - base = f"{artifact}-{version}" - outputs = [(folder / "target" / f"{base}.pom", f"{base}.pom")] - if packaging != "pom": - for classifier in ("", "-sources", "-javadoc"): - name = f"{base}{classifier}.jar" - outputs.append((folder / "target" / name, name)) - if reactor and config.get("maven", {}).get("cyclonedx", False): - outputs.append((folder / "target/bom.json", f"{base}-cyclonedx.json")) - for source, name in outputs: - expected.append((Path(maven_assets) / name if maven_assets is not None else source, name)) - - for item in [] if maven_only else config.get("files", []): - _keys(item, ("source", "name", "classifiers", "central"), "file rule") - for classifier in _classifiers(root, item.get("classifiers", [""])): - pattern = _format(item.get("source"), version, classifier) - _inside(root, pattern) - matches = sorted(root.glob(pattern)) - if len(matches) != 1: - errors.append(f"expected one release output matching {pattern}, found {len(matches)}") - continue - source = matches[0] - _inside(root, str(source.relative_to(root))) - name = _format(item["name"], version, classifier) if "name" in item else source.name - expected.append((source, name)) - - names = set() - for source, name in expected: - if (Path(name).name != name or name in (".", "..") or name.endswith(".asc") - or name in names or name + ".asc" in names): - raise ValueError(f"invalid or duplicate release asset filename: {name}") - names.update((name, name + ".asc")) - if destination == source.parent.resolve(): - raise ValueError("asset destination overlaps release inputs") - destination.mkdir(parents=True, exist_ok=True) - if any(destination.iterdir()): - raise ValueError(f"asset destination must be empty: {destination}") - for source, name in expected: - for suffix in ("", ".asc"): - path = Path(str(source) + suffix).resolve() - if not path.is_file() or path.stat().st_size == 0: - shown = path.relative_to(root) if path.is_relative_to(root) else path - errors.append(f"missing or empty release output: {shown.as_posix()}") - continue - shutil.copyfile(path, destination / (name + suffix)) - return errors - - -def signature_gaps(directory): - """Presence/size check for every file type; signing itself verifies cryptographic validity.""" - directory = Path(directory) - files = sorted(directory.iterdir()) if directory.is_dir() else [] - artifacts = [path for path in files if not path.name.endswith(".asc")] - if not artifacts: - return -1, [f"no artifacts in {directory} -- the collection step produced nothing"] - errors = [] - for path in artifacts: - signature = Path(str(path) + ".asc") - if not path.is_file() or path.stat().st_size == 0: - errors.append(f"missing or empty artifact: {path.name}") - elif not signature.is_file() or signature.stat().st_size == 0: - errors.append(f"unsigned: {path.name} has no nonempty detached .asc") - for path in files: - if path.name.endswith(".asc") and not path.with_suffix("").is_file(): - errors.append(f"orphan signature: {path.name}") - return len(errors), errors diff --git a/.github/buildcheck/releasegate.py b/.github/buildcheck/releasegate.py deleted file mode 100644 index 729e215..0000000 --- a/.github/buildcheck/releasegate.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# -# SPDX-License-Identifier: MIT OR Apache-2.0 -"""Every job of publish.yml gates both publish jobs, unless the repository lists it in -.github/release-gate-exemptions.txt with a reason. - -A job that nothing waits for can go red and a release still ships -- the natives-build jobs that -`package` once forgot to wait for, and the aarch64 fat jars that were signed and attached for -releases without any job launching them, were both of that shape. So "not gating" has to be a -decision written down here, not the default a new job gets by being forgotten in two `needs:` lists. - -The check runs both ways: a job outside the gates and outside the exemptions fails, and so does an -exemption that names no job or a job that gates both publish jobs after all (a stale -exemption would hide the next job of that name). -""" - -from . import workflow - -GATES = ("publish-snapshot", "publish-release") - -# Each repository lists its own exemptions here, one `: ` per line (# comments). -EXEMPTIONS_FILE = ".github/release-gate-exemptions.txt" - - -def read_exemptions(text): - """The exemptions file: job -> reason. A line without a reason is an error -- the point of the - file is that every job allowed to stay red says why.""" - exemptions = {} - for number, line in enumerate(text.splitlines(), 1): - line = line.strip() - if not line or line.startswith("#"): - continue - job, _, reason = line.partition(":") - if not reason.strip() or not job.strip(): - raise ValueError(f"line {number}: expected `: `, got {line!r}") - if job.strip() in exemptions: - raise ValueError(f"line {number}: {job.strip()} is listed twice") - exemptions[job.strip()] = reason.strip() - return exemptions - - -def check(jobs, non_gating, gates=GATES): - missing = [g for g in gates if g not in jobs] - if missing: - return [f"publish.yml has no job {g}" for g in missing] - closures = [workflow.closure(jobs, g) for g in gates] - failures = [] - for name in jobs: - gated = [g for g, c in zip(gates, closures) if name in c] - if name in non_gating: - if len(gated) == len(gates): - failures.append(f"release-gate-exemptions.txt lists {name}, which gates {', '.join(gates)} " - f"-- remove the stale exemption") - elif len(gated) != len(gates): - ungated = [g for g in gates if g not in gated] - failures.append(f"publish.yml: {', '.join(ungated)} does not wait for {name} -- add it to the " - f"needs, or to release-gate-exemptions.txt with the reason it may stay red") - failures += [f"release-gate-exemptions.txt lists {name}, which is no job of publish.yml" - for name in sorted(set(non_gating) - set(jobs))] - return failures diff --git a/.github/buildcheck/runscripts.py b/.github/buildcheck/runscripts.py deleted file mode 100644 index ea68505..0000000 --- a/.github/buildcheck/runscripts.py +++ /dev/null @@ -1,273 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# -# SPDX-License-Identifier: MIT OR Apache-2.0 -"""Every `run:` script of the workflows and composite actions that runs in bash, parsed with -`bash -n` -- so a broken script fails the `shared-files` job in the first minutes of a run instead -of the job that executes it, possibly hours later or only on a release path. - -The case this exists for: a cleanup lost the trailing backslash of six continued lines, leaving a -line that starts with `||` or `-e` -- in a step only a publish run or a native build reaches. Neither -actionlint (it parses scripts only when shellcheck is installed) nor a review saw it; `bash -n` sees -every one of them. - -Which scripts are bash is decided the way the runner decides it: the step's `shell:`, else the job's -and then the workflow's `defaults.run.shell`, else the runner's default -- PowerShell on a Windows -runner, bash everywhere else. A runner chosen by an expression (a matrix, a workflow input) counts -as Windows only when the step's `if:` says so; a step without `shell:` on such a job runs on every -OS of the matrix and has to be valid bash anyway. Scripts in `sh`, `bash -el {0}` or a bash given by -path are bash; pwsh, powershell, cmd and python are skipped. - -Like workflow.py this reads the two-space layout the workflows are written in, not general YAML. -The run value is taken as YAML defines it: a literal block (`|`) line for line, a folded block -(`>`) and a plain scalar folded into one line, a single-quoted scalar unquoted. A double-quoted -scalar is skipped (its escapes are not worth a YAML parser). Every `${{ ... }}` expression becomes -a word, which is what the runner substitutes before bash sees the script. -""" - -import glob -import os -import re -import shutil -import subprocess -import sys - -RUN = re.compile(r"^(?P\s*(?:- )?)run:(?:\s+(?P.*?))?\s*$") -BLOCK_HEADER = re.compile(r"^(?P