From 0ee6db00178cda1d9d9526fffca096ce1813ac90 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 4 Nov 2026 14:23:37 +0100 Subject: [PATCH] feat(tools): share package definitions across feature setup and Bazel --- .github/renovate.json5 | 24 -- .pre-commit-config.yaml | 4 +- MODULE.bazel | 10 + REUSE.toml | 1 + .../.devcontainer/bazel-feature/install.sh | 2 +- .../bazel-feature/tests/test_default.sh | 11 +- .../.devcontainer/bazel-feature/versions.yaml | 25 -- .../.devcontainer/s-core-local/install.sh | 16 +- .../s-core-local/tests/test_default.sh | 47 +-- .../.devcontainer/s-core-local/versions.yaml | 61 ---- tools/BUILD.bazel | 22 +- tools/README.md | 2 + tools/catalog_repository.bzl | 48 +++ tools/internal/README.md | 60 ++-- tools/internal/catalog.py | 139 ++++---- tools/internal/devcontainer/install.py | 54 ++- .../devcontainer/load_feature_versions.sh | 34 -- tools/lockfiles/python_tools.bzl | 30 -- tools/providers.json | 328 ++++++++++++++++++ tools/python_tool.bzl | 13 +- tools/tests/catalog_test.py | 76 ++-- tools/tests/metadata_test.py | 102 ++++++ tools/tests/python_tool_runner_test.sh | 18 + 23 files changed, 756 insertions(+), 371 deletions(-) delete mode 100644 src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml delete mode 100644 src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml create mode 100644 tools/catalog_repository.bzl delete mode 100755 tools/internal/devcontainer/load_feature_versions.sh delete mode 100644 tools/lockfiles/python_tools.bzl create mode 100644 tools/providers.json create mode 100644 tools/tests/metadata_test.py diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a1e81c8..2ea8bc5 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -81,30 +81,6 @@ "datasourceTemplate": "github-releases", "depNameTemplate": "astral-sh/uv", "versioningTemplate": "semver" - }, - { - "customType": "regex", - "fileMatch": [ - "versions\\.yaml$" - ], - "matchStrings": [ - "apm:\\n\\s+version:\\s+(?\\d+\\.\\d+\\.\\d+)" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "microsoft/apm", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "fileMatch": [ - "versions\\.yaml$" - ], - "matchStrings": [ - "opencode:\\n\\s+version:\\s+(?\\d+\\.\\d+\\.\\d+)" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "anomalyco/opencode", - "versioningTemplate": "semver" } ], "enabledManagers": [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c56e080..27fad98 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,8 +72,8 @@ repos: entry: python3 tools/internal/sync_readme.py language: system pass_filenames: false - # Watch the shared reader as well as both catalog formats. - files: ^tools/(README\.md|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/(.*\.lock\.json|python_tools\.bzl))$ + # Watch the shared reader, package definitions and native lockfiles. + files: ^tools/(README\.md|providers\.json|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/.*\.lock\.json)$ - repo: https://github.com/eclipse-score/tooling rev: 31ff8eee214e4e97ef8f5cb46e443273515b63ec hooks: diff --git a/MODULE.bazel b/MODULE.bazel index e11b42e..a93f9cc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,16 @@ bazel_dep(name = "rules_multitool", version = "1.11.1") # Bazel versions. bazel_dep(name = "rules_shell", version = "0.4.1") +# Python packages live beside the other provider definitions in JSON. Convert +# that data to a loadable Starlark value in Bazel's cache, without a second +# maintained package list or a host Python dependency. +catalog_repository = use_repo_rule("//tools:catalog_repository.bzl", "catalog_repository") + +catalog_repository( + name = "score_tool_catalog", + definitions = "//tools:providers.json", +) + multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool") multitool.hub(lockfile = "//tools:lockfiles/actionlint.lock.json") multitool.hub(lockfile = "//tools:lockfiles/apm.lock.json") diff --git a/REUSE.toml b/REUSE.toml index f4012b0..3e3a75b 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -27,6 +27,7 @@ SPDX-License-Identifier = "Apache-2.0" path = ["resources/reopen_in_container.png", "resources/devcontainer_success.png", "tools/lockfiles/*.lock.json", + "tools/providers.json", "MODULE.bazel.lock", ".bazelversion", ] diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh index ea34c32..3954cc7 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh @@ -29,7 +29,7 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe DEBIAN_FRONTEND=noninteractive # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/bazel/versions.yaml +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" ARCHITECTURE=$(dpkg --print-architecture) diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh index c6447ab..d971b77 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh @@ -16,20 +16,17 @@ set -euo pipefail # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/bazel/versions.yaml +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -bazelisk_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version bazelisk)" -buildifier_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version buildifier)" -starpls_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version starpls)" # Bazel-related tools ## This is the bazel version preinstalled in the devcontainer. ## A solid test would disable the network interface first to prevent a different version from being downloaded, ## but that requires CAP_NET_ADMIN, which is not yet added. export USE_BAZEL_VERSION=${bazel_version} -check "validate bazelisk is working and has the correct version" bash -c "bazelisk version | grep '${bazelisk_lockfile_version}'" +check "validate bazelisk is working and has the correct version" bash -c "bazelisk version | grep '${bazelisk_version}'" check "validate bazel is working and has the correct version" bash -c "bazel version | grep '${bazel_version}'" unset USE_BAZEL_VERSION -check "validate buildifier is working and has the correct version" bash -c "buildifier --version | grep '${buildifier_lockfile_version}'" -check "validate starpls is working and has the correct version" bash -c "starpls version | grep '${starpls_lockfile_version}'" +check "validate buildifier is working and has the correct version" bash -c "buildifier --version | grep '${buildifier_version}'" +check "validate starpls is working and has the correct version" bash -c "starpls version | grep '${starpls_version}'" check "validate bazel-compile-commands is working and has the correct version" bash -c "bazel-compile-commands --version 2>&1 | grep '${bazel_compile_commands_version}'" diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml b/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml deleted file mode 100644 index c72910e..0000000 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -bazel: - version: 8.6.0 - # no need to define sha256 here, as bazel is installed via bazelisk -bazel_compile_commands: - version: 0.18.0 - amd64: - # The following sha256sums are for the deb package bazel-compile-commands_-noble_amd64.deb - # It is generated by running 'sha256sum bazel-compile-commands_-noble_amd64.deb' - sha256: 6735ea846241497094719792ad3d4f67b1d3123048693d34fcdf7190f8c2da4e - arm64: - # The following sha256sums are for the deb package bazel-compile-commands_-noble_arm64.deb - # It is generated by running 'sha256sum bazel-compile-commands_-noble_arm64.deb' - sha256: d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88 diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh index 8e3ded8..aabdc1b 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh @@ -31,10 +31,9 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe DEBIAN_FRONTEND=noninteractive # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/s-core-local/versions.yaml +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" ARCHITECTURE=$(dpkg --print-architecture) -KERNEL=$(uname -s) apt-get update @@ -76,7 +75,7 @@ apt-get install -y gh # OpenJDK 21, via APT # Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) -apt-get install -y ca-certificates-java openjdk-21-jdk-headless="${openjdk_21_version}*" +apt-get install -y ca-certificates-java openjdk-21-jdk-headless="${openjdk_version}*" JAVA_HOME="$(dirname $(dirname $(realpath $(command -v javac))))" export JAVA_HOME echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh @@ -101,7 +100,7 @@ apt-get install -y pkg-config="${pkg_config_version}*" libcairo2-dev="${libcairo # additional developer tools apt-get install -y gdb="${gdb_version}*" -apt-get install -y valgrind="1:${valgrind_version}*" +apt-get install -y valgrind="${valgrind_version}*" apt-get install -y ripgrep="${ripgrep_version}*" @@ -114,13 +113,8 @@ if [ "${ARCHITECTURE}" = "amd64" ]; then VARIANT=linux64 SHA256SUM="${codeql_amd64_sha256}" elif [ "${ARCHITECTURE}" = "arm64" ]; then - if [ "${KERNEL}" = "Darwin" ]; then - VARIANT=osx64 - SHA256SUM="${codeql_arm64_sha256}" - else - echo "CodeQl unsupported architecture/os: ${ARCHITECTURE} on ${KERNEL}, skipping installation" - VARIANT=noinstall - fi + echo "CodeQL has no selected Linux ARM64 bundle; skipping installation" + VARIANT=noinstall else echo "Unsupported architecture: ${ARCHITECTURE} for CodeQL" exit 1 diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh index f83eb4a..ab665b7 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh @@ -16,30 +16,17 @@ set -euo pipefail ARCHITECTURE=$(dpkg --print-architecture) -KERNEL=$(uname -s) # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/s-core-local/versions.yaml +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -shellcheck_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version shellcheck)" -ruff_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version ruff)" -actionlint_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version actionlint)" -yamlfmt_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version yamlfmt)" -uv_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version uv)" -uvx_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version uvx)" -apm_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version apm)" -opencode_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version opencode)" - -# The shared catalog is the assertion source, so installation metadata and the -# executable exposed on PATH are checked against the same release. -pre_commit_catalog_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version pre-commit)" # Verify the system-wide uv installation exposes the catalogued release to the # non-root user running feature tests. -check "validate pre-commit is working and has the correct version" bash -c "pre-commit --version | grep '${pre_commit_catalog_version}'" +check "validate pre-commit is working and has the correct version" bash -c "pre-commit --version | grep '${pre_commit_version}'" # Common tooling -check "validate shellcheck is working and has the correct version" bash -c "shellcheck --version | grep '${shellcheck_lockfile_version}'" +check "validate shellcheck is working and has the correct version" bash -c "shellcheck --version | grep '${shellcheck_version}'" # For an unknown reason, dot -V reports on Ubuntu Noble a version 2.43.0, while the package has a different version. # Hence, we have to work around that. @@ -66,27 +53,27 @@ check "validate pytest is working" bash -c "pytest --version" check "validate pylint is working" bash -c "pylint --version" # OpenJDK -check "validate java is working and has the correct version" bash -c "java -version 2>&1 | grep '${openjdk_21_version}'" +check "validate java is working and has the correct version" bash -c "java -version 2>&1 | grep '${openjdk_version}'" check "validate JAVA_HOME is set correctly" bash -c "echo ${JAVA_HOME} | xargs readlink -f | grep \"java-21-openjdk\"" # ruff -check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_lockfile_version}'" +check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_version}'" # actionlint -check "validate actionlint is working and has the correct version" bash -c "actionlint --version | grep '${actionlint_lockfile_version}'" +check "validate actionlint is working and has the correct version" bash -c "actionlint --version | grep '${actionlint_version}'" # yamlfmt -check "validate yamlfmt is working and has the correct version" bash -c "yamlfmt --version | grep '${yamlfmt_lockfile_version}'" +check "validate yamlfmt is working and has the correct version" bash -c "yamlfmt --version | grep '${yamlfmt_version}'" # apm -check "validate apm is working and has the correct version" bash -c "apm --version | grep '${apm_lockfile_version}'" +check "validate apm is working and has the correct version" bash -c "apm --version | grep '${apm_version}'" # opencode -check "validate opencode is working and has the correct version" bash -c "opencode --version | grep '${opencode_lockfile_version}'" +check "validate opencode is working and has the correct version" bash -c "opencode --version | grep '${opencode_version}'" # uv -check "validate uv is working and has the correct version" bash -c "uv --version | grep '${uv_lockfile_version}'" -check "validate uvx is working and has the correct version" bash -c "uvx --version | grep '${uvx_lockfile_version}'" +check "validate uv is working and has the correct version" bash -c "uv --version | grep '${uv_version}'" +check "validate uvx is working and has the correct version" bash -c "uvx --version | grep '${uvx_version}'" # pycairo build from source (requires pkg-config and libcairo2-dev) check "validate pycairo can be built and installed from source" bash -c "pip install --break-system-packages pycairo==1.29.0" @@ -95,9 +82,11 @@ check "validate pycairo can be built and installed from source" bash -c "pip ins check "validate gdb is working and has the correct version" bash -c "gdb --version | grep '${gdb_version}'" check "validate pkg-config is working and has the correct version" bash -c "pkg-config --version | grep '${pkg_config_version}'" check "validate gh is working and has the correct version" bash -c "gh --version | grep '${gh_version}'" -check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version}'" +# Debian epochs (e.g. 1:3.22.0) are APT metadata, not executable versions. +# ${value#*:} removes that prefix for this comparison. +check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version#*:}'" check "validate ripgrep is working and has the correct version" bash -c "rg --version | grep '${ripgrep_version}'" -if [ "${ARCHITECTURE}" = "amd64" ] || { [ "${ARCHITECTURE}" = "arm64" ] && [ "${KERNEL}" = "Darwin" ]; }; then +if [ "${ARCHITECTURE}" = "amd64" ]; then check "validate codeql is working and has the correct version" bash -c "codeql --version | grep '${codeql_version}'" check "validate CODEQL_HOME is set correctly" bash -c "echo ${CODEQL_HOME} | grep \"/usr/local/codeql\"" fi @@ -105,7 +94,7 @@ fi # lcov check "validate lcov is working and has the correct version" bash -c "lcov --version | grep '${lcov_version}'" -# Qemu target-related tools -check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version}'" -check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version}'" +# Qemu target-related tools (strip the Debian epoch here too) +check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version#*:}'" +check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version#*:}'" check "validate sshpass is working and has the correct version" bash -c "sshpass -V | grep '${sshpass_version}'" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml b/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml deleted file mode 100644 index 279bad0..0000000 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -basedpyright: - version: 1.35.0 -graphviz: - version: 2.42.2 -protobuf_compiler: - version: 3.21.12 -qemu_system: - version: 1:8.2.2 -sshpass: - version: 1.09 -gdb: - version: 15.0.50 -git: - version: 2.43.0 -git_lfs: - version: 3.4.1 -python: - version: '3.12' -pytest: - version: 9.0.2 -gh: - version: 2.45.0 -openjdk_21: - version: 21.0.12 -codeql: - # the coding_standards_version below dictates the codeql version - version: 2.21.4 - amd64: - # see https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-linux64.tar.zst.checksum.txt - sha256: 6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545 - arm64: - # see https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-osx64.tar.zst.checksum.txt - sha256: 810a1fff48aeb081be754e46e255edd90d8695966e78431a65edb00e9e6cc399 -codeql_coding_standards: - version: 2.54.0 -valgrind: - version: 3.22.0 -ripgrep: - version: 14.1.0 -lcov: - version: 2.0 -apm: - version: 0.35.0 -opencode: - version: 2.2.3 -pkg_config: - version: 1.8.1 -libcairo2_dev: - version: 1.18.0 diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 7723840..a3635ed 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -13,17 +13,13 @@ load("@rules_shell//shell:sh_test.bzl", "sh_test") load(":multitool.bzl", "multitool_aliases") -load(":python_tool.bzl", "python_tool") +load(":python_tool.bzl", "python_tools") package(default_visibility = ["//visibility:public"]) -# Native lockfiles are consumed by rules_multitool; the Starlark catalog is -# loaded directly by python_tool.bzl. Export both because they are public -# metadata shipped with the score_devcontainer module. -exports_files(glob([ - "lockfiles/*.bzl", - "lockfiles/*.lock.json", -])) +# Native lockfiles are read by rules_multitool. The shared provider definitions +# also feed Bazel's generated Python package data; all remain module metadata. +exports_files(glob(["lockfiles/*.lock.json"]) + ["providers.json"]) multitool_aliases("actionlint") @@ -49,18 +45,24 @@ multitool_aliases("opencode") # Python distributions are resolved through pinned uvx because they do not # provide the platform-specific release binaries consumed by rules_multitool. -python_tool("pre-commit") +# Export every catalogued Python command. environments.json chooses which of +# these are installed in each container; it does not restrict the module API. +python_tools() # This contract test keeps Bazel's shell launcher and the container installer # aligned with their shared catalog. sh_test( name = "python_tool_runner_test", srcs = ["tests/python_tool_runner_test.sh"], + # Compare the generated analysis-time catalog with the JSON used by the + # installer. rootpath resolves the generated file inside test runfiles. + args = ["$(rootpath @score_tool_catalog//:python_tools.bzl)"], data = [ "internal/catalog.py", "internal/devcontainer/install.py", "internal/run_python_tool.sh", - "lockfiles/python_tools.bzl", + "providers.json", + "@score_tool_catalog//:python_tools.bzl", ], ) diff --git a/tools/README.md b/tools/README.md index 27437a0..973dc0f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,10 +41,12 @@ host-side Bazel invocation may require network access to download the binary. | --- | --- | --- | | `actionlint` | `1.7.7` | Check GitHub Actions workflow files | | `apm` | `0.35.0` | Manage APM packages | +| `basedpyright` | `1.35.0` | Type check Python and provide its language server | | `bazelisk` | `1.27.0` | Run the Bazel version selected by a repository | | `buildifier` | `8.2.1` | Format and lint Bazel files | | `opencode` | `2.2.3` | Run the OpenCode CLI | | `pre-commit` | `4.7.0` | Run repository pre-commit hooks | +| `pytest` | `9.0.2` | Run Python tests | | `ruff` | `0.11.13` | Check and format Python code | | `shellcheck` | `0.10.0` | Check shell scripts | | `starpls` | `0.1.22` | Provide language-server support for Starlark | diff --git a/tools/catalog_repository.bzl b/tools/catalog_repository.bzl new file mode 100644 index 0000000..20e3eba --- /dev/null +++ b/tools/catalog_repository.bzl @@ -0,0 +1,48 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Let Bazel read the same JSON package definitions as the container installer.""" + +def _catalog_repository_impl(repository_ctx): + definitions = json.decode(repository_ctx.read(repository_ctx.attr.definitions)) + if type(definitions) != "dict": + fail("Tool providers must be a dictionary") + tools = {} + for name, definition in definitions.items(): + if type(definition) != "dict": + fail("Invalid provider definition for '{}'".format(name)) + if definition.get("provider") != "python": + continue + tool = {} + for field in ("package", "version", "entrypoint", "description"): + value = definition.get(field) + if type(value) != "string" or not value: + fail("Malformed Python package definition for '{}'".format(name)) + tool[field] = value + dependencies = definition.get("requires", []) + if type(dependencies) != "list" or "uv" not in dependencies: + fail("Python tool '{}' must declare its uv dependency".format(name)) + tools[name] = tool + + # BUILD macros cannot read JSON source files. Produce their Starlark input + # inside Bazel's repository cache, directly from the tracked JSON file. + # repr quotes every string as data; catalog values cannot become code. + repository_ctx.file("python_tools.bzl", "PYTHON_TOOLS = " + repr(tools) + "\n") + repository_ctx.file("BUILD.bazel", 'exports_files(["python_tools.bzl"])\n') + +# The label is an explicit input: changing providers.json refreshes the generated +# data. No Python interpreter, network download or checked-in copy is needed. +catalog_repository = repository_rule( + implementation = _catalog_repository_impl, + attrs = {"definitions": attr.label(mandatory = True, allow_single_file = True)}, +) diff --git a/tools/internal/README.md b/tools/internal/README.md index cfae24e..d511333 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -28,7 +28,8 @@ One catalog is delivered through two execution paths: | Component | Responsibility | | --- | --- | -| `tools/lockfiles/*.lock.json` and `tools/lockfiles/python_tools.bzl` | Define versions and delivery metadata. | +| `tools/lockfiles/*.lock.json` | Define native tool versions and delivery metadata. | +| `tools/providers.json` | Define Python/npm packages, APT policies and special setup metadata. | | `MODULE.bazel` | Makes native tool lockfiles available to `rules_multitool`. | | `tools/BUILD.bazel` | Exposes public Bazel targets for each command. | | Feature installers | Install the commands exposed on the DevContainer's `PATH`. | @@ -87,23 +88,25 @@ supported platforms, download URLs, checksums, and archive layouts. The reader has no installation side effects; download, platform selection, and installation remain in the container installer. -Python command-line tool metadata lives in -[`tools/lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl). The file is -a data-only Starlark dictionary because Bazel must read the pin during analysis, -before the command can run. Its restricted literal form is also valid Python -syntax. During DevContainer setup, `install.py` runs as root so it can install -tools in system locations such as `/usr/local/bin`. The shared reader parses -the catalog with `ast.literal_eval` instead of executing it as Python, so -catalog changes can provide only literal data and cannot run code with the -installer's privileges. -This shared format keeps the package, console entrypoint, version, and -description in one source of truth while Bazel execution remains independent -of host Python. - -The catalogs have distinct ownership: `devcontainer-lock.json` records external -DevContainer features, `python_tools.bzl` records Python package releases, and -`uv.lock.json` records the installer runtime. Each version is therefore owned -by the component that resolves it. +Other package and setup definitions live in +[`providers.json`](../providers.json). Python commands declare their package, +version, console entrypoint and description there, alongside APT policies, npm +packages and feature-specific setup metadata. Native release artifacts keep +their multitool-compatible lockfiles. + +The installer reads JSON directly. Bazel uses +[`catalog_repository.bzl`](../catalog_repository.bzl) to derive loadable Python +package data in its cache, so there is no second maintained Python catalog. +The existing feature installers and tests use `install.py shell-metadata` to +read quoted version/checksum variables without downloading yq. Their package +installation steps remain in the feature scripts. + +APT `distribution` policies follow Ubuntu, while version prefixes retain Debian +epochs such as `1:3.22.0`. Executables omit that prefix when reporting their +version. CodeQL declares Linux x86_64 support and an explicit Linux ARM64 +omission; the former macOS checksum is not a Linux ARM64 artifact. +`devcontainer-lock.json` continues to record external Dev Container features, +and `uv.lock.json` records the native installer runtime. The published `score_devcontainer` Bazel module and DevContainer image share a release version. Consumers using both must pin the same release. @@ -139,18 +142,15 @@ changes together: 7. If this repository's own DevContainer needs the command, add it to [`.devcontainer/post_create_command.sh`](../../.devcontainer/post_create_command.sh). -For a Python command-line tool, add an entry to -`tools/lockfiles/python_tools.bzl`, add `python_tool("")` to -`tools/BUILD.bazel`, and install it through `install.py install-python` in -each applicable DevContainer installer. Keep the catalog as a single literal -`PYTHON_TOOLS` assignment: this is what makes it safe for the installer and -directly loadable by Bazel. The `package` selects the distribution passed to -`uv`, while `entrypoint` names the console command because those names need not -be identical. - -Bazel invokes Python tools on demand with the pinned `uvx`; DevContainers -install the same package with the pinned `uv`. Add launcher and installer -coverage to `python_tool_runner_test` when either delivery contract changes. +For a Python command-line tool, add its definition to `tools/providers.json` +with `provider: "python"`, `package`, `version`, `entrypoint`, `description` and +`requires: ["uv"]`. Bazel targets are created for every Python definition +automatically. Add the command to each applicable DevContainer installer. +The Python package and console command may have different names. + +Bazel executes the pinned package on demand with uvx; DevContainer installation +uses the same definition through uv. Add launcher and installer coverage to +`python_tool_runner_test` when either delivery contract changes. Keep related commands such as `uv` and `uvx` in one lockfile when upstream publishes them together. The installer can locate a command in a differently diff --git a/tools/internal/catalog.py b/tools/internal/catalog.py index 69d07ae..1f01410 100644 --- a/tools/internal/catalog.py +++ b/tools/internal/catalog.py @@ -10,13 +10,12 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Read pinned native and Python tool metadata without executing catalog code.""" +"""Read package definitions and native tool metadata without running installation code.""" # pyright: reportAny=false, reportUnusedCallResult=false, reportExplicitAny=false from __future__ import annotations -import ast import json from collections.abc import Iterator from pathlib import Path @@ -45,7 +44,6 @@ class ToolData(TypedDict): LOCKFILE_ROOT = Path(__file__).resolve().parents[1] / "lockfiles" -PYTHON_TOOL_CATALOG = LOCKFILE_ROOT / "python_tools.bzl" class PythonTool(TypedDict): @@ -73,77 +71,18 @@ def _iter_catalog() -> Iterator[tuple[str, str, ToolData]]: yield tool, path.name, definition -def _parse_python_tool_catalog() -> object: - """Return the literal assigned to ``PYTHON_TOOLS``. - - The feature installer runs as root, so importing or executing a repository - file would cross a privilege boundary. Parsing one assignment and accepting - only literal data lets Bazel and the installer share a catalog safely. - """ - try: - module = ast.parse( - PYTHON_TOOL_CATALOG.read_text(encoding="utf-8"), - filename=str(PYTHON_TOOL_CATALOG), - ) - except SyntaxError as exc: - raise SystemExit(f"Malformed Python tool catalog: {exc}") from exc - - # Exactly one statement preserves the catalog's data-only contract. - if len(module.body) != 1: - raise SystemExit( - "Python tool catalog must contain only a PYTHON_TOOLS assignment" - ) - - assignment = module.body[0] - if not isinstance(assignment, ast.Assign) or len(assignment.targets) != 1: - raise SystemExit( - "Python tool catalog must contain only a PYTHON_TOOLS assignment" - ) - - target = assignment.targets[0] - if not isinstance(target, ast.Name) or target.id != "PYTHON_TOOLS": - raise SystemExit("Python tool catalog must assign its data to PYTHON_TOOLS") - - try: - return ast.literal_eval(assignment.value) - except (TypeError, ValueError) as exc: - raise SystemExit("Python tool catalog must contain only literal data") from exc - - def load_python_tools() -> dict[str, PythonTool]: - """Validate the shared catalog and return typed Python tool metadata.""" - catalog = _parse_python_tool_catalog() - if not isinstance(catalog, dict): - raise SystemExit("PYTHON_TOOLS must be a dictionary") - - tools: dict[str, PythonTool] = {} - for command, metadata in catalog.items(): - if ( - not isinstance(command, str) - or not command - or not isinstance(metadata, dict) - ): - raise SystemExit(f"Malformed Python tool catalog entry for '{command}'") - - package = metadata.get("package") - version = metadata.get("version") - entrypoint = metadata.get("entrypoint") - description = metadata.get("description") - # Central validation gives Bazel, installation, and documentation the - # same required-field contract. - if not all( - isinstance(value, str) and value - for value in (package, version, entrypoint, description) - ): - raise SystemExit(f"Malformed Python tool catalog entry for '{command}'") - - tools[command] = { - "package": package, - "version": version, - "entrypoint": entrypoint, - "description": description, + """Read Python package definitions from the shared provider catalog.""" + return { + name: { + "package": definition["package"], + "version": definition["version"], + "entrypoint": definition["entrypoint"], + "description": definition["description"], } - return tools + for name, definition in load_providers().items() + if definition["provider"] == "python" + } def load_catalog_versions() -> dict[str, str]: @@ -233,3 +172,59 @@ def resolve_lockfile(tool: str, lockfile: str | None = None) -> str: if _lockfile_path(tool).exists(): return tool return _find_lockfile(tool) + + +PROVIDERS_PATH = LOCKFILE_ROOT.parent / "providers.json" + + +def load_providers() -> dict: + """Read and check package definitions and tool-specific setup metadata.""" + providers = json.loads(PROVIDERS_PATH.read_text(encoding="utf-8")) + if not isinstance(providers, dict): + raise SystemExit("Tool providers must be a dictionary") + for name, definition in providers.items(): + if not isinstance(definition, dict) or definition.get("provider") not in ( + "apt", + "npm", + "python", + "release", + "bazelisk", + ): + raise SystemExit(f"Invalid provider definition for '{name}'") + if definition["provider"] == "apt": + if definition.get("policy") not in ( + "distribution", + "prefix", + "package-name", + ): + raise SystemExit(f"Invalid APT policy for '{name}'") + packages = definition.get("packages") + if ( + not isinstance(packages, list) + or not packages + or any( + not isinstance(package, str) + or not package + or package.startswith("-") + for package in packages + ) + ): + raise SystemExit(f"Invalid APT packages for '{name}'") + if definition["policy"] != "distribution" and not definition.get("version"): + raise SystemExit(f"Missing APT version constraint for '{name}'") + elif definition["provider"] == "python": + if not all( + isinstance(definition.get(field), str) and definition[field] + for field in ("package", "version", "entrypoint", "description") + ): + raise SystemExit(f"Malformed Python package definition for '{name}'") + dependencies = definition.get("requires", []) + if not isinstance(dependencies, list) or "uv" not in dependencies: + raise SystemExit(f"Python tool '{name}' must declare its uv dependency") + elif definition["provider"] == "npm": + if ( + not isinstance(definition.get("package"), str) + or not definition["package"] + ): + raise SystemExit(f"Missing npm package for '{name}'") + return providers diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index ee3e2a2..ff7f5b3 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -48,6 +48,7 @@ ToolData, load_catalog_versions, load_python_tools, + load_providers, load_tool, resolve_lockfile, ) @@ -254,12 +255,63 @@ def _cmd_install(args: argparse.Namespace) -> int: return 0 +def _metadata_values() -> dict[str, str]: + """Turn definition fields into variables such as codeql_version for setup and tests.""" + import re + + values: dict[str, str] = {} + + def emit(name, value): + if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name): + raise SystemExit(f"Invalid metadata variable: {name}") + if isinstance(value, dict): + for key, child in value.items(): + emit(name + "_" + key, child) + elif isinstance(value, str): + values[name] = value + + for name, definition in load_providers().items(): + # Use only version and checksum fields. Other data, such as platform + # names containing slashes, cannot be used as shell variable names. + for field in ("version", "compatibility_version", "amd64", "arm64"): + if field in definition: + emit(name.replace("-", "_") + "_" + field, definition[field]) + # Unpinned APT tools can still have expected versions for feature tests. + # Give these expectations the same _version names as pinned tools. + if "compatibility_version" in definition: + emit( + name.replace("-", "_") + "_version", definition["compatibility_version"] + ) + # Native and Python command versions use the same variable names as the + # other definitions, so feature tests need just one metadata query. + for name, version in load_catalog_versions().items(): + emit(name.replace("-", "_") + "_version", version) + # Python package and entrypoint metadata is also available to setup scripts. + for name, definition in load_python_tools().items(): + emit(name.replace("-", "_"), definition) + return values + + +def _cmd_shell_metadata(args: argparse.Namespace) -> int: + """Print shell-quoted exports so feature tests can read versions in their shell.""" + import shlex + + for name, value in _metadata_values().items(): + print("export " + name + "=" + shlex.quote(value)) + return 0 + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Install pinned tools from multitool-compatible lockfiles.", ) subparsers = parser.add_subparsers(dest="command", required=True) + metadata_parser = subparsers.add_parser( + "shell-metadata", help="Print quoted version variables for feature scripts." + ) + metadata_parser.set_defaults(func=_cmd_shell_metadata) + install_parser = subparsers.add_parser( "install", help="Download, verify, and install tools.", @@ -273,7 +325,7 @@ def _build_parser() -> argparse.ArgumentParser: install_python_parser = subparsers.add_parser( "install-python", - help="Install Python command-line tools declared in python_tools.bzl.", + help="Install Python command-line tools declared in providers.json.", ) install_python_parser.add_argument( "tools", diff --git a/tools/internal/devcontainer/load_feature_versions.sh b/tools/internal/devcontainer/load_feature_versions.sh deleted file mode 100755 index bc2ce33..0000000 --- a/tools/internal/devcontainer/load_feature_versions.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -set -euo pipefail - -ARCHITECTURE=$(dpkg --print-architecture) -VERSION="v4.47.2" - -SHA256_FIELD="1bb99e1019e23de33c7e6afc23e93dad72aad6cf2cb03c797f068ea79814ddb0" # Default to amd64 -if [ "${ARCHITECTURE}" = "arm64" ]; then - SHA256_FIELD="05df1f6aed334f223bb3e6a967db259f7185e33650c3b6447625e16fea0ed31f" -fi - -yq_binary="$(mktemp)" -trap 'rm -f "${yq_binary}"' EXIT - -curl --fail --location "https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_linux_${ARCHITECTURE}" -o "${yq_binary}" -echo "${SHA256_FIELD} ${yq_binary}" | sha256sum -c - || exit 1 -chmod +x "${yq_binary}" - -# Read tool versions and metadata into environment variables -export $("${yq_binary}" eval '.. | select((tag == "!!map" or tag == "!!seq") | not) | (path | join("_")) + "=" + .' "$1" | awk '!/=$/{print }' | xargs) diff --git a/tools/lockfiles/python_tools.bzl b/tools/lockfiles/python_tools.bzl deleted file mode 100644 index 262d40b..0000000 --- a/tools/lockfiles/python_tools.bzl +++ /dev/null @@ -1,30 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# A literal Starlark dictionary lets Bazel consume the pins during analysis -# while ast.literal_eval preserves a data-only boundary for the root-running -# DevContainer installer. -PYTHON_TOOLS = { - # The key is the public command and Bazel target name. - "pre-commit": { - # `package` is resolved by uv; `entrypoint` is the executable exposed to - # users. They are separate because Python distributions may name them - # differently. - "package": "pre-commit", - "version": "4.7.0", - "entrypoint": "pre-commit", - # Documentation is generated from the catalog so version and purpose - # are reviewed together. - "description": "Run repository pre-commit hooks", - }, -} diff --git a/tools/providers.json b/tools/providers.json new file mode 100644 index 0000000..a21b73f --- /dev/null +++ b/tools/providers.json @@ -0,0 +1,328 @@ +{ + "graphviz": { + "provider": "apt", + "packages": ["graphviz"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "2.42.2" + }, + "protobuf_compiler": { + "provider": "apt", + "packages": ["protobuf-compiler"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "3.21.12" + }, + "qemu_system": { + "provider": "apt", + "packages": ["qemu-system-arm", "qemu-system-x86"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1:8.2.2", + "install_recommends": false, + "description": "Both ARM and x86 guest emulators are installed on each supported host platform." + }, + "sshpass": { + "provider": "apt", + "packages": ["sshpass"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.09" + }, + "gdb": { + "provider": "apt", + "packages": ["gdb"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "15.0.50" + }, + "git": { + "provider": "apt", + "packages": ["git"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "2.43.0" + }, + "git_lfs": { + "provider": "apt", + "packages": ["git-lfs"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "3.4.1" + }, + "gh": { + "provider": "apt", + "packages": ["gh"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "2.45.0" + }, + "openjdk": { + "provider": "apt", + "packages": ["openjdk-21-jdk-headless"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "21.0.12", + "setup_script": "java.sh" + }, + "valgrind": { + "provider": "apt", + "packages": ["valgrind"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1:3.22.0" + }, + "ripgrep": { + "provider": "apt", + "packages": ["ripgrep"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "14.1.0" + }, + "lcov": { + "provider": "apt", + "packages": ["lcov"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "2.0" + }, + "pkg_config": { + "provider": "apt", + "packages": ["pkg-config"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.8.1" + }, + "libcairo2_dev": { + "provider": "apt", + "packages": ["libcairo2-dev"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.18.0" + }, + "python": { + "provider": "apt", + "description": "Distribution Python interpreter", + "packages": ["python{version}"], + "policy": "package-name", + "version": "3.12", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "flake8": { + "provider": "apt", + "packages": ["flake8"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-autopep8": { + "provider": "apt", + "packages": ["python3-autopep8"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "black": { + "provider": "apt", + "packages": ["black"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "description": "Ubuntu-provided Black; distribution policy does not pin an exact version." + }, + "python3-yapf": { + "provider": "apt", + "packages": ["python3-yapf"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "mypy": { + "provider": "apt", + "packages": ["mypy"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pydocstyle": { + "provider": "apt", + "packages": ["pydocstyle"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pycodestyle": { + "provider": "apt", + "packages": ["pycodestyle"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "bandit": { + "provider": "apt", + "packages": ["bandit"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pipenv": { + "provider": "apt", + "packages": ["pipenv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "virtualenv": { + "provider": "apt", + "packages": ["virtualenv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pylint": { + "provider": "apt", + "packages": ["pylint"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "ca-certificates-java": { + "provider": "apt", + "packages": ["ca-certificates-java"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "description": "Populate the Java trust store from system CA certificates; follow distribution updates." + }, + "man-db": { + "provider": "apt", + "packages": ["man-db"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages": { + "provider": "apt", + "packages": ["manpages"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-dev": { + "provider": "apt", + "packages": ["manpages-dev"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-posix": { + "provider": "apt", + "packages": ["manpages-posix"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-posix-dev": { + "provider": "apt", + "packages": ["manpages-posix-dev"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "codeql": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64"], + "version": "2.21.4", + "amd64": { + "sha256": "6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545" + }, + "omissions": { + "linux/arm64": "The selected bundle has no supported Linux arm64 artifact" + }, + "setup_script": "codeql.sh", + "requires": ["zstd"] + }, + "codeql_coding_standards": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64"], + "version": "2.54.0", + "requires": ["codeql"], + "omissions": { + "linux/arm64": "Not installed without the CodeQL bundle" + }, + "packs": [ + "codeql/misra-cpp-coding-standards", + "codeql/misra-c-coding-standards", + "codeql/cert-cpp-coding-standards", + "codeql/cert-c-coding-standards" + ] + }, + "bazel": { + "provider": "bazelisk", + "policy": "exact", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "8.6.0", + "requires": ["bazelisk"], + "setup_script": "bazel.sh" + }, + "bazel_compile_commands": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "0.18.0", + "amd64": { + "sha256": "6735ea846241497094719792ad3d4f67b1d3123048693d34fcdf7190f8c2da4e" + }, + "arm64": { + "sha256": "d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88" + }, + "setup_script": "bazel_compile_commands.sh" + }, + "apt-transport-https": { + "provider": "apt", + "description": "Temporary container build dependency (removed after setup)", + "packages": ["apt-transport-https"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "phase": "build-only" + }, + "zstd": { + "provider": "apt", + "description": "Temporary container build dependency (removed after setup)", + "packages": ["zstd"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "phase": "build-only" + }, + "devcontainers-cli": { + "provider": "npm", + "description": "Build and test Dev Containers", + "package": "@devcontainers/cli", + "policy": "latest", + "platforms": ["Node.js-supported platforms"], + "phase": "repository setup" + }, + "python3-pip": { + "provider": "apt", + "description": "Distribution pip (independent of the interpreter package-name constraint)", + "packages": ["python3-pip"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-venv": { + "provider": "apt", + "description": "Distribution Python venv support", + "packages": ["python3-venv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pre-commit": { + "provider": "python", + "package": "pre-commit", + "version": "4.7.0", + "entrypoint": "pre-commit", + "description": "Run repository pre-commit hooks", + "requires": ["uv"] + }, + "basedpyright": { + "provider": "python", + "package": "basedpyright", + "version": "1.35.0", + "entrypoint": "basedpyright", + "description": "Type check Python and provide its language server", + "requires": ["uv"] + }, + "pytest": { + "provider": "python", + "package": "pytest", + "version": "9.0.2", + "entrypoint": "pytest", + "description": "Run Python tests", + "requires": ["uv"] + } +} diff --git a/tools/python_tool.bzl b/tools/python_tool.bzl index 0ff4c56..9e0969d 100644 --- a/tools/python_tool.bzl +++ b/tools/python_tool.bzl @@ -18,7 +18,7 @@ shell and the pinned uvx executable supplied by rules_multitool. """ load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load(":lockfiles/python_tools.bzl", "PYTHON_TOOLS") +load("@score_tool_catalog//:python_tools.bzl", "PYTHON_TOOLS") def python_tool(name): """Creates a runnable target from the shared Python tool catalog.""" @@ -26,7 +26,7 @@ def python_tool(name): # Fail during analysis so a misspelled target cannot degrade into an # unpinned uvx invocation at runtime. if name not in PYTHON_TOOLS: - fail("Python tool '{}' is not defined in python_tools.bzl".format(name)) + fail("Python tool '{}' is not defined in providers.json".format(name)) tool = PYTHON_TOOLS[name] @@ -51,3 +51,12 @@ def python_tool(name): # its version and platform selection. data = [":uvx_binary"], ) + +def python_tools(): + """Expose every catalogued Python command as a public Bazel target.""" + + # Environment selections control image contents. The module API includes + # every command in the catalog so removing a tool from one image does not + # remove a public target used by other repositories. + for name in PYTHON_TOOLS: + python_tool(name) diff --git a/tools/tests/catalog_test.py b/tools/tests/catalog_test.py index 6631065..c747ca2 100644 --- a/tools/tests/catalog_test.py +++ b/tools/tests/catalog_test.py @@ -14,7 +14,6 @@ from __future__ import annotations -import ast import json import shutil import subprocess @@ -49,10 +48,17 @@ def test_metadata_matches_definitions(self) -> None: if "description" in metadata: descriptions[name] = metadata["description"] - source = (TOOLS_DIR / "lockfiles/python_tools.bzl").read_text(encoding="utf-8") - python_tools = ast.literal_eval(ast.parse(source).body[0].value) + providers = json.loads((TOOLS_DIR / "providers.json").read_text()) + python_tools = { + name: { + field: definition[field] + for field in ("package", "version", "entrypoint", "description") + } + for name, definition in providers.items() + if definition["provider"] == "python" + } self.assertEqual(catalog.load_python_tools(), python_tools) - self.assertEqual(set(python_tools), {"pre-commit"}) + self.assertEqual(set(python_tools), {"pre-commit", "basedpyright", "pytest"}) versions.update( (name, metadata["version"]) for name, metadata in python_tools.items() ) @@ -64,38 +70,42 @@ def test_metadata_matches_definitions(self) -> None: self.assertEqual(catalog.resolve_lockfile("uvx"), "uv") self.assertEqual(catalog.resolve_lockfile("uvx", "explicit"), "explicit") - def test_rejects_malformed_and_executable_python_catalogs(self) -> None: + def test_rejects_python_packages_with_missing_fields_or_uv_dependency(self) -> None: + valid = { + "provider": "python", + "package": "example", + "version": "1", + "entrypoint": "example", + "description": "Example tool", + "requires": ["uv"], + } with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - path = root / "python_tools.bzl" - marker = root / "executed" - payload = f"__import__('pathlib').Path({str(marker)!r}).touch()" - cases = [ - ("PYTHON_TOOLS = {", "Malformed Python tool catalog"), - ("OTHER = {}", "must assign its data to PYTHON_TOOLS"), - ("PYTHON_TOOLS = []", "PYTHON_TOOLS must be a dictionary"), - ( - "PYTHON_TOOLS = {'bad': {'version': '1'}}", - "Malformed Python tool catalog entry", - ), - (f"PYTHON_TOOLS = {payload}", "must contain only literal data"), - ( - f"PYTHON_TOOLS = {{}}\n{payload}", - "must contain only a PYTHON_TOOLS assignment", - ), - ] - with patch.object(catalog, "PYTHON_TOOL_CATALOG", path): - for source, message in cases: - with self.subTest(source=source): - path.write_text(source, encoding="utf-8") + path = Path(directory) / "providers.json" + with patch.object(catalog, "PROVIDERS_PATH", path): + for field in ( + "package", + "version", + "entrypoint", + "description", + "requires", + ): + with self.subTest(field=field): + definition = { + key: value for key, value in valid.items() if key != field + } + path.write_text(json.dumps({"example": definition})) + message = ( + "uv dependency" + if field == "requires" + else "Malformed Python package definition" + ) with self.assertRaisesRegex(SystemExit, message): catalog.load_python_tools() - self.assertFalse(marker.exists()) def test_duplicate_definitions_retain_rejection(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - python_path = root / "python_tools.bzl" + python_path = root / "providers.json" metadata = { "version": "1", "description": "Example tool", @@ -105,6 +115,8 @@ def test_duplicate_definitions_retain_rejection(self) -> None: native_path.write_text(json.dumps({"example": metadata}), encoding="utf-8") duplicate_path = root / "two.lock.json" python_metadata = { + "provider": "python", + "requires": ["uv"], "package": "example", "version": "1", "entrypoint": "example", @@ -112,16 +124,16 @@ def test_duplicate_definitions_retain_rejection(self) -> None: } with ( patch.object(catalog, "LOCKFILE_ROOT", root), - patch.object(catalog, "PYTHON_TOOL_CATALOG", python_path), + patch.object(catalog, "PROVIDERS_PATH", python_path), ): for duplicate in ("lockfiles", "catalogs"): if duplicate == "lockfiles": duplicate_path.write_bytes(native_path.read_bytes()) - python_path.write_text("PYTHON_TOOLS = {}", encoding="utf-8") + python_path.write_text("{}", encoding="utf-8") else: duplicate_path.unlink() python_path.write_text( - f"PYTHON_TOOLS = {{'example': {python_metadata!r}}}", + json.dumps({"example": python_metadata}), encoding="utf-8", ) for query in ( diff --git a/tools/tests/metadata_test.py b/tools/tests/metadata_test.py new file mode 100644 index 0000000..e36183d --- /dev/null +++ b/tools/tests/metadata_test.py @@ -0,0 +1,102 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Check existing feature scripts can consume the shared package definitions.""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.internal.devcontainer import install as installer + +TOOLS_DIR = Path(__file__).resolve().parents[1] + + +class MetadataTest(unittest.TestCase): + def test_every_feature_version_and_checksum_input_is_exported(self) -> None: + # Removing the YAML loader must not leave an install or test command with + # an undefined version/checksum variable, even in an architecture branch. + metadata = installer._metadata_values() + for feature in ("s-core-local", "bazel-feature"): + for script in ("install.sh", "tests/test_default.sh"): + path = ( + TOOLS_DIR.parent + / "src/s-core-devcontainer/.devcontainer" + / feature + / script + ) + required = set( + re.findall( + r"\$\{([A-Za-z_][A-Za-z_0-9]*(?:_version|_sha256))(?:[^}]*)?\}", + path.read_text(), + ) + ) + with self.subTest(script=path): + self.assertTrue(required) + self.assertFalse( + required - metadata.keys(), required - metadata.keys() + ) + + def test_shell_metadata_keeps_command_substitution_as_literal_data(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + marker = root / "must-not-exist" + value = "$(touch " + str(marker) + ") 'quoted value'" + providers = {"fixture": {"provider": "release", "version": value}} + (tools / "providers.json").write_text(json.dumps(providers)) + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(tools / "internal/devcontainer/install.py"), + "shell-metadata", + ], + cwd=root, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + check = subprocess.run( + [ + "bash", + "-c", + 'eval "$1"; test "$fixture_version" = "$2"', + "metadata-test", + result.stdout, + value, + ], + cwd=root, + capture_output=True, + text=True, + ) + self.assertEqual(check.returncode, 0, check.stderr) + self.assertFalse(marker.exists()) + + def _copy_tools(self, root: Path) -> Path: + tools = root / "copied-tools" + shutil.copytree(TOOLS_DIR, tools, ignore=shutil.ignore_patterns("__pycache__")) + return tools + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python_tool_runner_test.sh b/tools/tests/python_tool_runner_test.sh index 74f1f91..089c5cd 100755 --- a/tools/tests/python_tool_runner_test.sh +++ b/tools/tests/python_tool_runner_test.sh @@ -20,6 +20,24 @@ runfiles_root="${TEST_SRCDIR}/${TEST_WORKSPACE}" installer="${runfiles_root}/tools/internal/devcontainer/install.py" runner="${runfiles_root}/tools/internal/run_python_tool.sh" +# Bazel and the container installer must consume the same package definitions. +# Only the cached Starlark representation is generated; JSON remains the source. +python3 - "${runfiles_root}/tools/providers.json" "${runfiles_root}/$1" <<'PYTHON' +import ast +import json +import sys +from pathlib import Path + +providers = json.loads(Path(sys.argv[1]).read_text()) +expected = { + name: {field: definition[field] for field in ("package", "version", "entrypoint", "description")} + for name, definition in providers.items() + if definition["provider"] == "python" +} +generated = ast.literal_eval(ast.parse(Path(sys.argv[2]).read_text()).body[0].value) +assert generated == expected, (generated, expected) +PYTHON + fake_uvx="${TEST_TMPDIR}/uvx" output="${TEST_TMPDIR}/uvx.args" working_directory="${TEST_TMPDIR}/working-directory"