From ebe65d13dcb5e5f632a1e3964cff54a461776c7c Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Tue, 3 Nov 2026 13:04:01 +0100 Subject: [PATCH 1/6] refactor(tools): declare complete environment selections --- .devcontainer/post_create_command.sh | 15 +- .github/renovate.json5 | 24 - .pre-commit-config.yaml | 2 +- REUSE.toml | 2 + .../.devcontainer/bazel-feature/install.sh | 57 +-- .../bazel-feature/tests/test_default.sh | 2 +- .../.devcontainer/bazel-feature/versions.yaml | 25 -- .../.devcontainer/s-core-local/install.sh | 220 ++++----- .../s-core-local/tests/test_default.sh | 8 +- .../.devcontainer/s-core-local/versions.yaml | 61 --- tools/BUILD.bazel | 4 + tools/README.md | 2 + tools/environments.json | 96 ++++ tools/internal/README.md | 100 ++++- tools/internal/catalog.py | 135 ++++++ tools/internal/devcontainer/install.py | 163 ++++++- .../devcontainer/load_feature_versions.sh | 22 +- tools/internal/sync_readme.py | 4 +- tools/lockfiles/python_tools.bzl | 12 + tools/providers.json | 299 +++++++++++++ tools/tests/catalog_test.py | 2 +- tools/tests/environment_test.py | 423 ++++++++++++++++++ 22 files changed, 1361 insertions(+), 317 deletions(-) delete mode 100644 src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml delete mode 100644 src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml create mode 100644 tools/environments.json create mode 100644 tools/providers.json create mode 100644 tools/tests/environment_test.py diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index d1cc20f2..d6f60d7d 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -13,20 +13,13 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -npm install -g @devcontainers/cli - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" REPOSITORY_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -P)" -# Install uv and uvx from the native lockfile first. The pinned uv release then -# resolves every Python distribution declared in the shared catalog. -sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install actionlint bazelisk buildifier ruff shellcheck uv uvx yamlfmt - -# Install the catalogued Python distribution system-wide. The explicit -# directories avoid root-specific uv defaults and make pre-commit available to -# the non-root development user. -sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-python pre-commit \ - --bin-dir /usr/local/bin --tool-dir /usr/local/share/uv/tools +# npm's global directory belongs to the development user in the Node.js base +# image. Keep npm setup in that user's environment; system tools require root. +"${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer --providers npm +sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer --providers native python # Hooks can only be registered after the catalogued executable is on PATH. pre-commit install diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 63722ed3..86236012 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -79,30 +79,6 @@ "datasourceTemplate": "github-releases", "depNameTemplate": "astral-sh/uv", "versioningTemplate": "semver" - }, - { - "customType": "regex", - "fileMatch": [ - "versions\\.yaml$" - ], - "matchStrings": [ - "apm:\\n\\s+version:\\s+(?\\d+\\.\\d+\\.\\d+)" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "microsoft/apm", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "fileMatch": [ - "versions\\.yaml$" - ], - "matchStrings": [ - "opencode:\\n\\s+version:\\s+(?\\d+\\.\\d+\\.\\d+)" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "anomalyco/opencode", - "versioningTemplate": "semver" } ], "enabledManagers": [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c56e0800..fec82e4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: language: system pass_filenames: false # Watch the shared reader as well as both catalog formats. - files: ^tools/(README\.md|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/(.*\.lock\.json|python_tools\.bzl))$ + files: ^tools/(README\.md|(environments|providers)\.json|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/(.*\.lock\.json|python_tools\.bzl))$ - repo: https://github.com/eclipse-score/tooling rev: 31ff8eee214e4e97ef8f5cb46e443273515b63ec hooks: diff --git a/REUSE.toml b/REUSE.toml index f4012b00..5bb950db 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -27,6 +27,8 @@ SPDX-License-Identifier = "Apache-2.0" path = ["resources/reopen_in_container.png", "resources/devcontainer_success.png", "tools/lockfiles/*.lock.json", + "tools/environments.json", + "tools/providers.json", "MODULE.bazel.lock", ".bazelversion", ] diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh index ea34c32c..29be9982 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh @@ -26,10 +26,10 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe # shellcheck disable=SC2034 # used by apt-get only inside this script -DEBIAN_FRONTEND=noninteractive +export DEBIAN_FRONTEND=noninteractive # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/bazel/versions.yaml +. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh ARCHITECTURE=$(dpkg --print-architecture) @@ -37,43 +37,48 @@ apt-get update # INSTALL CONTAINER BUILD DEPENDENCIES # Container build dependencies are not pinned, since they are removed anyway after container creation. -apt-get install apt-transport-https -y +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase build # Lockfile-managed Bazel tooling -/usr/local/share/score-tools/internal/devcontainer/install.py install bazelisk buildifier starpls +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature -# Bazelisk + Bazel -ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel +if /usr/local/share/score-tools/internal/devcontainer/install.py selected bazel-feature bazel; then + # Bazelisk + Bazel + ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel -# Pre-install a fixed Bazel version, setup the bash command completion -export USE_BAZEL_VERSION=${bazel_version} -# bazelisk downloads Bazel into the home directory of the user running the command -# lets assume there is only one user in the devcontainer -su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" -mkdir -p /etc/bash_completion.d -mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash -sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" + # Pre-install a fixed Bazel version, setup the bash command completion + export USE_BAZEL_VERSION=${bazel_version} + # Bazelisk caches the release for the development user selected by Dev Containers. + su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" + mkdir -p /etc/bash_completion.d + mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash + sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" + +fi # Configure Bazel to use system trust store for SSL/TLS connections # This is required for corporate environments with custom CA certificates echo 'startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit' >> /etc/bazel.bazelrc -# Code completion for C++ code of Bazel projects -# (see https://github.com/kiron1/bazel-compile-commands) -source /etc/lsb-release -curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb -# Extract correct sha256 for current DISTRIB_CODENAME and check -SHA256SUM="${bazel_compile_commands_amd64_sha256}" -if [ "${ARCHITECTURE}" = "arm64" ]; then - SHA256SUM="${bazel_compile_commands_arm64_sha256}" +if /usr/local/share/score-tools/internal/devcontainer/install.py selected bazel-feature bazel_compile_commands; then + # Code completion for C++ code of Bazel projects + # (see https://github.com/kiron1/bazel-compile-commands) + source /etc/lsb-release + curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb + # Extract correct sha256 for current DISTRIB_CODENAME and check + SHA256SUM="${bazel_compile_commands_amd64_sha256}" + if [ "${ARCHITECTURE}" = "arm64" ]; then + SHA256SUM="${bazel_compile_commands_arm64_sha256}" + fi + echo "${SHA256SUM} /tmp/bazel-compile-commands.deb" | sha256sum -c - || exit 1 + apt-get install -y --no-install-recommends --fix-broken /tmp/bazel-compile-commands.deb + rm /tmp/bazel-compile-commands.deb + fi -echo "${SHA256SUM} /tmp/bazel-compile-commands.deb" | sha256sum -c - || exit 1 -apt-get install -y --no-install-recommends --fix-broken /tmp/bazel-compile-commands.deb -rm /tmp/bazel-compile-commands.deb # Cleanup # REMOVE CONTAINER BUILD DEPENDENCIES -apt-get remove --purge -y apt-transport-https +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase cleanup apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh index c6447abc..23af6254 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh @@ -16,7 +16,7 @@ set -euo pipefail # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/bazel/versions.yaml +. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh bazelisk_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version bazelisk)" buildifier_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version buildifier)" diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml b/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml deleted file mode 100644 index c72910ee..00000000 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -bazel: - version: 8.6.0 - # no need to define sha256 here, as bazel is installed via bazelisk -bazel_compile_commands: - version: 0.18.0 - amd64: - # The following sha256sums are for the deb package bazel-compile-commands_-noble_amd64.deb - # It is generated by running 'sha256sum bazel-compile-commands_-noble_amd64.deb' - sha256: 6735ea846241497094719792ad3d4f67b1d3123048693d34fcdf7190f8c2da4e - arm64: - # The following sha256sums are for the deb package bazel-compile-commands_-noble_arm64.deb - # It is generated by running 'sha256sum bazel-compile-commands_-noble_arm64.deb' - sha256: d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88 diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh index 8e3ded87..c3bcab63 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh @@ -28,10 +28,10 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe # shellcheck disable=SC2034 # used by apt-get only inside this script -DEBIAN_FRONTEND=noninteractive +export DEBIAN_FRONTEND=noninteractive # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/s-core-local/versions.yaml +. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh ARCHITECTURE=$(dpkg --print-architecture) KERNEL=$(uname -s) @@ -40,147 +40,103 @@ apt-get update # Unminimize the image to include standard packages like man pages bash -c "yes || true" | unminimize -apt-get install -y man-db manpages manpages-dev manpages-posix manpages-posix-dev # INSTALL CONTAINER BUILD DEPENDENCIES # Container build dependencies are not pinned, since they are removed anyway after container creation. -apt-get install apt-transport-https -y - -# Python, via APT -apt-get install -y "python${python_version}" python3-pip python3-venv -# The following packages correspond to the list of packages installed by the -# devcontainer feature "python" (cf. https://github.com/devcontainers/features/tree/main/src/python ) -apt-get install -y flake8 python3-autopep8 black python3-yapf mypy pydocstyle pycodestyle bandit pipenv virtualenv pylint - -# uv must be installed from the native lockfile before it can install the -# Python distributions declared in the shared Python tool catalog below. -/usr/local/share/score-tools/internal/devcontainer/install.py install shellcheck ruff actionlint yamlfmt uv uvx apm opencode - -# Keep Python CLIs in system directories so every DevContainer user sees the -# same entrypoint and isolated environment. Bazel reads the same package pin -# from this catalog but executes it on demand with uvx. -/usr/local/share/score-tools/internal/devcontainer/install.py install-python pre-commit \ - --bin-dir /usr/local/bin --tool-dir /usr/local/share/uv/tools - -# GraphViz -# The Ubuntu Noble package of GraphViz -apt-get install -y graphviz="${graphviz_version}*" - -# Protobuf compiler, via APT (needed by FEO) -apt-get install -y protobuf-compiler="${protobuf_compiler_version}*" - -# Git and Git LFS, via APT -apt-get install -y git -apt-get install -y git-lfs -apt-get install -y gh - -# OpenJDK 21, via APT -# Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) -apt-get install -y ca-certificates-java openjdk-21-jdk-headless="${openjdk_21_version}*" -JAVA_HOME="$(dirname $(dirname $(realpath $(command -v javac))))" -export JAVA_HOME -echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh - -# QEMU system emulators for ARM and x86 -apt-get install -y --no-install-recommends --fix-broken \ - qemu-system-arm="${qemu_system_version}*" \ - qemu-system-x86="${qemu_system_version}*" - -# basedpyright -su $(ls /home) -c "uv tool install basedpyright@\"${basedpyright_version}\"" - -# pytest -su $(ls /home) -c "uv tool install pytest@\"${pytest_version}\"" - -# sshpass -apt-get install -y sshpass="${sshpass_version}*" - -# pycairo build dependencies (pkg-config + cairo headers needed when pip builds pycairo from source) -apt-get install -y pkg-config="${pkg_config_version}*" libcairo2-dev="${libcairo2_dev_version}*" - -# additional developer tools -apt-get install -y gdb="${gdb_version}*" - -apt-get install -y valgrind="1:${valgrind_version}*" - -apt-get install -y ripgrep="${ripgrep_version}*" - -# lcov - code coverage reporting for C/C++ -apt-get install -y lcov="${lcov_version}*" - -# CodeQL -apt-get install -y zstd -if [ "${ARCHITECTURE}" = "amd64" ]; then - VARIANT=linux64 - SHA256SUM="${codeql_amd64_sha256}" -elif [ "${ARCHITECTURE}" = "arm64" ]; then - if [ "${KERNEL}" = "Darwin" ]; then - VARIANT=osx64 - SHA256SUM="${codeql_arm64_sha256}" +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase build + +# One environment selection owns APT, native and Python membership. Provider +# installation runs before tool-specific lifecycle configuration below. +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local + +if /usr/local/share/score-tools/internal/devcontainer/install.py selected s-core-local openjdk_21; then + # OpenJDK 21, via APT + # Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) + JAVA_HOME="$(dirname "$(dirname "$(realpath "$(command -v javac)")")")" + export JAVA_HOME + echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh + +fi + +# Special setup also consumes environment membership, so removing a selection +# removes its installation rather than only changing the generated overview. +if /usr/local/share/score-tools/internal/devcontainer/install.py selected s-core-local codeql; then + # CodeQL + if [ "${ARCHITECTURE}" = "amd64" ]; then + VARIANT=linux64 + SHA256SUM="${codeql_amd64_sha256}" + elif [ "${ARCHITECTURE}" = "arm64" ]; then + if [ "${KERNEL}" = "Darwin" ]; then + VARIANT=osx64 + SHA256SUM="${codeql_arm64_sha256}" + else + echo "CodeQl unsupported architecture/os: ${ARCHITECTURE} on ${KERNEL}, skipping installation" + VARIANT=noinstall + fi else - echo "CodeQl unsupported architecture/os: ${ARCHITECTURE} on ${KERNEL}, skipping installation" - VARIANT=noinstall + echo "Unsupported architecture: ${ARCHITECTURE} for CodeQL" + exit 1 fi -else - echo "Unsupported architecture: ${ARCHITECTURE} for CodeQL" - exit 1 -fi -if [ "${VARIANT}" != "noinstall" ]; then - codeql_install_dir="/usr/local" - curl -L "https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-${VARIANT}.tar.zst" -o /tmp/codeql.tar.zst - echo "${SHA256SUM} /tmp/codeql.tar.zst" | sha256sum -c - || exit 1 - tar -I zstd -xf /tmp/codeql.tar.zst -C "${codeql_install_dir}" - ln -s "${codeql_install_dir}"/codeql/codeql /usr/local/bin/codeql - rm /tmp/codeql.tar.zst - export CODEQL_HOME=${codeql_install_dir}/codeql - echo "export CODEQL_HOME=${codeql_install_dir}/codeql" > /etc/profile.d/codeql.sh - - codeql pack download codeql/misra-cpp-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - codeql pack download codeql/misra-c-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - codeql pack download codeql/cert-cpp-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - codeql pack download codeql/cert-c-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - - # slim down codeql bundle (1.7GB -> 1.1 GB) by removing unnecessary language extractors and qlpacks - codeql_purge_dirs=( - "${codeql_install_dir}/codeql/csharp" - "${codeql_install_dir}/codeql/go" - "${codeql_install_dir}/codeql/java" - "${codeql_install_dir}/codeql/javascript" - "${codeql_install_dir}/codeql/python" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/swift-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/swift-queries" - "${codeql_install_dir}/codeql/ruby" - "${codeql_install_dir}/codeql/swift" - ) - for dir in "${codeql_purge_dirs[@]}"; do - if [ -d "${dir}" ]; then - rm -rf "${dir}" + if [ "${VARIANT}" != "noinstall" ]; then + codeql_install_dir="/usr/local" + curl -L "https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-${VARIANT}.tar.zst" -o /tmp/codeql.tar.zst + echo "${SHA256SUM} /tmp/codeql.tar.zst" | sha256sum -c - || exit 1 + tar -I zstd -xf /tmp/codeql.tar.zst -C "${codeql_install_dir}" + ln -s "${codeql_install_dir}"/codeql/codeql /usr/local/bin/codeql + rm /tmp/codeql.tar.zst + export CODEQL_HOME=${codeql_install_dir}/codeql + echo "export CODEQL_HOME=${codeql_install_dir}/codeql" > /etc/profile.d/codeql.sh + + if /usr/local/share/score-tools/internal/devcontainer/install.py selected s-core-local codeql_coding_standards; then + codeql_packs="$(/usr/local/share/score-tools/internal/devcontainer/install.py items codeql_coding_standards packs)" + while IFS= read -r pack; do + codeql pack download "${pack}@${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" + done <<< "${codeql_packs}" fi - done + + # slim down codeql bundle (1.7GB -> 1.1 GB) by removing unnecessary language extractors and qlpacks + codeql_purge_dirs=( + "${codeql_install_dir}/codeql/csharp" + "${codeql_install_dir}/codeql/go" + "${codeql_install_dir}/codeql/java" + "${codeql_install_dir}/codeql/javascript" + "${codeql_install_dir}/codeql/python" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/swift-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/swift-queries" + "${codeql_install_dir}/codeql/ruby" + "${codeql_install_dir}/codeql/swift" + ) + for dir in "${codeql_purge_dirs[@]}"; do + if [ -d "${dir}" ]; then + rm -rf "${dir}" + fi + done + fi + fi # Cleanup # REMOVE CONTAINER BUILD DEPENDENCIES -apt-get remove --purge -y apt-transport-https zstd +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh index f83eb4ad..19dd6df1 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh @@ -19,7 +19,7 @@ ARCHITECTURE=$(dpkg --print-architecture) KERNEL=$(uname -s) # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/s-core-local/versions.yaml +. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh shellcheck_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version shellcheck)" ruff_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version ruff)" @@ -95,7 +95,7 @@ check "validate pycairo can be built and installed from source" bash -c "pip ins check "validate gdb is working and has the correct version" bash -c "gdb --version | grep '${gdb_version}'" check "validate pkg-config is working and has the correct version" bash -c "pkg-config --version | grep '${pkg_config_version}'" check "validate gh is working and has the correct version" bash -c "gh --version | grep '${gh_version}'" -check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version}'" +check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version#*:}'" check "validate ripgrep is working and has the correct version" bash -c "rg --version | grep '${ripgrep_version}'" if [ "${ARCHITECTURE}" = "amd64" ] || { [ "${ARCHITECTURE}" = "arm64" ] && [ "${KERNEL}" = "Darwin" ]; }; then check "validate codeql is working and has the correct version" bash -c "codeql --version | grep '${codeql_version}'" @@ -106,6 +106,6 @@ fi check "validate lcov is working and has the correct version" bash -c "lcov --version | grep '${lcov_version}'" # Qemu target-related tools -check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version}'" -check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version}'" +check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version#*:}'" +check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version#*:}'" check "validate sshpass is working and has the correct version" bash -c "sshpass -V | grep '${sshpass_version}'" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml b/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml deleted file mode 100644 index 279bad08..00000000 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -basedpyright: - version: 1.35.0 -graphviz: - version: 2.42.2 -protobuf_compiler: - version: 3.21.12 -qemu_system: - version: 1:8.2.2 -sshpass: - version: 1.09 -gdb: - version: 15.0.50 -git: - version: 2.43.0 -git_lfs: - version: 3.4.1 -python: - version: '3.12' -pytest: - version: 9.0.2 -gh: - version: 2.45.0 -openjdk_21: - version: 21.0.12 -codeql: - # the coding_standards_version below dictates the codeql version - version: 2.21.4 - amd64: - # see https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-linux64.tar.zst.checksum.txt - sha256: 6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545 - arm64: - # see https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-osx64.tar.zst.checksum.txt - sha256: 810a1fff48aeb081be754e46e255edd90d8695966e78431a65edb00e9e6cc399 -codeql_coding_standards: - version: 2.54.0 -valgrind: - version: 3.22.0 -ripgrep: - version: 14.1.0 -lcov: - version: 2.0 -apm: - version: 0.35.0 -opencode: - version: 2.2.3 -pkg_config: - version: 1.8.1 -libcairo2_dev: - version: 1.18.0 diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 77238405..c1a267ab 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -51,6 +51,10 @@ multitool_aliases("opencode") # provide the platform-specific release binaries consumed by rules_multitool. python_tool("pre-commit") +python_tool("basedpyright") + +python_tool("pytest") + # This contract test keeps Bazel's shell launcher and the container installer # aligned with their shared catalog. sh_test( diff --git a/tools/README.md b/tools/README.md index 88127973..6bfa6c19 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,10 +41,12 @@ host-side Bazel invocation may require network access to download the binary. | --- | --- | --- | | `actionlint` | `1.7.7` | Check GitHub Actions workflow files | | `apm` | `0.35.0` | Manage APM packages | +| `basedpyright` | `1.35.0` | Type check Python and provide its language server | | `bazelisk` | `1.27.0` | Run the Bazel version selected by a repository | | `buildifier` | `8.2.1` | Format and lint Bazel files | | `opencode` | `2.2.3` | Run the OpenCode CLI | | `pre-commit` | `4.7.0` | Run repository pre-commit hooks | +| `pytest` | `9.0.2` | Run Python tests | | `ruff` | `0.11.13` | Check and format Python code | | `shellcheck` | `0.10.0` | Check shell scripts | | `starpls` | `0.1.22` | Provide language-server support for Starlark | diff --git a/tools/environments.json b/tools/environments.json new file mode 100644 index 00000000..3059df30 --- /dev/null +++ b/tools/environments.json @@ -0,0 +1,96 @@ +{ + "s-core-local": { + "native": [ + "shellcheck", + "ruff", + "actionlint", + "yamlfmt", + "uv", + "uvx", + "apm", + "opencode" + ], + "python": [ + "pre-commit", + "basedpyright", + "pytest" + ], + "apt": [ + "graphviz", + "protobuf_compiler", + "qemu_system", + "sshpass", + "gdb", + "git", + "git_lfs", + "gh", + "openjdk_21", + "valgrind", + "ripgrep", + "lcov", + "pkg_config", + "libcairo2_dev", + "python", + "python3-pip", + "python3-venv", + "flake8", + "python3-autopep8", + "black", + "python3-yapf", + "mypy", + "pydocstyle", + "pycodestyle", + "bandit", + "pipenv", + "virtualenv", + "pylint", + "ca-certificates-java", + "man-db", + "manpages", + "manpages-dev", + "manpages-posix", + "manpages-posix-dev" + ], + "setup": [ + "codeql", + "codeql_coding_standards" + ], + "build-apt": [ + "apt-transport-https", + "zstd" + ] + }, + "bazel-feature": { + "native": [ + "bazelisk", + "buildifier", + "starpls" + ], + "python": [], + "setup": [ + "bazel", + "bazel_compile_commands" + ], + "build-apt": [ + "apt-transport-https" + ] + }, + "repository-devcontainer": { + "native": [ + "actionlint", + "bazelisk", + "buildifier", + "ruff", + "shellcheck", + "uv", + "uvx", + "yamlfmt" + ], + "python": [ + "pre-commit" + ], + "npm": [ + "devcontainers-cli" + ] + } +} diff --git a/tools/internal/README.md b/tools/internal/README.md index cfae24e4..9e7653fa 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -31,7 +31,9 @@ One catalog is delivered through two execution paths: | `tools/lockfiles/*.lock.json` and `tools/lockfiles/python_tools.bzl` | Define versions and delivery metadata. | | `MODULE.bazel` | Makes native tool lockfiles available to `rules_multitool`. | | `tools/BUILD.bazel` | Exposes public Bazel targets for each command. | -| Feature installers | Install the commands exposed on the DevContainer's `PATH`. | +| `tools/providers.json` | Define APT policies, npm packages and special feature setup metadata. | +| `tools/environments.json` | Record the complete tool selection of each existing setup caller. | +| Feature installers | Apply selections and install the commands exposed on the DevContainer's `PATH`. | | `tools/run-tool` | Uses the command on `PATH` in a container or locally when available; otherwise uses the public Bazel alias (`--strict` forces Bazel). | | `tools/README.md` | Documents every command and its version from the catalog. | @@ -86,6 +88,10 @@ supported platforms, download URLs, checksums, and archive layouts. [`devcontainer/install.py`](devcontainer/install.py) and README generation. The reader has no installation side effects; download, platform selection, and installation remain in the container installer. +Other direct packages and special setup are defined in +[`providers.json`](../providers.json): APT constraints, npm package identities, +Bazel preinstallation and CodeQL metadata. The selections reference these names +without copying their versions. Python command-line tool metadata lives in [`tools/lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl). The file is @@ -100,6 +106,13 @@ This shared format keeps the package, console entrypoint, version, and description in one source of truth while Bazel execution remains independent of host Python. +APT `distribution` policies intentionally follow the distribution, `prefix` +constraints include Debian epochs, and `package-name` selects a versioned package +name (for example `python3.12`). Compatibility expectations for Git, Git LFS and +gh remain separate from their unpinned APT requests. The shell metadata bridge +quotes catalog data and uses the image's standard-library Python; it does not +download another parser. + The catalogs have distinct ownership: `devcontainer-lock.json` records external DevContainer features, `python_tools.bzl` records Python package releases, and `uv.lock.json` records the installer runtime. Each version is therefore owned @@ -108,6 +121,68 @@ by the component that resolves it. The published `score_devcontainer` Bazel module and DevContainer image share a release version. Consumers using both must pin the same release. +## Tool selections and image variants + +Definitions answer **which version and installation method?** Named selections +in [`tools/environments.json`](../environments.json) answer **which tools belong +here?** Image and feature setup determines the base OS and lifecycle. +Selections contain names, not another copy of versions or download metadata. + +| Selection | Existing caller | +| --- | --- | +| `s-core-local` | The published image's `s-core-local` feature | +| `bazel-feature` | Its `bazel-feature` feature | +| `repository-devcontainer` | This repository's post-create setup | + +These names identify the current callers and their historical tool membership; +they are not newly designed tool groups. The published image combines the first +two through its existing features. A future lean CI image can define its own +selection based on the jobs it needs to run. This does not add another image. + +Each selection groups its tools by installation method: + +| Group | Definition owner | Applied by | +| --- | --- | --- | +| `native` | `lockfiles/*.lock.json` | The native downloader | +| `python` | `lockfiles/python_tools.bzl` | Catalogued uv, with system-wide entrypoints | +| `apt` | `providers.json` | apt, preserving package/version policy and recommendations | +| `build-apt` | `providers.json` | apt during build; removed during cleanup | +| `npm` | `providers.json` | npm in the appropriate user's environment | +| `setup` | `providers.json` | The feature's existing special setup, gated by membership | + +For example, the local feature applies its selected groups during setup: + +```console +$ python3 tools/internal/devcontainer/install.py install-environment s-core-local --phase build +$ python3 tools/internal/devcontainer/install.py install-environment s-core-local +# Feature-specific configuration follows, then temporary packages are removed: +$ python3 tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup +``` + +The repository setup applies `npm` as the development user and `native python` +with sudo, using `--providers`. This preserves the Node base image's npm access +while putting Python commands in system paths. The feature's pre-commit, +basedpyright and pytest use `/usr/local/bin` and shared uv tool environments; +all container users can access them. Package versions are unchanged. + +`--destination`, `--os` and `--cpu` retain their native-install meanings. +APT/setup platform declarations are also checked; `--os` and `--cpu` select +metadata, not an emulated runtime for apt or uv. The whole selection is checked +before applying any phase: unknown names, wrong provider groups, missing direct +dependencies and unsupported platform artifacts fail before installation. +CodeQL's existing Linux ARM64 omission is explicit. Download or installation +failure can still leave earlier tools installed; application is not transactional. + +Feature scripts query `selected ENVIRONMENT TOOL` before special setup such as +JAVA_HOME, CodeQL or Bazel preinstallation. CodeQL pack names come from the owned +bundle through `items`, not a second list in the script. Removing membership stops +new setup from offering the tool; it does not uninstall existing host copies. + +This file covers the direct tools managed by our three setup scripts. Base-image +contents, external Dev Container features (such as LLVM, common-utils and DinD), +and their options and locks retain their existing configuration owners. It is +not an inventory of transitive packages, editor extensions or image contents. + ## Adding or updating a tool For a tool distributed as a native release artifact, make the following @@ -120,11 +195,9 @@ changes together: [`MODULE.bazel`](../../MODULE.bazel). 3. Add `multitool_aliases("")` to [`tools/BUILD.bazel`](../BUILD.bazel). -4. Add the command to the appropriate feature installer: - [`s-core-local/install.sh`](../../src/s-core-devcontainer/.devcontainer/s-core-local/install.sh) - for general tools or - [`bazel-feature/install.sh`](../../src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh) - for Bazel-specific tools. +4. Add the command to the applicable native selection in + [`tools/environments.json`](../environments.json): `s-core-local` for the + `s-core-local` feature or `bazel-feature` for the `bazel-feature` feature. 5. Read the lockfile version and test the installed command in the matching feature test: [`s-core-local/tests/test_default.sh`](../../src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh) @@ -136,20 +209,21 @@ changes together: $ python3 tools/internal/sync_readme.py ``` -7. If this repository's own DevContainer needs the command, add it to - [`.devcontainer/post_create_command.sh`](../../.devcontainer/post_create_command.sh). +7. If this repository's own DevContainer needs the command, add it to the + `repository-devcontainer` selection in `tools/environments.json`. For a Python command-line tool, add an entry to `tools/lockfiles/python_tools.bzl`, add `python_tool("")` to -`tools/BUILD.bazel`, and install it through `install.py install-python` in -each applicable DevContainer installer. Keep the catalog as a single literal +`tools/BUILD.bazel`, and add its name to the `python` group of each applicable +selection in `tools/environments.json`. Keep the catalog as a single literal `PYTHON_TOOLS` assignment: this is what makes it safe for the installer and directly loadable by Bazel. The `package` selects the distribution passed to `uv`, while `entrypoint` names the console command because those names need not be identical. Bazel invokes Python tools on demand with the pinned `uvx`; DevContainers -install the same package with the pinned `uv`. Add launcher and installer +install the same package with the pinned `uv`. A Python selection must also +select native `uv`; Bazel already carries pinned uvx in its launcher runfiles. Add launcher and installer coverage to `python_tool_runner_test` when either delivery contract changes. Keep related commands such as `uv` and `uvx` in one lockfile when upstream @@ -170,7 +244,9 @@ description: $ python3 tools/internal/sync_readme.py ``` -The pre-commit hook runs the same command; pre-commit rejects the commit if +The pre-commit hook also validates selection names, provider groups, direct +dependencies and tool references. +It runs the same command; pre-commit rejects the commit if running it changes `tools/README.md`, so a stale table cannot be committed. Run the feature test for every installer changed. The feature tests read their diff --git a/tools/internal/catalog.py b/tools/internal/catalog.py index 69d07aec..4b663a9c 100644 --- a/tools/internal/catalog.py +++ b/tools/internal/catalog.py @@ -46,6 +46,7 @@ class ToolData(TypedDict): LOCKFILE_ROOT = Path(__file__).resolve().parents[1] / "lockfiles" PYTHON_TOOL_CATALOG = LOCKFILE_ROOT / "python_tools.bzl" +ENVIRONMENTS_PATH = LOCKFILE_ROOT.parent / "environments.json" class PythonTool(TypedDict): @@ -233,3 +234,137 @@ def resolve_lockfile(tool: str, lockfile: str | None = None) -> str: if _lockfile_path(tool).exists(): return tool return _find_lockfile(tool) + + +PROVIDERS_PATH = LOCKFILE_ROOT.parent / "providers.json" + + +def load_providers() -> dict: + """Read APT, npm and feature-specific setup definitions.""" + providers = json.loads(PROVIDERS_PATH.read_text(encoding="utf-8")) + if not isinstance(providers, dict): + raise SystemExit("Tool providers must be a dictionary") + for name, definition in providers.items(): + if not isinstance(definition, dict) or definition.get("provider") not in ( + "apt", + "npm", + "release", + "bazelisk", + ): + raise SystemExit(f"Invalid provider definition for '{name}'") + if definition["provider"] == "apt": + if definition.get("policy") not in ( + "distribution", + "prefix", + "package-name", + ): + raise SystemExit(f"Invalid APT policy for '{name}'") + packages = definition.get("packages") + if ( + not isinstance(packages, list) + or not packages + or any( + not isinstance(package, str) + or not package + or package.startswith("-") + for package in packages + ) + ): + raise SystemExit(f"Invalid APT packages for '{name}'") + if definition["policy"] != "distribution" and not definition.get("version"): + raise SystemExit(f"Missing APT version constraint for '{name}'") + elif definition["provider"] == "npm": + if ( + not isinstance(definition.get("package"), str) + or not definition["package"] + ): + raise SystemExit(f"Missing npm package for '{name}'") + return providers + + +def all_definitions() -> dict: + """Combine metadata for reference validation without loading repository configs.""" + tools = {} + sources = ( + ((name, {**data, "provider": "native"}) for name, _, data in _iter_catalog()), + ( + (name, {**data, "provider": "python", "requires": ["uv"]}) + for name, data in load_python_tools().items() + ), + load_providers().items(), + ) + for source in sources: + for name, definition in source: + if name in tools: + raise SystemExit(f"Duplicate tool definition: '{name}'") + tools[name] = definition + return tools + + +def load_environments() -> dict[str, dict[str, list[str]]]: + """Validate membership, provider categories and deliberately selected dependencies.""" + data = json.loads(ENVIRONMENTS_PATH.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SystemExit("Tool environments must be a dictionary") + tools = all_definitions() + categories = { + "native": {"native"}, + "python": {"python"}, + "apt": {"apt"}, + "build-apt": {"apt"}, + "npm": {"npm"}, + "setup": {"release", "bazelisk"}, + } + for name, selection in data.items(): + if not isinstance(name, str) or not name: + raise SystemExit("Tool environment names must be nonempty strings") + if not isinstance(selection, dict) or selection.keys() - categories.keys(): + raise SystemExit(f"Invalid provider categories in environment '{name}'") + for category, names in selection.items(): + if not isinstance(names, list) or any( + not isinstance(tool, str) or not tool for tool in names + ): + raise SystemExit(f"Environment '{name}/{category}' must be a tool list") + if len(names) != len(set(names)): + raise SystemExit( + f"Environment '{name}/{category}' lists a tool more than once" + ) + for tool in names: + if tool not in tools: + raise SystemExit(f"Unknown tool '{tool}' in environment '{name}'") + if tools[tool]["provider"] not in categories[category]: + raise SystemExit( + f"Wrong provider for '{tool}' in environment '{name}/{category}'" + ) + if (category == "build-apt") != ( + tools[tool].get("phase") == "build-only" + ): + raise SystemExit( + f"Wrong installation phase for '{tool}' in environment '{name}'" + ) + selected = {tool for names in selection.values() for tool in names} + for tool in selected: + for dependency in tools[tool].get("requires", []): + if dependency not in selected: + raise SystemExit( + f"Tool '{tool}' requires '{dependency}' in environment '{name}'" + ) + return data + + +def apt_requirements( + selection: dict, definitions: dict, *, recommends: bool +) -> list[str]: + """Translate selected APT policies to requests, preserving Debian epochs.""" + requirements = [] + for name in selection.get("apt", []): + tool = definitions[name] + if tool.get("install_recommends", True) != recommends: + continue + for package in tool["packages"]: + if tool["policy"] == "package-name": + package = package.format(version=tool["version"]) + elif tool["policy"] == "prefix": + package += "=" + tool["version"] + "*" + requirements.append(package) + return requirements diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index ee3e2a28..8ad62e43 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -19,6 +19,7 @@ Usage: install.py install shellcheck yamlfmt + install.py install-environment s-core-local install.py install-python pre-commit install.py version shellcheck """ @@ -46,8 +47,12 @@ from catalog import ( Binary, ToolData, + all_definitions, + apt_requirements, load_catalog_versions, + load_environments, load_python_tools, + load_providers, load_tool, resolve_lockfile, ) @@ -209,11 +214,19 @@ def _cmd_install(args: argparse.Namespace) -> int: """Download, verify, and install tools from the lockfile catalog.""" dest_dir = Path(args.destination) + # Resolve the complete selection before downloading or writing anything. + # Otherwise a missing tool/platform late in the list leaves a partial set. + binaries: list[tuple[str, Binary]] = [] for tool in args.tools: lockfile = resolve_lockfile(tool, args.lockfile) tool_data = load_tool(lockfile, tool) - binary = _select_binary(tool_data, args.os, args.cpu) + try: + binary = _select_binary(tool_data, args.os, args.cpu) + except SystemExit as exc: + raise SystemExit(f"Tool '{tool}': {exc}") from exc + binaries.append((tool, binary)) + for tool, binary in binaries: kind = binary["kind"] url = binary["url"] expected_sha256 = binary["sha256"] @@ -254,12 +267,139 @@ def _cmd_install(args: argparse.Namespace) -> int: return 0 +def _cmd_install_environment(args: argparse.Namespace) -> int: + """Apply selected provider groups; feature scripts retain special setup.""" + selection = load_environments().get(args.environment) + if selection is None: + raise SystemExit(f"Unknown tool environment '{args.environment}'") + definitions = all_definitions() + system = args.os + "/" + args.cpu + # Preflight the complete environment, even for a build/cleanup phase. A + # missing runtime artifact must be reported before installing build packages. + for names in selection.values(): + for name in names: + definition = definitions[name] + if definition["provider"] == "native": + try: + _select_binary(definition, args.os, args.cpu) + except SystemExit as exc: + raise SystemExit(f"Tool '{name}': {exc}") from exc + elif definition["provider"] in ("apt", "release", "bazelisk"): + if system not in definition[ + "platforms" + ] and system not in definition.get("omissions", {}): + raise SystemExit(f"Tool '{name}' is unsupported on {system}") + + if args.phase != "runtime": + packages = [ + package + for name in selection.get("build-apt", []) + for package in definitions[name]["packages"] + ] + if packages: + action = ( + ["install", "-y"] + if args.phase == "build" + else ["remove", "--purge", "-y"] + ) + subprocess.run(["apt-get", *action, *packages], check=True) + return 0 + + # QEMU deliberately omits recommendations. Other packages keep apt's normal + # recommendations, including the headers needed to build Python extensions. + for recommends in (True, False) if "apt" in args.providers else (): + packages = apt_requirements(selection, definitions, recommends=recommends) + if packages: + flags = [] if recommends else ["--no-install-recommends", "--fix-broken"] + subprocess.run(["apt-get", "install", "-y", *flags, *packages], check=True) + if "native" in args.providers: + args.tools = selection.get("native", []) + args.lockfile = None + _cmd_install(args) + if "python" in args.providers and selection.get("python"): + args.tools = selection["python"] + args.uv = str(Path(args.destination).resolve() / "uv") + args.bin_dir = args.destination + _cmd_install_python(args) + for name in selection.get("npm", []) if "npm" in args.providers else (): + subprocess.run( + ["npm", "install", "-g", definitions[name]["package"]], check=True + ) + return 0 + + +def _cmd_shell_metadata(args: argparse.Namespace) -> int: + """Emit quoted data for the provider-specific feature setup, never shell code from definitions.""" + import re + import shlex + + def emit(name, value): + if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name): + raise SystemExit(f"Invalid metadata variable: {name}") + if isinstance(value, dict): + for key, child in value.items(): + emit(name + "_" + key, child) + elif isinstance(value, str): + print("export " + name + "=" + shlex.quote(value)) + + for name, definition in load_providers().items(): + # Export only lifecycle setup inputs. Inventory metadata can use keys + # such as linux/arm64 which deliberately are not shell identifiers. + for field in ("version", "compatibility_version", "amd64", "arm64"): + if field in definition: + emit(name.replace("-", "_") + "_" + field, definition[field]) + if "compatibility_version" in definition: + emit( + name.replace("-", "_") + "_version", definition["compatibility_version"] + ) + # Python distributions share their pins with Bazel and container setup. + for name, definition in load_python_tools().items(): + emit(name.replace("-", "_"), definition) + return 0 + + +def _cmd_selected(args: argparse.Namespace) -> int: + """Return whether a tool is selected, for provider-specific lifecycle setup.""" + selections = load_environments()[args.environment] + return 0 if any(args.tool in names for names in selections.values()) else 1 + + +def _cmd_items(args: argparse.Namespace) -> int: + """Expose a provider-owned bundle list to lifecycle setup and verification.""" + value = load_providers()[args.tool][args.field] + if ( + not isinstance(value, list) + or not value + or any(not isinstance(item, str) or not item or "\n" in item for item in value) + ): + raise SystemExit( + f"{args.tool}/{args.field} must be a non-empty list of single-line strings" + ) + print("\n".join(value)) + return 0 + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Install pinned tools from multitool-compatible lockfiles.", ) subparsers = parser.add_subparsers(dest="command", required=True) + metadata_parser = subparsers.add_parser( + "shell-metadata", help="Read feature setup metadata." + ) + metadata_parser.set_defaults(func=_cmd_shell_metadata) + selected_parser = subparsers.add_parser( + "selected", help="Query membership for feature-specific setup." + ) + selected_parser.add_argument("environment") + selected_parser.add_argument("tool") + selected_parser.set_defaults(func=_cmd_selected) + items_parser = subparsers.add_parser("items", help="Read a setup bundle's members.") + items_parser.add_argument("tool") + items_parser.add_argument("field") + items_parser.set_defaults(func=_cmd_items) + install_parser = subparsers.add_parser( "install", help="Download, verify, and install tools.", @@ -271,6 +411,27 @@ def _build_parser() -> argparse.ArgumentParser: install_parser.add_argument("--cpu", default=_detect_cpu()) install_parser.set_defaults(func=_cmd_install) + environment_parser = subparsers.add_parser( + "install-environment", + help="Apply a tool selection from tools/environments.json.", + ) + environment_parser.add_argument("environment") + environment_parser.add_argument( + "--providers", + nargs="+", + choices=("apt", "native", "python", "npm"), + default=("apt", "native", "python", "npm"), + help="Apply only these provider groups, e.g. npm as the development user.", + ) + environment_parser.add_argument( + "--phase", choices=("runtime", "build", "cleanup"), default="runtime" + ) + environment_parser.add_argument("--tool-dir", default="/usr/local/share/uv/tools") + environment_parser.add_argument("--destination", default="/usr/local/bin") + environment_parser.add_argument("--os", default=_detect_os()) + environment_parser.add_argument("--cpu", default=_detect_cpu()) + environment_parser.set_defaults(func=_cmd_install_environment) + install_python_parser = subparsers.add_parser( "install-python", help="Install Python command-line tools declared in python_tools.bzl.", diff --git a/tools/internal/devcontainer/load_feature_versions.sh b/tools/internal/devcontainer/load_feature_versions.sh index bc2ce333..74e242f6 100755 --- a/tools/internal/devcontainer/load_feature_versions.sh +++ b/tools/internal/devcontainer/load_feature_versions.sh @@ -15,20 +15,8 @@ set -euo pipefail -ARCHITECTURE=$(dpkg --print-architecture) -VERSION="v4.47.2" - -SHA256_FIELD="1bb99e1019e23de33c7e6afc23e93dad72aad6cf2cb03c797f068ea79814ddb0" # Default to amd64 -if [ "${ARCHITECTURE}" = "arm64" ]; then - SHA256_FIELD="05df1f6aed334f223bb3e6a967db259f7185e33650c3b6447625e16fea0ed31f" -fi - -yq_binary="$(mktemp)" -trap 'rm -f "${yq_binary}"' EXIT - -curl --fail --location "https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_linux_${ARCHITECTURE}" -o "${yq_binary}" -echo "${SHA256_FIELD} ${yq_binary}" | sha256sum -c - || exit 1 -chmod +x "${yq_binary}" - -# Read tool versions and metadata into environment variables -export $("${yq_binary}" eval '.. | select((tag == "!!map" or tag == "!!seq") | not) | (path | join("_")) + "=" + .' "$1" | awk '!/=$/{print }' | xargs) +# Shared definitions are literal data. The bridge quotes values and validates +# variable names before emitting exports for provider-specific shell setup. +score_feature_metadata="$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" +eval "${score_feature_metadata}" +unset score_feature_metadata diff --git a/tools/internal/sync_readme.py b/tools/internal/sync_readme.py index 3f7aae77..b38bfa05 100755 --- a/tools/internal/sync_readme.py +++ b/tools/internal/sync_readme.py @@ -21,7 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) -from catalog import load_catalog_descriptions, load_catalog_versions +from catalog import load_catalog_descriptions, load_catalog_versions, load_environments TOOLS_DIR = Path(__file__).resolve().parent.parent @@ -67,6 +67,8 @@ def main(argv: list[str] | None = None) -> int: ) parser.parse_args(argv) + # Validate selection references alongside the catalog in the documentation hook. + load_environments() current = README_PATH.read_text(encoding="utf-8") table = _render_table(load_catalog_versions(), load_catalog_descriptions()) expected = _updated_readme(current, table) diff --git a/tools/lockfiles/python_tools.bzl b/tools/lockfiles/python_tools.bzl index 262d40b0..94ff046a 100644 --- a/tools/lockfiles/python_tools.bzl +++ b/tools/lockfiles/python_tools.bzl @@ -27,4 +27,16 @@ PYTHON_TOOLS = { # are reviewed together. "description": "Run repository pre-commit hooks", }, + "basedpyright": { + "package": "basedpyright", + "version": "1.35.0", + "entrypoint": "basedpyright", + "description": "Type check Python and provide its language server", + }, + "pytest": { + "package": "pytest", + "version": "9.0.2", + "entrypoint": "pytest", + "description": "Run Python tests", + }, } diff --git a/tools/providers.json b/tools/providers.json new file mode 100644 index 00000000..22a0d9ba --- /dev/null +++ b/tools/providers.json @@ -0,0 +1,299 @@ +{ + "graphviz": { + "provider": "apt", + "packages": ["graphviz"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "2.42.2" + }, + "protobuf_compiler": { + "provider": "apt", + "packages": ["protobuf-compiler"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "3.21.12" + }, + "qemu_system": { + "provider": "apt", + "packages": ["qemu-system-arm", "qemu-system-x86"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1:8.2.2", + "install_recommends": false + }, + "sshpass": { + "provider": "apt", + "packages": ["sshpass"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.09" + }, + "gdb": { + "provider": "apt", + "packages": ["gdb"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "15.0.50" + }, + "git": { + "provider": "apt", + "packages": ["git"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "2.43.0" + }, + "git_lfs": { + "provider": "apt", + "packages": ["git-lfs"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "3.4.1" + }, + "gh": { + "provider": "apt", + "packages": ["gh"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "2.45.0" + }, + "openjdk_21": { + "provider": "apt", + "packages": ["openjdk-21-jdk-headless"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "21.0.12" + }, + "valgrind": { + "provider": "apt", + "packages": ["valgrind"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1:3.22.0" + }, + "ripgrep": { + "provider": "apt", + "packages": ["ripgrep"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "14.1.0" + }, + "lcov": { + "provider": "apt", + "packages": ["lcov"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "2.0" + }, + "pkg_config": { + "provider": "apt", + "packages": ["pkg-config"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.8.1" + }, + "libcairo2_dev": { + "provider": "apt", + "packages": ["libcairo2-dev"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.18.0" + }, + "python": { + "provider": "apt", + "description": "Distribution Python interpreter", + "packages": ["python{version}"], + "policy": "package-name", + "version": "3.12", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "flake8": { + "provider": "apt", + "packages": ["flake8"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-autopep8": { + "provider": "apt", + "packages": ["python3-autopep8"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "black": { + "provider": "apt", + "packages": ["black"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-yapf": { + "provider": "apt", + "packages": ["python3-yapf"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "mypy": { + "provider": "apt", + "packages": ["mypy"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pydocstyle": { + "provider": "apt", + "packages": ["pydocstyle"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pycodestyle": { + "provider": "apt", + "packages": ["pycodestyle"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "bandit": { + "provider": "apt", + "packages": ["bandit"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pipenv": { + "provider": "apt", + "packages": ["pipenv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "virtualenv": { + "provider": "apt", + "packages": ["virtualenv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pylint": { + "provider": "apt", + "packages": ["pylint"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "ca-certificates-java": { + "provider": "apt", + "packages": ["ca-certificates-java"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "man-db": { + "provider": "apt", + "packages": ["man-db"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages": { + "provider": "apt", + "packages": ["manpages"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-dev": { + "provider": "apt", + "packages": ["manpages-dev"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-posix": { + "provider": "apt", + "packages": ["manpages-posix"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-posix-dev": { + "provider": "apt", + "packages": ["manpages-posix-dev"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "codeql": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64"], + "version": "2.21.4", + "amd64": { + "sha256": "6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545" + }, + "arm64": { + "sha256": "810a1fff48aeb081be754e46e255edd90d8695966e78431a65edb00e9e6cc399" + }, + "omissions": { + "linux/arm64": "The selected bundle has no supported Linux arm64 artifact" + } + }, + "codeql_coding_standards": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64"], + "version": "2.54.0", + "requires": ["codeql"], + "omissions": { + "linux/arm64": "Not installed without the CodeQL bundle" + }, + "packs": [ + "codeql/misra-cpp-coding-standards", + "codeql/misra-c-coding-standards", + "codeql/cert-cpp-coding-standards", + "codeql/cert-c-coding-standards" + ] + }, + "bazel": { + "provider": "bazelisk", + "policy": "exact", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "8.6.0", + "requires": ["bazelisk"] + }, + "bazel_compile_commands": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "0.18.0", + "amd64": { + "sha256": "6735ea846241497094719792ad3d4f67b1d3123048693d34fcdf7190f8c2da4e" + }, + "arm64": { + "sha256": "d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88" + } + }, + "apt-transport-https": { + "provider": "apt", + "description": "Temporary container build dependency (removed after setup)", + "packages": ["apt-transport-https"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "phase": "build-only" + }, + "zstd": { + "provider": "apt", + "description": "Temporary container build dependency (removed after setup)", + "packages": ["zstd"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "phase": "build-only" + }, + "devcontainers-cli": { + "provider": "npm", + "description": "Build and test Dev Containers", + "package": "@devcontainers/cli", + "policy": "latest", + "platforms": ["Node.js-supported platforms"], + "phase": "repository setup" + }, + "python3-pip": { + "provider": "apt", + "description": "Distribution pip (independent of the interpreter package-name constraint)", + "packages": ["python3-pip"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-venv": { + "provider": "apt", + "description": "Distribution Python venv support", + "packages": ["python3-venv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + } +} diff --git a/tools/tests/catalog_test.py b/tools/tests/catalog_test.py index 66310650..0d74c7dd 100644 --- a/tools/tests/catalog_test.py +++ b/tools/tests/catalog_test.py @@ -52,7 +52,7 @@ def test_metadata_matches_definitions(self) -> None: source = (TOOLS_DIR / "lockfiles/python_tools.bzl").read_text(encoding="utf-8") python_tools = ast.literal_eval(ast.parse(source).body[0].value) self.assertEqual(catalog.load_python_tools(), python_tools) - self.assertEqual(set(python_tools), {"pre-commit"}) + self.assertEqual(set(python_tools), {"pre-commit", "basedpyright", "pytest"}) versions.update( (name, metadata["version"]) for name, metadata in python_tools.items() ) diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py new file mode 100644 index 00000000..4274a96f --- /dev/null +++ b/tools/tests/environment_test.py @@ -0,0 +1,423 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Tool selections are portable and fail before applying an invalid set.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.internal import catalog + +TOOLS_DIR = Path(__file__).resolve().parents[1] + + +class EnvironmentTest(unittest.TestCase): + def test_rejects_invalid_selection_data_and_wrong_providers(self) -> None: + cases = ( + ([], "must be a dictionary"), + ({"": {"native": []}}, "names must be nonempty"), + ({"ci": []}, "Invalid provider categories"), + ({"ci": {"unknown": []}}, "Invalid provider categories"), + ({"ci": {"native": "ruff"}}, "must be a tool list"), + ({"ci": {"native": [7]}}, "must be a tool list"), + ({"ci": {"native": ["ruff", "ruff"]}}, "more than once"), + ({"ci": {"native": ["missing"]}}, "Unknown tool 'missing'"), + ({"ci": {"native": ["pre-commit"]}}, "Wrong provider for 'pre-commit'"), + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "environments.json" + with patch.object(catalog, "ENVIRONMENTS_PATH", path): + for data, message in cases: + with self.subTest(data=data): + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(SystemExit, message): + catalog.load_environments() + + def test_copied_installer_applies_an_independent_ci_selection(self) -> None: + # A new image selection needs no changes to the installer or its caller. + # Local artifacts exercise actual download, checksum and install code. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + artifacts = root / "artifacts" + artifacts.mkdir() + metadata = {} + for name in ("lint", "check"): + source = artifacts / name + source.write_text(f"#!/bin/sh\nprintf '%s\\n' '{name} 1.0'\n") + metadata[name] = { + "version": "1.0", + "binaries": [ + { + "os": "linux", + "cpu": "arm64", + "kind": "file", + "url": source.as_uri(), + "sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + } + ], + } + (tools / "lockfiles/ci.lock.json").write_text(json.dumps(metadata)) + (tools / "environments.json").write_text( + json.dumps( + { + "ci": {"native": ["lint", "check"]}, + "unused": {"native": ["ruff"]}, + } + ) + ) + destination = root / "ci tools" + result = self._run(tools, root, "ci", destination) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual({p.name for p in destination.iterdir()}, {"lint", "check"}) + for name in ("lint", "check"): + self.assertEqual( + (destination / name).read_bytes(), (artifacts / name).read_bytes() + ) + self.assertTrue((destination / name).stat().st_mode & 0o111) + + def test_invalid_later_tool_or_platform_prevents_any_download(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + # The first URL deliberately does not exist. If installation starts + # before validating the second entry, this produces the wrong error. + first = { + "version": "1.0", + "binaries": [ + { + "os": "linux", + "cpu": "arm64", + "kind": "file", + "url": (root / "must-not-be-downloaded").as_uri(), + "sha256": "0" * 64, + } + ], + } + unsupported = {"version": "1.0", "binaries": []} + (tools / "lockfiles/fixture.lock.json").write_text( + json.dumps( + { + "first": first, + "unsupported": unsupported, + } + ) + ) + for second, message in ( + ("missing", "Unknown tool 'missing'"), + ("unsupported", "Tool 'unsupported': No binary defined"), + ): + with self.subTest(second=second): + (tools / "environments.json").write_text( + json.dumps( + { + "ci": {"native": ["first", second]}, + } + ) + ) + destination = root / "not-created" + result = self._run(tools, root, "ci", destination) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + self.assertFalse(destination.exists()) + + def test_unknown_environment_is_reported_without_installing(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + destination = root / "not-created" + result = self._run(tools, root, "no-such-environment", destination) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "Unknown tool environment 'no-such-environment'", result.stderr + ) + self.assertFalse(destination.exists()) + + def test_complete_selection_preserves_provider_order_and_package_policies( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + fake_bin = root / "fake-bin" + fake_bin.mkdir() + # Real subprocesses record the apt/npm/uv boundary without changing + # host packages. uv itself is installed from a checked local artifact. + recorder = ( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "with open(os.environ['SCORE_TEST_LOG'], 'a') as log:\n" + " log.write(json.dumps({'command': Path(sys.argv[0]).name, " + "'args': sys.argv[1:], 'bin_dir': os.environ.get('UV_TOOL_BIN_DIR'), " + "'tool_dir': os.environ.get('UV_TOOL_DIR')}) + '\\n')\n" + ) + for name in ("apt-get", "npm"): + executable = fake_bin / name + executable.write_text(recorder) + executable.chmod(0o755) + uv_artifact = root / "uv-artifact" + uv_artifact.write_text(recorder) + (tools / "lockfiles/uv.lock.json").write_text( + json.dumps( + { + "uv": { + "version": "test", + "binaries": [ + { + "os": "linux", + "cpu": "arm64", + "kind": "file", + "url": uv_artifact.as_uri(), + "sha256": hashlib.sha256( + uv_artifact.read_bytes() + ).hexdigest(), + } + ], + }, + } + ) + ) + providers = { + "regular": { + "provider": "apt", + "packages": ["regular"], + "policy": "prefix", + "version": "1:2.3", + "platforms": ["linux/arm64"], + }, + "minimal": { + "provider": "apt", + "packages": ["minimal"], + "policy": "distribution", + "install_recommends": False, + "platforms": ["linux/arm64"], + }, + "build-helper": { + "provider": "apt", + "packages": ["build-helper"], + "policy": "distribution", + "phase": "build-only", + "platforms": ["linux/arm64"], + }, + "cli": {"provider": "npm", "package": "@fixture/cli"}, + } + (tools / "providers.json").write_text(json.dumps(providers)) + selection = { + "native": ["uv"], + "python": ["pre-commit"], + "apt": ["regular", "minimal"], + "build-apt": ["build-helper"], + "npm": ["cli"], + } + (tools / "environments.json").write_text(json.dumps({"ci": selection})) + destination = root / "bin" + tool_dir = root / "python tools" + log = root / "calls.jsonl" + environment = dict( + os.environ, + PATH=str(fake_bin) + os.pathsep + os.environ["PATH"], + SCORE_TEST_LOG=str(log), + ) + for phase in ("build", "runtime", "cleanup"): + result = self._run( + tools, + root, + "ci", + destination, + "--phase", + phase, + "--tool-dir", + str(tool_dir), + env=environment, + ) + self.assertEqual(result.returncode, 0, result.stderr) + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual( + [(c["command"], c["args"]) for c in calls], + [ + ("apt-get", ["install", "-y", "build-helper"]), + ("apt-get", ["install", "-y", "regular=1:2.3*"]), + ( + "apt-get", + [ + "install", + "-y", + "--no-install-recommends", + "--fix-broken", + "minimal", + ], + ), + ("uv", ["tool", "install", "--force", "pre-commit==4.7.0"]), + ("npm", ["install", "-g", "@fixture/cli"]), + ("apt-get", ["remove", "--purge", "-y", "build-helper"]), + ], + ) + self.assertEqual(calls[3]["bin_dir"], str(destination)) + self.assertEqual(calls[3]["tool_dir"], str(tool_dir)) + # Repository setup can apply npm in the user's environment while + # applying system tools in a separate privileged invocation. + log.unlink() + result = self._run( + tools, root, "ci", destination, "--providers", "npm", env=environment + ) + self.assertEqual(result.returncode, 0, result.stderr) + npm_only = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual( + [(c["command"], c["args"]) for c in npm_only], + [("npm", ["install", "-g", "@fixture/cli"])], + ) + # Deselecting an entire provider group changes the actual setup, + # rather than merely updating the inventory. + (tools / "environments.json").write_text( + json.dumps({"ci": {"apt": ["minimal"]}}) + ) + log.unlink() + result = self._run(tools, root, "ci", destination, env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + remaining = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual([c["command"] for c in remaining], ["apt-get"]) + self.assertEqual(remaining[0]["args"][-1], "minimal") + + def test_python_dependency_and_build_only_phase_are_validated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "environments.json" + with patch.object(catalog, "ENVIRONMENTS_PATH", path): + for data, message in ( + ({"ci": {"python": ["pre-commit"]}}, "requires 'uv'"), + ({"ci": {"apt": ["zstd"]}}, "Wrong installation phase"), + ( + {"ci": {"setup": ["codeql_coding_standards"]}}, + "requires 'codeql'", + ), + ): + with self.subTest(data=data): + path.write_text(json.dumps(data)) + with self.assertRaisesRegex(SystemExit, message): + catalog.load_environments() + + def test_special_setup_uses_membership_and_owned_bundle_items(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + installer = tools / "internal/devcontainer/install.py" + + def run(*args): + return subprocess.run( + [sys.executable, "-I", "-B", str(installer), *args], + cwd=root, + capture_output=True, + text=True, + ) + + self.assertEqual(run("selected", "s-core-local", "codeql").returncode, 0) + selection_path = tools / "environments.json" + selections = json.loads(selection_path.read_text()) + selections["s-core-local"]["setup"] = [] + selection_path.write_text(json.dumps(selections)) + self.assertEqual(run("selected", "s-core-local", "codeql").returncode, 1) + packs = run("items", "codeql_coding_standards", "packs") + self.assertEqual(packs.returncode, 0, packs.stderr) + definitions = json.loads((tools / "providers.json").read_text()) + self.assertEqual( + packs.stdout.splitlines(), + definitions["codeql_coding_standards"]["packs"], + ) + + def test_shell_metadata_keeps_command_substitution_as_literal_data(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + marker = root / "must-not-exist" + value = "$(touch " + str(marker) + ") 'quoted value'" + providers = {"fixture": {"provider": "release", "version": value}} + (tools / "providers.json").write_text(json.dumps(providers)) + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(tools / "internal/devcontainer/install.py"), + "shell-metadata", + ], + cwd=root, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + check = subprocess.run( + [ + "bash", + "-c", + 'eval "$1"; test "$fixture_version" = "$2"', + "metadata-test", + result.stdout, + value, + ], + cwd=root, + capture_output=True, + text=True, + ) + self.assertEqual(check.returncode, 0, check.stderr) + self.assertFalse(marker.exists()) + + def _copy_tools(self, root: Path) -> Path: + tools = root / "copied-tools" + shutil.copytree(TOOLS_DIR, tools, ignore=shutil.ignore_patterns("__pycache__")) + return tools + + def _run( + self, + tools: Path, + cwd: Path, + environment: str, + destination: Path, + *arguments, + env=None, + ): + return subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(tools / "internal/devcontainer/install.py"), + "install-environment", + environment, + "--destination", + str(destination), + "--os", + "linux", + "--cpu", + "arm64", + *arguments, + ], + cwd=cwd, + env=env, + capture_output=True, + text=True, + ) + + +if __name__ == "__main__": + unittest.main() From ab2964c3328154e3b9a3c1c57ae16e65c21a8111 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Tue, 3 Nov 2026 14:48:08 +0100 Subject: [PATCH 2/6] refactor(tools): isolate declared setup and clarify provider policies --- .../.devcontainer/bazel-feature/install.sh | 48 +------- .../bazel-feature/tests/test_default.sh | 2 +- .../fix_config_volume_ownership.sh | 12 +- .../.devcontainer/s-core-local/install.sh | 106 +---------------- .../s-core-local/tests/test_default.sh | 8 +- tools/environments.json | 2 +- tools/internal/README.md | 46 ++++++-- tools/internal/devcontainer/install.py | 46 ++++++-- tools/internal/devcontainer/setup/bazel.sh | 32 +++++ .../setup/bazel_compile_commands.sh | 32 +++++ tools/internal/devcontainer/setup/codeql.sh | 77 ++++++++++++ .../java.sh} | 10 +- tools/providers.json | 27 +++-- tools/tests/environment_test.py | 110 ++++++++++++++++++ 14 files changed, 366 insertions(+), 192 deletions(-) create mode 100755 tools/internal/devcontainer/setup/bazel.sh create mode 100755 tools/internal/devcontainer/setup/bazel_compile_commands.sh create mode 100755 tools/internal/devcontainer/setup/codeql.sh rename tools/internal/devcontainer/{load_feature_versions.sh => setup/java.sh} (65%) diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh index 29be9982..264c5fe9 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh @@ -28,57 +28,13 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe # used by apt-get only inside this script export DEBIAN_FRONTEND=noninteractive -# Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh - -ARCHITECTURE=$(dpkg --print-architecture) - apt-get update -# INSTALL CONTAINER BUILD DEPENDENCIES -# Container build dependencies are not pinned, since they are removed anyway after container creation. +# Selection metadata owns the tool list; the installer runs any declared setup. /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase build - -# Lockfile-managed Bazel tooling /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature - -if /usr/local/share/score-tools/internal/devcontainer/install.py selected bazel-feature bazel; then - # Bazelisk + Bazel - ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel - - # Pre-install a fixed Bazel version, setup the bash command completion - export USE_BAZEL_VERSION=${bazel_version} - # Bazelisk caches the release for the development user selected by Dev Containers. - su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" - mkdir -p /etc/bash_completion.d - mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash - sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" - -fi - -# Configure Bazel to use system trust store for SSL/TLS connections -# This is required for corporate environments with custom CA certificates -echo 'startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit' >> /etc/bazel.bazelrc - -if /usr/local/share/score-tools/internal/devcontainer/install.py selected bazel-feature bazel_compile_commands; then - # Code completion for C++ code of Bazel projects - # (see https://github.com/kiron1/bazel-compile-commands) - source /etc/lsb-release - curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb - # Extract correct sha256 for current DISTRIB_CODENAME and check - SHA256SUM="${bazel_compile_commands_amd64_sha256}" - if [ "${ARCHITECTURE}" = "arm64" ]; then - SHA256SUM="${bazel_compile_commands_arm64_sha256}" - fi - echo "${SHA256SUM} /tmp/bazel-compile-commands.deb" | sha256sum -c - || exit 1 - apt-get install -y --no-install-recommends --fix-broken /tmp/bazel-compile-commands.deb - rm /tmp/bazel-compile-commands.deb - -fi - -# Cleanup -# REMOVE CONTAINER BUILD DEPENDENCIES /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase cleanup + apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh index 23af6254..02e08a6b 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh @@ -16,7 +16,7 @@ set -euo pipefail # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" bazelisk_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version bazelisk)" buildifier_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version buildifier)" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh index ea3362df..4ce7998d 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh @@ -36,6 +36,12 @@ ensure_owner() { fi } -ensure_owner "/home/vscode/.config/opencode" -ensure_owner "/home/vscode/.local/share/opencode" -ensure_owner "/home/vscode/.config/gh" +# Docker creates missing parents of nested volume mounts as root. These are +# user-owned XDG directories; writable mount roots alone do not let tools create +# sibling state/cache directories. Change only the parents, not their contents. +sudo install -d -o "$(id -un)" -g "$(id -gn)" \ + "${HOME}/.config" "${HOME}/.local" "${HOME}/.local/share" + +ensure_owner "${HOME}/.config/opencode" +ensure_owner "${HOME}/.local/share/opencode" +ensure_owner "${HOME}/.config/gh" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh index c3bcab63..a196bedd 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh @@ -12,9 +12,6 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# shellcheck disable=SC2016 -# Variables in strings with single quotes are expected to be evaluated later by `eval` - set -euo pipefail # Copy feature sources and tests to expected location @@ -30,113 +27,16 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe # used by apt-get only inside this script export DEBIAN_FRONTEND=noninteractive -# Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh - -ARCHITECTURE=$(dpkg --print-architecture) -KERNEL=$(uname -s) - apt-get update -# Unminimize the image to include standard packages like man pages +# Restore standard OS documentation in the development image. bash -c "yes || true" | unminimize -# INSTALL CONTAINER BUILD DEPENDENCIES -# Container build dependencies are not pinned, since they are removed anyway after container creation. +# Selection metadata owns the tool list; the installer runs any declared setup. /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase build - -# One environment selection owns APT, native and Python membership. Provider -# installation runs before tool-specific lifecycle configuration below. /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local - -if /usr/local/share/score-tools/internal/devcontainer/install.py selected s-core-local openjdk_21; then - # OpenJDK 21, via APT - # Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) - JAVA_HOME="$(dirname "$(dirname "$(realpath "$(command -v javac)")")")" - export JAVA_HOME - echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh - -fi - -# Special setup also consumes environment membership, so removing a selection -# removes its installation rather than only changing the generated overview. -if /usr/local/share/score-tools/internal/devcontainer/install.py selected s-core-local codeql; then - # CodeQL - if [ "${ARCHITECTURE}" = "amd64" ]; then - VARIANT=linux64 - SHA256SUM="${codeql_amd64_sha256}" - elif [ "${ARCHITECTURE}" = "arm64" ]; then - if [ "${KERNEL}" = "Darwin" ]; then - VARIANT=osx64 - SHA256SUM="${codeql_arm64_sha256}" - else - echo "CodeQl unsupported architecture/os: ${ARCHITECTURE} on ${KERNEL}, skipping installation" - VARIANT=noinstall - fi - else - echo "Unsupported architecture: ${ARCHITECTURE} for CodeQL" - exit 1 - fi - - if [ "${VARIANT}" != "noinstall" ]; then - codeql_install_dir="/usr/local" - curl -L "https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-${VARIANT}.tar.zst" -o /tmp/codeql.tar.zst - echo "${SHA256SUM} /tmp/codeql.tar.zst" | sha256sum -c - || exit 1 - tar -I zstd -xf /tmp/codeql.tar.zst -C "${codeql_install_dir}" - ln -s "${codeql_install_dir}"/codeql/codeql /usr/local/bin/codeql - rm /tmp/codeql.tar.zst - export CODEQL_HOME=${codeql_install_dir}/codeql - echo "export CODEQL_HOME=${codeql_install_dir}/codeql" > /etc/profile.d/codeql.sh - - if /usr/local/share/score-tools/internal/devcontainer/install.py selected s-core-local codeql_coding_standards; then - codeql_packs="$(/usr/local/share/score-tools/internal/devcontainer/install.py items codeql_coding_standards packs)" - while IFS= read -r pack; do - codeql pack download "${pack}@${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - done <<< "${codeql_packs}" - fi - - # slim down codeql bundle (1.7GB -> 1.1 GB) by removing unnecessary language extractors and qlpacks - codeql_purge_dirs=( - "${codeql_install_dir}/codeql/csharp" - "${codeql_install_dir}/codeql/go" - "${codeql_install_dir}/codeql/java" - "${codeql_install_dir}/codeql/javascript" - "${codeql_install_dir}/codeql/python" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/swift-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/swift-queries" - "${codeql_install_dir}/codeql/ruby" - "${codeql_install_dir}/codeql/swift" - ) - for dir in "${codeql_purge_dirs[@]}"; do - if [ -d "${dir}" ]; then - rm -rf "${dir}" - fi - done - fi - -fi - -# Cleanup -# REMOVE CONTAINER BUILD DEPENDENCIES /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup + apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh index 19dd6df1..00edfb26 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh @@ -19,7 +19,7 @@ ARCHITECTURE=$(dpkg --print-architecture) KERNEL=$(uname -s) # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" shellcheck_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version shellcheck)" ruff_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version ruff)" @@ -66,8 +66,10 @@ check "validate pytest is working" bash -c "pytest --version" check "validate pylint is working" bash -c "pylint --version" # OpenJDK -check "validate java is working and has the correct version" bash -c "java -version 2>&1 | grep '${openjdk_21_version}'" -check "validate JAVA_HOME is set correctly" bash -c "echo ${JAVA_HOME} | xargs readlink -f | grep \"java-21-openjdk\"" +check "validate java is working and has the correct version" bash -c "java -version 2>&1 | grep '${openjdk_version}'" +# Expansion belongs to the child shell that performs the check. +# shellcheck disable=SC2016 +check "validate JAVA_HOME matches javac" bash -c 'test "$(readlink -f "$JAVA_HOME")" = "$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"' # ruff check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_lockfile_version}'" diff --git a/tools/environments.json b/tools/environments.json index 3059df30..d1b7c0cb 100644 --- a/tools/environments.json +++ b/tools/environments.json @@ -24,7 +24,7 @@ "git", "git_lfs", "gh", - "openjdk_21", + "openjdk", "valgrind", "ripgrep", "lcov", diff --git a/tools/internal/README.md b/tools/internal/README.md index 9e7653fa..f56e91ea 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -109,9 +109,9 @@ of host Python. APT `distribution` policies intentionally follow the distribution, `prefix` constraints include Debian epochs, and `package-name` selects a versioned package name (for example `python3.12`). Compatibility expectations for Git, Git LFS and -gh remain separate from their unpinned APT requests. The shell metadata bridge -quotes catalog data and uses the image's standard-library Python; it does not -download another parser. +gh remain separate from their unpinned APT requests. The installer passes version variables directly to its setup subprocesses. +Feature tests use `shell-metadata` to read quoted assignments in their own shell; +there is no separate loader script or downloaded parser. The catalogs have distinct ownership: `devcontainer-lock.json` records external DevContainer features, `python_tools.bzl` records Python package releases, and @@ -148,14 +148,14 @@ Each selection groups its tools by installation method: | `apt` | `providers.json` | apt, preserving package/version policy and recommendations | | `build-apt` | `providers.json` | apt during build; removed during cleanup | | `npm` | `providers.json` | npm in the appropriate user's environment | -| `setup` | `providers.json` | The feature's existing special setup, gated by membership | +| `setup` | `providers.json` | Setup scripts declared by `setup_script`, gated by membership | For example, the local feature applies its selected groups during setup: ```console $ python3 tools/internal/devcontainer/install.py install-environment s-core-local --phase build $ python3 tools/internal/devcontainer/install.py install-environment s-core-local -# Feature-specific configuration follows, then temporary packages are removed: +# Declared setup runs during installation, then temporary packages are removed: $ python3 tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup ``` @@ -173,10 +173,38 @@ dependencies and unsupported platform artifacts fail before installation. CodeQL's existing Linux ARM64 omission is explicit. Download or installation failure can still leave earlier tools installed; application is not transactional. -Feature scripts query `selected ENVIRONMENT TOOL` before special setup such as -JAVA_HOME, CodeQL or Bazel preinstallation. CodeQL pack names come from the owned -bundle through `items`, not a second list in the script. Removing membership stops -new setup from offering the tool; it does not uninstall existing host copies. +Feature entry scripts only apply the selection's build, runtime and cleanup +phases. Tool-specific steps live in `internal/devcontainer/setup/`; definitions +name the required script with `setup_script`. For example, `openjdk` selects an +APT package and declares Java home configuration. Its name carries no version; +`packages` and `version` select the actual OpenJDK release. + +The installer gives setup scripts version variables directly as their process +environment. Python can set a child's environment, but cannot change the shell +that invoked it. Java and CodeQL setup therefore also write profile files for +later terminals. The feature tests read quoted `shell-metadata` output directly. + +Setup scripts can query `selected ENVIRONMENT TOOL` for optional bundle members. +CodeQL pack names come from the owned bundle through `items`, not a second list. +CodeQL declares `zstd` as a build dependency, so missing decompression support is +rejected before the build phase. Removing membership stops new setup from +offering the tool; it does not uninstall existing host copies. + +For APT definitions, `packages` and `platforms` are independent lists. Every +listed package is installed on each supported host; apt selects its host build. +For example, both QEMU guest emulators are installed on both Linux host CPUs. +Their order does not map packages to architectures. + +A distribution policy is not an exact pin: Black and the other APT-provided +Python CLIs retain the current Ubuntu version selection. Changing their provider +to pinned uv distributions requires explicit version choices and removal of the +APT copies. It is a separate change from recording existing selections. + +`ca-certificates-java` builds the Java trust store from system certificates. +Pinning that helper alone does not freeze the trust data or imported corporate +CAs. These continue to receive distribution updates. Reproducible base/APT and +certificate snapshots, and rebuilding releases for trust updates, are outside +this change's guarantee. This file covers the direct tools managed by our three setup scripts. Base-image contents, external Dev Container features (such as LLVM, common-utils and DinD), diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index 8ad62e43..c5f60cf0 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -268,7 +268,7 @@ def _cmd_install(args: argparse.Namespace) -> int: def _cmd_install_environment(args: argparse.Namespace) -> int: - """Apply selected provider groups; feature scripts retain special setup.""" + """Install the tools selected for one environment.""" selection = load_environments().get(args.environment) if selection is None: raise SystemExit(f"Unknown tool environment '{args.environment}'") @@ -290,6 +290,18 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: ] and system not in definition.get("omissions", {}): raise SystemExit(f"Tool '{name}' is unsupported on {system}") + setup_scripts = [] + setup_root = Path(__file__).resolve().parent / "setup" + for name in selection.get("apt", []) + selection.get("setup", []): + definition = definitions[name] + script = definition.get("setup_script") + if script is None: + continue + if Path(script).name != script or not (setup_root / script).is_file(): + raise SystemExit(f"Invalid setup script for '{name}': {script}") + if system not in definition.get("omissions", {}): + setup_scripts.append(setup_root / script) + if args.phase != "runtime": packages = [ package @@ -325,13 +337,20 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: subprocess.run( ["npm", "install", "-g", definitions[name]["package"]], check=True ) + if "setup" in args.providers and setup_scripts: + environment = {**os.environ, **_metadata_values()} + environment["SCORE_TOOL_ENVIRONMENT"] = args.environment + environment["SCORE_TOOLS_INSTALLER"] = str(Path(__file__).resolve()) + for script in setup_scripts: + subprocess.run(["bash", str(script)], check=True, env=environment) return 0 -def _cmd_shell_metadata(args: argparse.Namespace) -> int: - """Emit quoted data for the provider-specific feature setup, never shell code from definitions.""" +def _metadata_values() -> dict[str, str]: + """Collect version variables for setup scripts and feature tests.""" import re - import shlex + + values: dict[str, str] = {} def emit(name, value): if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name): @@ -340,7 +359,7 @@ def emit(name, value): for key, child in value.items(): emit(name + "_" + key, child) elif isinstance(value, str): - print("export " + name + "=" + shlex.quote(value)) + values[name] = value for name, definition in load_providers().items(): # Export only lifecycle setup inputs. Inventory metadata can use keys @@ -355,17 +374,26 @@ def emit(name, value): # Python distributions share their pins with Bazel and container setup. for name, definition in load_python_tools().items(): emit(name.replace("-", "_"), definition) + return values + + +def _cmd_shell_metadata(args: argparse.Namespace) -> int: + """Print quoted variable assignments for the feature tests.""" + import shlex + + for name, value in _metadata_values().items(): + print("export " + name + "=" + shlex.quote(value)) return 0 def _cmd_selected(args: argparse.Namespace) -> int: - """Return whether a tool is selected, for provider-specific lifecycle setup.""" + """Exit with 0 if the environment selects this tool, or 1 if it does not.""" selections = load_environments()[args.environment] return 0 if any(args.tool in names for names in selections.values()) else 1 def _cmd_items(args: argparse.Namespace) -> int: - """Expose a provider-owned bundle list to lifecycle setup and verification.""" + """Print a list from a tool definition, one item per line.""" value = load_providers()[args.tool][args.field] if ( not isinstance(value, list) @@ -419,8 +447,8 @@ def _build_parser() -> argparse.ArgumentParser: environment_parser.add_argument( "--providers", nargs="+", - choices=("apt", "native", "python", "npm"), - default=("apt", "native", "python", "npm"), + choices=("apt", "native", "python", "npm", "setup"), + default=("apt", "native", "python", "npm", "setup"), help="Apply only these provider groups, e.g. npm as the development user.", ) environment_parser.add_argument( diff --git a/tools/internal/devcontainer/setup/bazel.sh b/tools/internal/devcontainer/setup/bazel.sh new file mode 100755 index 00000000..58da5087 --- /dev/null +++ b/tools/internal/devcontainer/setup/bazel.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# install.py supplies catalog version variables in the process environment. +# shellcheck disable=SC2154 +set -euo pipefail + +# Bazelisk + Bazel +ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel + +# Pre-install a fixed Bazel version, setup the bash command completion +export USE_BAZEL_VERSION=${bazel_version} +# Bazelisk caches the release for the development user selected by Dev Containers. +su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" +mkdir -p /etc/bash_completion.d +mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash +sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" + +# Configure Bazel to use system trust store for SSL/TLS connections +# This is required for corporate environments with custom CA certificates +echo 'startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit' >> /etc/bazel.bazelrc diff --git a/tools/internal/devcontainer/setup/bazel_compile_commands.sh b/tools/internal/devcontainer/setup/bazel_compile_commands.sh new file mode 100755 index 00000000..63a88efc --- /dev/null +++ b/tools/internal/devcontainer/setup/bazel_compile_commands.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# install.py supplies catalog version variables in the process environment. +# shellcheck disable=SC2154 +set -euo pipefail + +ARCHITECTURE=$(dpkg --print-architecture) + +# Code completion for C++ code of Bazel projects +# (see https://github.com/kiron1/bazel-compile-commands) +source /etc/lsb-release +curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb +# Extract correct sha256 for current DISTRIB_CODENAME and check +SHA256SUM="${bazel_compile_commands_amd64_sha256}" +if [ "${ARCHITECTURE}" = "arm64" ]; then + SHA256SUM="${bazel_compile_commands_arm64_sha256}" +fi +echo "${SHA256SUM} /tmp/bazel-compile-commands.deb" | sha256sum -c - || exit 1 +apt-get install -y --no-install-recommends --fix-broken /tmp/bazel-compile-commands.deb +rm /tmp/bazel-compile-commands.deb diff --git a/tools/internal/devcontainer/setup/codeql.sh b/tools/internal/devcontainer/setup/codeql.sh new file mode 100755 index 00000000..307e2386 --- /dev/null +++ b/tools/internal/devcontainer/setup/codeql.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# install.py supplies catalog version variables in the process environment. +# shellcheck disable=SC2154 +set -euo pipefail + +ARCHITECTURE=$(dpkg --print-architecture) +if [ "${ARCHITECTURE}" != "amd64" ]; then + echo "No CodeQL bundle selected for Linux ${ARCHITECTURE}" >&2 + exit 1 +fi +VARIANT=linux64 +SHA256SUM="${codeql_amd64_sha256}" + +codeql_install_dir="/usr/local" +curl -L "https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-${VARIANT}.tar.zst" -o /tmp/codeql.tar.zst +echo "${SHA256SUM} /tmp/codeql.tar.zst" | sha256sum -c - || exit 1 +tar -I zstd -xf /tmp/codeql.tar.zst -C "${codeql_install_dir}" +ln -s "${codeql_install_dir}"/codeql/codeql /usr/local/bin/codeql +rm /tmp/codeql.tar.zst +export CODEQL_HOME=${codeql_install_dir}/codeql +echo "export CODEQL_HOME=${codeql_install_dir}/codeql" > /etc/profile.d/codeql.sh + +if "${SCORE_TOOLS_INSTALLER}" selected "${SCORE_TOOL_ENVIRONMENT}" codeql_coding_standards; then + codeql_packs="$("${SCORE_TOOLS_INSTALLER}" items codeql_coding_standards packs)" + while IFS= read -r pack; do + codeql pack download "${pack}@${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" + done <<< "${codeql_packs}" +fi + +# slim down codeql bundle (1.7GB -> 1.1 GB) by removing unnecessary language extractors and qlpacks +codeql_purge_dirs=( + "${codeql_install_dir}/codeql/csharp" + "${codeql_install_dir}/codeql/go" + "${codeql_install_dir}/codeql/java" + "${codeql_install_dir}/codeql/javascript" + "${codeql_install_dir}/codeql/python" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/swift-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/swift-queries" + "${codeql_install_dir}/codeql/ruby" + "${codeql_install_dir}/codeql/swift" +) +for dir in "${codeql_purge_dirs[@]}"; do + if [ -d "${dir}" ]; then + rm -rf "${dir}" + fi +done diff --git a/tools/internal/devcontainer/load_feature_versions.sh b/tools/internal/devcontainer/setup/java.sh similarity index 65% rename from tools/internal/devcontainer/load_feature_versions.sh rename to tools/internal/devcontainer/setup/java.sh index 74e242f6..f8e694e3 100755 --- a/tools/internal/devcontainer/load_feature_versions.sh +++ b/tools/internal/devcontainer/setup/java.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # ******************************************************************************* # Copyright (c) 2026 Contributors to the Eclipse Foundation # @@ -15,8 +14,7 @@ set -euo pipefail -# Shared definitions are literal data. The bridge quotes values and validates -# variable names before emitting exports for provider-specific shell setup. -score_feature_metadata="$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -eval "${score_feature_metadata}" -unset score_feature_metadata +# Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) +JAVA_HOME="$(dirname "$(dirname "$(realpath "$(command -v javac)")")")" +export JAVA_HOME +echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh diff --git a/tools/providers.json b/tools/providers.json index 22a0d9ba..0f5516c1 100644 --- a/tools/providers.json +++ b/tools/providers.json @@ -19,7 +19,8 @@ "policy": "prefix", "platforms": ["linux/x86_64", "linux/arm64"], "version": "1:8.2.2", - "install_recommends": false + "install_recommends": false, + "description": "Both ARM and x86 guest emulators are installed on each supported host platform." }, "sshpass": { "provider": "apt", @@ -56,12 +57,13 @@ "platforms": ["linux/x86_64", "linux/arm64"], "compatibility_version": "2.45.0" }, - "openjdk_21": { + "openjdk": { "provider": "apt", "packages": ["openjdk-21-jdk-headless"], "policy": "prefix", "platforms": ["linux/x86_64", "linux/arm64"], - "version": "21.0.12" + "version": "21.0.12", + "setup_script": "java.sh" }, "valgrind": { "provider": "apt", @@ -122,7 +124,8 @@ "provider": "apt", "packages": ["black"], "policy": "distribution", - "platforms": ["linux/x86_64", "linux/arm64"] + "platforms": ["linux/x86_64", "linux/arm64"], + "description": "Ubuntu-provided Black; distribution policy does not pin an exact version." }, "python3-yapf": { "provider": "apt", @@ -176,7 +179,8 @@ "provider": "apt", "packages": ["ca-certificates-java"], "policy": "distribution", - "platforms": ["linux/x86_64", "linux/arm64"] + "platforms": ["linux/x86_64", "linux/arm64"], + "description": "Populate the Java trust store from system CA certificates; follow distribution updates." }, "man-db": { "provider": "apt", @@ -216,12 +220,11 @@ "amd64": { "sha256": "6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545" }, - "arm64": { - "sha256": "810a1fff48aeb081be754e46e255edd90d8695966e78431a65edb00e9e6cc399" - }, "omissions": { "linux/arm64": "The selected bundle has no supported Linux arm64 artifact" - } + }, + "setup_script": "codeql.sh", + "requires": ["zstd"] }, "codeql_coding_standards": { "provider": "release", @@ -244,7 +247,8 @@ "policy": "exact", "platforms": ["linux/x86_64", "linux/arm64"], "version": "8.6.0", - "requires": ["bazelisk"] + "requires": ["bazelisk"], + "setup_script": "bazel.sh" }, "bazel_compile_commands": { "provider": "release", @@ -256,7 +260,8 @@ }, "arm64": { "sha256": "d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88" - } + }, + "setup_script": "bazel_compile_commands.sh" }, "apt-transport-https": { "provider": "apt", diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py index 4274a96f..0b206efd 100644 --- a/tools/tests/environment_test.py +++ b/tools/tests/environment_test.py @@ -382,6 +382,116 @@ def test_shell_metadata_keeps_command_substitution_as_literal_data(self) -> None self.assertEqual(check.returncode, 0, check.stderr) self.assertFalse(marker.exists()) + def test_missing_codeql_build_dependency_is_rejected_before_build(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + path = tools / "environments.json" + selections = json.loads(path.read_text()) + selections["s-core-local"]["build-apt"].remove("zstd") + path.write_text(json.dumps(selections)) + marker = root / "apt-was-called" + fake_bin = root / "fake-bin" + fake_bin.mkdir() + apt = fake_bin / "apt-get" + apt.write_text(f"#!/bin/sh\ntouch '{marker}'\n") + apt.chmod(0o755) + environment = dict( + os.environ, PATH=str(fake_bin) + os.pathsep + os.environ["PATH"] + ) + result = self._run( + tools, + root, + "s-core-local", + root / "bin", + "--phase", + "build", + env=environment, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Tool 'codeql' requires 'zstd'", result.stderr) + self.assertFalse(marker.exists()) + + def test_declared_setup_receives_metadata_without_changing_the_parent_shell( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + provider = { + "provider": "release", + "version": "a 'literal' value", + "platforms": ["linux/arm64"], + "setup_script": "fixture.sh", + } + providers_path = tools / "providers.json" + providers_path.write_text(json.dumps({"fixture": provider})) + selections_path = tools / "environments.json" + selections_path.write_text(json.dumps({"ci": {"setup": ["fixture"]}})) + script = tools / "internal/devcontainer/setup/fixture.sh" + script.write_text( + 'printf "%s\\n" "$fixture_version" "$SCORE_TOOL_ENVIRONMENT" > "$SCORE_TEST_OUTPUT"\n' + ) + output = root / "setup-output" + environment = dict(os.environ, SCORE_TEST_OUTPUT=str(output)) + result = self._run(tools, root, "ci", root / "bin", env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + output.read_text().splitlines(), [provider["version"], "ci"] + ) + output.unlink() + # A platform omission must skip the setup script as well as accept + # the selection; deselection must also prevent running the script. + provider["platforms"] = ["linux/x86_64"] + provider["omissions"] = {"linux/arm64": "Fixture has no ARM artifact"} + providers_path.write_text(json.dumps({"fixture": provider})) + result = self._run(tools, root, "ci", root / "bin", env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(output.exists()) + selections_path.write_text(json.dumps({"ci": {"setup": []}})) + result = self._run(tools, root, "ci", root / "bin", env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(output.exists()) + + def test_volume_setup_prepares_writable_user_parent_directories(self) -> None: + # A nested Docker volume mount may create .local as root. Repair the + # parent directories explicitly instead of relying on a user tool install. + script = ( + TOOLS_DIR.parent + / "src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh" + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home = root / "home" + fake_bin = root / "bin" + fake_bin.mkdir() + for name in (".config/opencode", ".config/gh", ".local/share/opencode"): + (home / name).mkdir(parents=True, exist_ok=True) + sudo = fake_bin / "sudo" + sudo.write_text( + '#!/bin/sh\nprintf "%s\\n" "$@" >> "$SCORE_TEST_LOG"\nexec "$@"\n' + ) + sudo.chmod(0o755) + log = root / "calls" + result = subprocess.run( + ["bash", str(script)], + env=dict( + os.environ, + HOME=str(home), + PATH=str(fake_bin) + os.pathsep + os.environ["PATH"], + SCORE_TEST_LOG=str(log), + ), + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + calls = log.read_text().splitlines() + self.assertEqual(calls[0], "install") + self.assertIn("-o", calls) + self.assertIn("-g", calls) + for name in (".config", ".local", ".local/share"): + self.assertIn(str(home / name), calls) + def _copy_tools(self, root: Path) -> Path: tools = root / "copied-tools" shutil.copytree(TOOLS_DIR, tools, ignore=shutil.ignore_patterns("__pycache__")) From 0b8e858dbb6b16aa71f74dea97592a634ac86e1d Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Tue, 3 Nov 2026 20:58:21 +0100 Subject: [PATCH 3/6] docs(tools): clarify provisioning comments and docstrings --- .devcontainer/post_create_command.sh | 4 +-- .../.devcontainer/bazel-feature/install.sh | 2 +- .../fix_config_volume_ownership.sh | 6 ++-- .../.devcontainer/s-core-local/install.sh | 2 +- tools/internal/catalog.py | 20 ++++++++--- tools/internal/devcontainer/install.py | 35 ++++++++++++------- tools/internal/devcontainer/setup/bazel.sh | 12 +++---- .../setup/bazel_compile_commands.sh | 6 ++-- tools/internal/devcontainer/setup/codeql.sh | 4 +-- tools/internal/devcontainer/setup/java.sh | 3 +- tools/internal/sync_readme.py | 5 +-- tools/tests/environment_test.py | 22 ++++++------ 12 files changed, 71 insertions(+), 50 deletions(-) diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index d6f60d7d..21573a72 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -16,8 +16,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" REPOSITORY_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -P)" -# npm's global directory belongs to the development user in the Node.js base -# image. Keep npm setup in that user's environment; system tools require root. +# This repo uses a Node.js image whose global npm directory belongs to the +# development user. Install npm packages as that user; system tools need sudo. "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer --providers npm sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer --providers native python diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh index 264c5fe9..b5a3ab19 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh @@ -30,7 +30,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update -# Selection metadata owns the tool list; the installer runs any declared setup. +# environments.json selects the tools; providers.json names their extra setup scripts. /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase build /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase cleanup diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh index 4ce7998d..beb5e230 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh @@ -36,9 +36,9 @@ ensure_owner() { fi } -# Docker creates missing parents of nested volume mounts as root. These are -# user-owned XDG directories; writable mount roots alone do not let tools create -# sibling state/cache directories. Change only the parents, not their contents. +# Docker may create .local and .local/share as root when mounting a volume +# below them. Give the user ownership of these parents too, so tools can create +# directories such as .local/state. Leave existing contents alone. sudo install -d -o "$(id -un)" -g "$(id -gn)" \ "${HOME}/.config" "${HOME}/.local" "${HOME}/.local/share" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh index a196bedd..aba3f6af 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh @@ -32,7 +32,7 @@ apt-get update # Restore standard OS documentation in the development image. bash -c "yes || true" | unminimize -# Selection metadata owns the tool list; the installer runs any declared setup. +# environments.json selects the tools; providers.json names their extra setup scripts. /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase build /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup diff --git a/tools/internal/catalog.py b/tools/internal/catalog.py index 4b663a9c..e18b8124 100644 --- a/tools/internal/catalog.py +++ b/tools/internal/catalog.py @@ -10,7 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Read pinned native and Python tool metadata without executing catalog code.""" +"""Read tool definitions and environment selections without running installation code.""" # pyright: reportAny=false, reportUnusedCallResult=false, reportExplicitAny=false @@ -240,7 +240,7 @@ def resolve_lockfile(tool: str, lockfile: str | None = None) -> str: def load_providers() -> dict: - """Read APT, npm and feature-specific setup definitions.""" + """Read and check installation details for APT, npm and tools with custom setup.""" providers = json.loads(PROVIDERS_PATH.read_text(encoding="utf-8")) if not isinstance(providers, dict): raise SystemExit("Tool providers must be a dictionary") @@ -283,7 +283,9 @@ def load_providers() -> dict: def all_definitions() -> dict: - """Combine metadata for reference validation without loading repository configs.""" + """Look up every defined tool by name, regardless of how it is installed. + + Only the copied tools/ directory is needed, so this also works inside images.""" tools = {} sources = ( ((name, {**data, "provider": "native"}) for name, _, data in _iter_catalog()), @@ -302,11 +304,13 @@ def all_definitions() -> dict: def load_environments() -> dict[str, dict[str, list[str]]]: - """Validate membership, provider categories and deliberately selected dependencies.""" + """Read environments.json and check tool names, installation groups and dependencies.""" data = json.loads(ENVIRONMENTS_PATH.read_text(encoding="utf-8")) if not isinstance(data, dict): raise SystemExit("Tool environments must be a dictionary") tools = all_definitions() + # Build-only packages use the same APT definitions as regular packages, + # but are removed once image setup is complete. categories = { "native": {"native"}, "python": {"python"}, @@ -342,6 +346,8 @@ def load_environments() -> dict[str, dict[str, list[str]]]: raise SystemExit( f"Wrong installation phase for '{tool}' in environment '{name}'" ) + # Dependencies can be in another group: CodeQL needs zstd during the + # build, while Python tools need uv from the native tool selection. selected = {tool for names in selection.values() for tool in names} for tool in selected: for dependency in tools[tool].get("requires", []): @@ -355,15 +361,19 @@ def load_environments() -> dict[str, dict[str, list[str]]]: def apt_requirements( selection: dict, definitions: dict, *, recommends: bool ) -> list[str]: - """Translate selected APT policies to requests, preserving Debian epochs.""" + """Build apt package arguments for the chosen recommendation setting. + + Keep version strings intact, including Debian's optional prefix such as "1:".""" requirements = [] for name in selection.get("apt", []): tool = definitions[name] if tool.get("install_recommends", True) != recommends: continue for package in tool["packages"]: + # Some releases are selected by package name, such as python3.12. if tool["policy"] == "package-name": package = package.format(version=tool["version"]) + # Include Ubuntu's package revisions, e.g. graphviz=2.42.2*. elif tool["policy"] == "prefix": package += "=" + tool["version"] + "*" requirements.append(package) diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index c5f60cf0..48c1e7bd 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -214,8 +214,8 @@ def _cmd_install(args: argparse.Namespace) -> int: """Download, verify, and install tools from the lockfile catalog.""" dest_dir = Path(args.destination) - # Resolve the complete selection before downloading or writing anything. - # Otherwise a missing tool/platform late in the list leaves a partial set. + # Check every requested tool has a download for this OS and CPU first. + # A bad name late in the list must not leave earlier tools installed. binaries: list[tuple[str, Binary]] = [] for tool in args.tools: lockfile = resolve_lockfile(tool, args.lockfile) @@ -268,14 +268,14 @@ def _cmd_install(args: argparse.Namespace) -> int: def _cmd_install_environment(args: argparse.Namespace) -> int: - """Install the tools selected for one environment.""" + """Install an environment's tools, or add/remove its temporary build packages.""" selection = load_environments().get(args.environment) if selection is None: raise SystemExit(f"Unknown tool environment '{args.environment}'") definitions = all_definitions() system = args.os + "/" + args.cpu - # Preflight the complete environment, even for a build/cleanup phase. A - # missing runtime artifact must be reported before installing build packages. + # Even the build phase checks tools needed by the finished image. Report + # missing downloads before installing packages used only during setup. for names in selection.values(): for name in names: definition = definitions[name] @@ -290,6 +290,8 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: ] and system not in definition.get("omissions", {}): raise SystemExit(f"Tool '{name}' is unsupported on {system}") + # Some tools need extra configuration, such as setting JAVA_HOME. Only + # setup scripts shipped next to this installer can be selected. setup_scripts = [] setup_root = Path(__file__).resolve().parent / "setup" for name in selection.get("apt", []) + selection.get("setup", []): @@ -302,6 +304,8 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: if system not in definition.get("omissions", {}): setup_scripts.append(setup_root / script) + # Build packages, such as zstd for unpacking CodeQL, are added before tool + # installation and removed after all setup steps have finished. if args.phase != "runtime": packages = [ package @@ -317,13 +321,14 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: subprocess.run(["apt-get", *action, *packages], check=True) return 0 - # QEMU deliberately omits recommendations. Other packages keep apt's normal - # recommendations, including the headers needed to build Python extensions. + # APT normally installs recommended extra packages. QEMU opts out; the + # other packages keep that default, including Python build dependencies. for recommends in (True, False) if "apt" in args.providers else (): packages = apt_requirements(selection, definitions, recommends=recommends) if packages: flags = [] if recommends else ["--no-install-recommends", "--fix-broken"] subprocess.run(["apt-get", "install", "-y", *flags, *packages], check=True) + # Install uv as a native tool before using it to install Python tools. if "native" in args.providers: args.tools = selection.get("native", []) args.lockfile = None @@ -337,6 +342,8 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: subprocess.run( ["npm", "install", "-g", definitions[name]["package"]], check=True ) + # Pass variables to the setup processes. Python cannot change the + # environment of the shell that launched this installer. if "setup" in args.providers and setup_scripts: environment = {**os.environ, **_metadata_values()} environment["SCORE_TOOL_ENVIRONMENT"] = args.environment @@ -347,7 +354,7 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: def _metadata_values() -> dict[str, str]: - """Collect version variables for setup scripts and feature tests.""" + """Turn definition fields into variables such as codeql_version for setup and tests.""" import re values: dict[str, str] = {} @@ -362,23 +369,25 @@ def emit(name, value): values[name] = value for name, definition in load_providers().items(): - # Export only lifecycle setup inputs. Inventory metadata can use keys - # such as linux/arm64 which deliberately are not shell identifiers. + # Use only version and checksum fields. Other data, such as platform + # names containing slashes, cannot be used as shell variable names. for field in ("version", "compatibility_version", "amd64", "arm64"): if field in definition: emit(name.replace("-", "_") + "_" + field, definition[field]) + # Unpinned APT tools can still have expected versions for feature tests. + # Give these expectations the same _version names as pinned tools. if "compatibility_version" in definition: emit( name.replace("-", "_") + "_version", definition["compatibility_version"] ) - # Python distributions share their pins with Bazel and container setup. + # Tests and Python installation read the same version definitions. for name, definition in load_python_tools().items(): emit(name.replace("-", "_"), definition) return values def _cmd_shell_metadata(args: argparse.Namespace) -> int: - """Print quoted variable assignments for the feature tests.""" + """Print shell-quoted exports so feature tests can read versions in their shell.""" import shlex for name, value in _metadata_values().items(): @@ -393,7 +402,7 @@ def _cmd_selected(args: argparse.Namespace) -> int: def _cmd_items(args: argparse.Namespace) -> int: - """Print a list from a tool definition, one item per line.""" + """Print a tool definition's list, such as CodeQL packs, one item per line.""" value = load_providers()[args.tool][args.field] if ( not isinstance(value, list) diff --git a/tools/internal/devcontainer/setup/bazel.sh b/tools/internal/devcontainer/setup/bazel.sh index 58da5087..dba36496 100755 --- a/tools/internal/devcontainer/setup/bazel.sh +++ b/tools/internal/devcontainer/setup/bazel.sh @@ -12,21 +12,21 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# install.py supplies catalog version variables in the process environment. +# install.py passes version and checksum variables from providers.json. # shellcheck disable=SC2154 set -euo pipefail -# Bazelisk + Bazel +# Let the bazel command use Bazelisk to select and download the requested version. ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel -# Pre-install a fixed Bazel version, setup the bash command completion +# Download the chosen Bazel release into the development user's cache and save +# shell completion for later terminals. export USE_BAZEL_VERSION=${bazel_version} -# Bazelisk caches the release for the development user selected by Dev Containers. +# Run as the development user so their first Bazel call can reuse this download. su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" mkdir -p /etc/bash_completion.d mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" -# Configure Bazel to use system trust store for SSL/TLS connections -# This is required for corporate environments with custom CA certificates +# Make Bazel trust the same certificates as the OS, including company CAs. echo 'startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit' >> /etc/bazel.bazelrc diff --git a/tools/internal/devcontainer/setup/bazel_compile_commands.sh b/tools/internal/devcontainer/setup/bazel_compile_commands.sh index 63a88efc..5b6a1a7e 100755 --- a/tools/internal/devcontainer/setup/bazel_compile_commands.sh +++ b/tools/internal/devcontainer/setup/bazel_compile_commands.sh @@ -12,17 +12,17 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# install.py supplies catalog version variables in the process environment. +# install.py passes version and checksum variables from providers.json. # shellcheck disable=SC2154 set -euo pipefail ARCHITECTURE=$(dpkg --print-architecture) -# Code completion for C++ code of Bazel projects +# Install the helper that writes compile_commands.json for editors using Bazel. # (see https://github.com/kiron1/bazel-compile-commands) source /etc/lsb-release curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb -# Extract correct sha256 for current DISTRIB_CODENAME and check +# Check the checksum for this CPU before installing the downloaded package. SHA256SUM="${bazel_compile_commands_amd64_sha256}" if [ "${ARCHITECTURE}" = "arm64" ]; then SHA256SUM="${bazel_compile_commands_arm64_sha256}" diff --git a/tools/internal/devcontainer/setup/codeql.sh b/tools/internal/devcontainer/setup/codeql.sh index 307e2386..42c76bfa 100755 --- a/tools/internal/devcontainer/setup/codeql.sh +++ b/tools/internal/devcontainer/setup/codeql.sh @@ -12,7 +12,7 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# install.py supplies catalog version variables in the process environment. +# install.py passes version and checksum variables from providers.json. # shellcheck disable=SC2154 set -euo pipefail @@ -40,7 +40,7 @@ if "${SCORE_TOOLS_INSTALLER}" selected "${SCORE_TOOL_ENVIRONMENT}" codeql_coding done <<< "${codeql_packs}" fi -# slim down codeql bundle (1.7GB -> 1.1 GB) by removing unnecessary language extractors and qlpacks +# Remove unused language tools and query packs to keep the image smaller. codeql_purge_dirs=( "${codeql_install_dir}/codeql/csharp" "${codeql_install_dir}/codeql/go" diff --git a/tools/internal/devcontainer/setup/java.sh b/tools/internal/devcontainer/setup/java.sh index f8e694e3..0d6e9e15 100755 --- a/tools/internal/devcontainer/setup/java.sh +++ b/tools/internal/devcontainer/setup/java.sh @@ -14,7 +14,8 @@ set -euo pipefail -# Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) +# Make JAVA_HOME available in later terminals so tools such as Bazel can find +# Java. Derive the path from the installed javac, not a release number. JAVA_HOME="$(dirname "$(dirname "$(realpath "$(command -v javac)")")")" export JAVA_HOME echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh diff --git a/tools/internal/sync_readme.py b/tools/internal/sync_readme.py index b38bfa05..b89edf90 100755 --- a/tools/internal/sync_readme.py +++ b/tools/internal/sync_readme.py @@ -11,7 +11,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Synchronize the documented tool table with the lockfile catalog.""" +"""Update the documented tool table and check environment selections.""" from __future__ import annotations @@ -67,7 +67,8 @@ def main(argv: list[str] | None = None) -> int: ) parser.parse_args(argv) - # Validate selection references alongside the catalog in the documentation hook. + # The pre-commit documentation check also catches unknown tools and missing + # dependencies in environments.json, even when the table does not change. load_environments() current = README_PATH.read_text(encoding="utf-8") table = _render_table(load_catalog_versions(), load_catalog_descriptions()) diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py index 0b206efd..ca4089ad 100644 --- a/tools/tests/environment_test.py +++ b/tools/tests/environment_test.py @@ -10,7 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Tool selections are portable and fail before applying an invalid set.""" +"""Check selected tools are installed and invalid selections fail before installation.""" from __future__ import annotations @@ -55,8 +55,8 @@ def test_rejects_invalid_selection_data_and_wrong_providers(self) -> None: catalog.load_environments() def test_copied_installer_applies_an_independent_ci_selection(self) -> None: - # A new image selection needs no changes to the installer or its caller. - # Local artifacts exercise actual download, checksum and install code. + # A new image can declare its own tool list without changing the + # installer. Local files exercise the real download/checksum/install path. with tempfile.TemporaryDirectory() as directory: root = Path(directory) tools = self._copy_tools(root) @@ -162,8 +162,8 @@ def test_complete_selection_preserves_provider_order_and_package_policies( tools = self._copy_tools(root) fake_bin = root / "fake-bin" fake_bin.mkdir() - # Real subprocesses record the apt/npm/uv boundary without changing - # host packages. uv itself is installed from a checked local artifact. + # Fake apt/npm/uv commands record what would be installed without + # changing host packages. uv itself comes from a checksum-checked file. recorder = ( f"#!{sys.executable}\n" "import json, os, sys\n" @@ -288,8 +288,8 @@ def test_complete_selection_preserves_provider_order_and_package_policies( [(c["command"], c["args"]) for c in npm_only], [("npm", ["install", "-g", "@fixture/cli"])], ) - # Deselecting an entire provider group changes the actual setup, - # rather than merely updating the inventory. + # Removing a group from the selection must stop its install commands, + # not just change the displayed tool list. (tools / "environments.json").write_text( json.dumps({"ci": {"apt": ["minimal"]}}) ) @@ -440,8 +440,8 @@ def test_declared_setup_receives_metadata_without_changing_the_parent_shell( output.read_text().splitlines(), [provider["version"], "ci"] ) output.unlink() - # A platform omission must skip the setup script as well as accept - # the selection; deselection must also prevent running the script. + # An explicitly unsupported platform must skip this setup script. + # Removing the tool from the selection must also stop it running. provider["platforms"] = ["linux/x86_64"] provider["omissions"] = {"linux/arm64": "Fixture has no ARM artifact"} providers_path.write_text(json.dumps({"fixture": provider})) @@ -454,8 +454,8 @@ def test_declared_setup_receives_metadata_without_changing_the_parent_shell( self.assertFalse(output.exists()) def test_volume_setup_prepares_writable_user_parent_directories(self) -> None: - # A nested Docker volume mount may create .local as root. Repair the - # parent directories explicitly instead of relying on a user tool install. + # Docker can create .local as root when mounting a volume below it. + # Its owner must be fixed so the user can create sibling state directories. script = ( TOOLS_DIR.parent / "src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh" From 6d87cb2a7652397985bd064e183e3f9e0e602e28 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 4 Nov 2026 11:43:31 +0100 Subject: [PATCH 4/6] refactor(tools): let installer manage setup phases and privileges --- .devcontainer/post_create_command.sh | 5 +- .../.devcontainer/bazel-feature/install.sh | 2 - .../bazel-feature/tests/test_default.sh | 9 +- .../fix_config_volume_ownership.sh | 4 +- .../.devcontainer/s-core-local/install.sh | 2 - .../s-core-local/tests/test_default.sh | 34 ++-- tools/internal/README.md | 25 ++- tools/internal/devcontainer/install.py | 94 ++++++++- tools/tests/environment_test.py | 189 ++++++++++++++++-- 9 files changed, 297 insertions(+), 67 deletions(-) diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index 21573a72..e09b98d3 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -16,10 +16,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" REPOSITORY_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -P)" -# This repo uses a Node.js image whose global npm directory belongs to the -# development user. Install npm packages as that user; system tools need sudo. -"${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer --providers npm -sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer --providers native python +"${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer # Hooks can only be registered after the catalogued executable is on PATH. pre-commit install diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh index b5a3ab19..3ac1d62d 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh @@ -31,9 +31,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update # environments.json selects the tools; providers.json names their extra setup scripts. -/usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase build /usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature -/usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature --phase cleanup apt-get autoremove -y apt-get clean diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh index 02e08a6b..d971b776 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh @@ -18,18 +18,15 @@ set -euo pipefail # Read tool versions + metadata into environment variables eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -bazelisk_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version bazelisk)" -buildifier_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version buildifier)" -starpls_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version starpls)" # Bazel-related tools ## This is the bazel version preinstalled in the devcontainer. ## A solid test would disable the network interface first to prevent a different version from being downloaded, ## but that requires CAP_NET_ADMIN, which is not yet added. export USE_BAZEL_VERSION=${bazel_version} -check "validate bazelisk is working and has the correct version" bash -c "bazelisk version | grep '${bazelisk_lockfile_version}'" +check "validate bazelisk is working and has the correct version" bash -c "bazelisk version | grep '${bazelisk_version}'" check "validate bazel is working and has the correct version" bash -c "bazel version | grep '${bazel_version}'" unset USE_BAZEL_VERSION -check "validate buildifier is working and has the correct version" bash -c "buildifier --version | grep '${buildifier_lockfile_version}'" -check "validate starpls is working and has the correct version" bash -c "starpls version | grep '${starpls_lockfile_version}'" +check "validate buildifier is working and has the correct version" bash -c "buildifier --version | grep '${buildifier_version}'" +check "validate starpls is working and has the correct version" bash -c "starpls version | grep '${starpls_version}'" check "validate bazel-compile-commands is working and has the correct version" bash -c "bazel-compile-commands --version 2>&1 | grep '${bazel_compile_commands_version}'" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh index beb5e230..87337d05 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh @@ -39,7 +39,9 @@ ensure_owner() { # Docker may create .local and .local/share as root when mounting a volume # below them. Give the user ownership of these parents too, so tools can create # directories such as .local/state. Leave existing contents alone. -sudo install -d -o "$(id -un)" -g "$(id -gn)" \ +# mkdir preserves existing directory permissions, including private mode 0700. +sudo mkdir -p "${HOME}/.config" "${HOME}/.local" "${HOME}/.local/share" +sudo chown "${current_user_group}" \ "${HOME}/.config" "${HOME}/.local" "${HOME}/.local/share" ensure_owner "${HOME}/.config/opencode" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh index aba3f6af..f5a6ec65 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh @@ -33,9 +33,7 @@ apt-get update bash -c "yes || true" | unminimize # environments.json selects the tools; providers.json names their extra setup scripts. -/usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase build /usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local -/usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup apt-get autoremove -y apt-get clean diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh index 00edfb26..ca88e87e 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh @@ -21,25 +21,13 @@ KERNEL=$(uname -s) # Read tool versions + metadata into environment variables eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -shellcheck_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version shellcheck)" -ruff_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version ruff)" -actionlint_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version actionlint)" -yamlfmt_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version yamlfmt)" -uv_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version uv)" -uvx_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version uvx)" -apm_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version apm)" -opencode_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version opencode)" - -# The shared catalog is the assertion source, so installation metadata and the -# executable exposed on PATH are checked against the same release. -pre_commit_catalog_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version pre-commit)" # Verify the system-wide uv installation exposes the catalogued release to the # non-root user running feature tests. -check "validate pre-commit is working and has the correct version" bash -c "pre-commit --version | grep '${pre_commit_catalog_version}'" +check "validate pre-commit is working and has the correct version" bash -c "pre-commit --version | grep '${pre_commit_version}'" # Common tooling -check "validate shellcheck is working and has the correct version" bash -c "shellcheck --version | grep '${shellcheck_lockfile_version}'" +check "validate shellcheck is working and has the correct version" bash -c "shellcheck --version | grep '${shellcheck_version}'" # For an unknown reason, dot -V reports on Ubuntu Noble a version 2.43.0, while the package has a different version. # Hence, we have to work around that. @@ -72,23 +60,23 @@ check "validate java is working and has the correct version" bash -c "java -vers check "validate JAVA_HOME matches javac" bash -c 'test "$(readlink -f "$JAVA_HOME")" = "$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"' # ruff -check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_lockfile_version}'" +check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_version}'" # actionlint -check "validate actionlint is working and has the correct version" bash -c "actionlint --version | grep '${actionlint_lockfile_version}'" +check "validate actionlint is working and has the correct version" bash -c "actionlint --version | grep '${actionlint_version}'" # yamlfmt -check "validate yamlfmt is working and has the correct version" bash -c "yamlfmt --version | grep '${yamlfmt_lockfile_version}'" +check "validate yamlfmt is working and has the correct version" bash -c "yamlfmt --version | grep '${yamlfmt_version}'" # apm -check "validate apm is working and has the correct version" bash -c "apm --version | grep '${apm_lockfile_version}'" +check "validate apm is working and has the correct version" bash -c "apm --version | grep '${apm_version}'" # opencode -check "validate opencode is working and has the correct version" bash -c "opencode --version | grep '${opencode_lockfile_version}'" +check "validate opencode is working and has the correct version" bash -c "opencode --version | grep '${opencode_version}'" # uv -check "validate uv is working and has the correct version" bash -c "uv --version | grep '${uv_lockfile_version}'" -check "validate uvx is working and has the correct version" bash -c "uvx --version | grep '${uvx_lockfile_version}'" +check "validate uv is working and has the correct version" bash -c "uv --version | grep '${uv_version}'" +check "validate uvx is working and has the correct version" bash -c "uvx --version | grep '${uvx_version}'" # pycairo build from source (requires pkg-config and libcairo2-dev) check "validate pycairo can be built and installed from source" bash -c "pip install --break-system-packages pycairo==1.29.0" @@ -97,6 +85,8 @@ check "validate pycairo can be built and installed from source" bash -c "pip ins check "validate gdb is working and has the correct version" bash -c "gdb --version | grep '${gdb_version}'" check "validate pkg-config is working and has the correct version" bash -c "pkg-config --version | grep '${pkg_config_version}'" check "validate gh is working and has the correct version" bash -c "gh --version | grep '${gh_version}'" +# Debian package versions may start with an epoch, e.g. 1:3.22.0. +# ${value#*:} removes that prefix; the executable reports only 3.22.0. check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version#*:}'" check "validate ripgrep is working and has the correct version" bash -c "rg --version | grep '${ripgrep_version}'" if [ "${ARCHITECTURE}" = "amd64" ] || { [ "${ARCHITECTURE}" = "arm64" ] && [ "${KERNEL}" = "Darwin" ]; }; then @@ -107,7 +97,7 @@ fi # lcov check "validate lcov is working and has the correct version" bash -c "lcov --version | grep '${lcov_version}'" -# Qemu target-related tools +# Qemu target-related tools (strip the Debian epoch here too). check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version#*:}'" check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version#*:}'" check "validate sshpass is working and has the correct version" bash -c "sshpass -V | grep '${sshpass_version}'" diff --git a/tools/internal/README.md b/tools/internal/README.md index f56e91ea..0ef85ee9 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -139,6 +139,14 @@ they are not newly designed tool groups. The published image combines the first two through its existing features. A future lean CI image can define its own selection based on the jobs it needs to run. This does not add another image. +Callers request the selection with one command, for example +`install.py install-environment repository-devcontainer`. The installer checks +the complete selection before starting installation. It uses `sudo` for system +installation when needed and keeps npm in the calling user's Node environment. +Image builds already running as root install directly. Native and Python tools +can also be installed without `sudo` when their destination directories are +writable by the caller. + Each selection groups its tools by installation method: | Group | Definition owner | Applied by | @@ -153,15 +161,16 @@ Each selection groups its tools by installation method: For example, the local feature applies its selected groups during setup: ```console -$ python3 tools/internal/devcontainer/install.py install-environment s-core-local --phase build $ python3 tools/internal/devcontainer/install.py install-environment s-core-local -# Declared setup runs during installation, then temporary packages are removed: -$ python3 tools/internal/devcontainer/install.py install-environment s-core-local --phase cleanup ``` -The repository setup applies `npm` as the development user and `native python` -with sudo, using `--providers`. This preserves the Node base image's npm access -while putting Python commands in system paths. The feature's pre-commit, +By default, the installer adds temporary build packages, installs the tools and +runs their setup, then removes the temporary packages. `--phase` can select one +phase explicitly; `--providers` can restrict installation to a provider group. +The repository setup uses the same single invocation. The installer keeps npm +in the development user's environment and elevates system installation as needed. +This preserves the Node base image's npm access while putting Python commands +in system paths. The feature's pre-commit, basedpyright and pytest use `/usr/local/bin` and shared uv tool environments; all container users can access them. Package versions are unchanged. @@ -173,8 +182,8 @@ dependencies and unsupported platform artifacts fail before installation. CodeQL's existing Linux ARM64 omission is explicit. Download or installation failure can still leave earlier tools installed; application is not transactional. -Feature entry scripts only apply the selection's build, runtime and cleanup -phases. Tool-specific steps live in `internal/devcontainer/setup/`; definitions +Feature entry scripts request their selection once; the installer handles the +build, runtime and cleanup phases. Tool-specific steps live in `internal/devcontainer/setup/`; definitions name the required script with `setup_script`. For example, `openjdk` selects an APT package and declares Java home configuration. Its name carries no version; `packages` and `version` select the actual OpenJDK release. diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index 48c1e7bd..dd4a225f 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -267,6 +267,67 @@ def _cmd_install(args: argparse.Namespace) -> int: return 0 +def _is_writable(path: str) -> bool: + """Check the nearest existing parent when an install directory is still missing.""" + directory = Path(path).resolve() + while not directory.exists(): + directory = directory.parent + return os.access(directory, os.W_OK) + + +def _delegate_system_installation( + args: argparse.Namespace, selection: dict[str, list[str]] +) -> bool: + """Return whether system installation was completed by a privileged child.""" + if os.geteuid() == 0: + return False + providers = [ + provider + for provider in args.providers + if provider != "npm" and selection.get(provider) + ] + if args.phase != "runtime": + providers = ["apt"] + needs_root = bool(selection.get("build-apt")) + else: + needs_root = ( + any(provider in providers for provider in ("apt", "setup")) + or ("native" in providers and not _is_writable(args.destination)) + or ( + "python" in providers + and not all(_is_writable(p) for p in (args.destination, args.tool_dir)) + ) + ) + if not needs_root: + return False + + # Reuse the same installer and explicit paths under sudo. Do not elevate the + # whole invocation: npm must retain the user's PATH and global package prefix. + subprocess.run( + [ + "sudo", + sys.executable, + str(Path(__file__).resolve()), + "install-environment", + args.environment, + "--phase", + args.phase, + "--destination", + str(Path(args.destination).resolve()), + "--tool-dir", + str(Path(args.tool_dir).resolve()), + "--os", + args.os, + "--cpu", + args.cpu, + "--providers", + *providers, + ], + check=True, + ) + return True + + def _cmd_install_environment(args: argparse.Namespace) -> int: """Install an environment's tools, or add/remove its temporary build packages.""" selection = load_environments().get(args.environment) @@ -304,6 +365,26 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: if system not in definition.get("omissions", {}): setup_scripts.append(setup_root / script) + if args.phase == "all": + phases = ("build", "runtime", "cleanup") + if set(args.providers) == {"npm"}: + phases = ("runtime",) + for phase in phases: + phase_args = argparse.Namespace(**vars(args)) + phase_args.phase = phase + _apply_environment(phase_args, selection, definitions, setup_scripts) + return 0 + return _apply_environment(args, selection, definitions, setup_scripts) + + +def _apply_environment(args, selection, definitions, setup_scripts) -> int: + """Apply one validated installation phase, elevating system steps when needed.""" + if _delegate_system_installation(args, selection): + if args.phase != "runtime": + return 0 + # System groups are complete; npm still runs with the caller's Node setup. + args.providers = [p for p in args.providers if p == "npm"] + # Build packages, such as zstd for unpacking CodeQL, are added before tool # installation and removed after all setup steps have finished. if args.phase != "runtime": @@ -380,7 +461,11 @@ def emit(name, value): emit( name.replace("-", "_") + "_version", definition["compatibility_version"] ) - # Tests and Python installation read the same version definitions. + # Native and Python command versions use the same variable names as the + # other definitions, so feature tests need just one metadata query. + for name, version in load_catalog_versions().items(): + emit(name.replace("-", "_") + "_version", version) + # Python package and entrypoint metadata is also available to setup scripts. for name, definition in load_python_tools().items(): emit(name.replace("-", "_"), definition) return values @@ -458,10 +543,13 @@ def _build_parser() -> argparse.ArgumentParser: nargs="+", choices=("apt", "native", "python", "npm", "setup"), default=("apt", "native", "python", "npm", "setup"), - help="Apply only these provider groups, e.g. npm as the development user.", + help="Apply only these provider groups; installation rights are handled automatically.", ) environment_parser.add_argument( - "--phase", choices=("runtime", "build", "cleanup"), default="runtime" + "--phase", + choices=("all", "runtime", "build", "cleanup"), + default="all", + help="Run all installation phases by default, or apply just one phase.", ) environment_parser.add_argument("--tool-dir", default="/usr/local/share/uv/tools") environment_parser.add_argument("--destination", default="/usr/local/bin") diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py index ca4089ad..1a694ae1 100644 --- a/tools/tests/environment_test.py +++ b/tools/tests/environment_test.py @@ -28,6 +28,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from tools.internal import catalog +from tools.internal.devcontainer import install as installer TOOLS_DIR = Path(__file__).resolve().parents[1] @@ -171,7 +172,8 @@ def test_complete_selection_preserves_provider_order_and_package_policies( "with open(os.environ['SCORE_TEST_LOG'], 'a') as log:\n" " log.write(json.dumps({'command': Path(sys.argv[0]).name, " "'args': sys.argv[1:], 'bin_dir': os.environ.get('UV_TOOL_BIN_DIR'), " - "'tool_dir': os.environ.get('UV_TOOL_DIR')}) + '\\n')\n" + "'tool_dir': os.environ.get('UV_TOOL_DIR'), " + "'privileged': os.environ.get('SCORE_TEST_PRIVILEGED')}) + '\\n')\n" ) for name in ("apt-get", "npm"): executable = fake_bin / name @@ -240,19 +242,18 @@ def test_complete_selection_preserves_provider_order_and_package_policies( PATH=str(fake_bin) + os.pathsep + os.environ["PATH"], SCORE_TEST_LOG=str(log), ) - for phase in ("build", "runtime", "cleanup"): - result = self._run( - tools, - root, - "ci", - destination, - "--phase", - phase, - "--tool-dir", - str(tool_dir), - env=environment, - ) - self.assertEqual(result.returncode, 0, result.stderr) + # One ordinary invocation installs build dependencies, applies tools, + # and removes the temporary dependencies after successful setup. + result = self._run( + tools, + root, + "ci", + destination, + "--tool-dir", + str(tool_dir), + env=environment, + ) + self.assertEqual(result.returncode, 0, result.stderr) calls = [json.loads(line) for line in log.read_text().splitlines()] self.assertEqual( [(c["command"], c["args"]) for c in calls], @@ -276,8 +277,18 @@ def test_complete_selection_preserves_provider_order_and_package_policies( ) self.assertEqual(calls[3]["bin_dir"], str(destination)) self.assertEqual(calls[3]["tool_dir"], str(tool_dir)) - # Repository setup can apply npm in the user's environment while - # applying system tools in a separate privileged invocation. + # The installer elevates system tools but keeps npm in the user's + # environment. This fixture's sudo records that boundary without root. + expected_privilege = "1" if os.geteuid() != 0 else None + self.assertTrue( + all( + c["privileged"] == expected_privilege + for c in calls + if c["command"] != "npm" + ) + ) + self.assertIsNone(calls[4]["privileged"]) + # A provider filter still permits installing just one group. log.unlink() result = self._run( tools, root, "ci", destination, "--providers", "npm", env=environment @@ -300,6 +311,125 @@ def test_complete_selection_preserves_provider_order_and_package_policies( self.assertEqual([c["command"] for c in remaining], ["apt-get"]) self.assertEqual(remaining[0]["args"][-1], "minimal") + def test_repository_setup_elevates_system_tools_and_keeps_npm_in_caller( + self, + ) -> None: + with ( + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer, "_is_writable", return_value=False), + patch.object(installer, "_cmd_install") as native, + patch.object(installer, "_cmd_install_python") as python, + patch.object(installer.subprocess, "run") as run, + ): + result = installer.main( + [ + "install-environment", + "repository-devcontainer", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + self.assertEqual(result, 0) + self.assertEqual(run.call_count, 2) + system, npm = [call.args[0] for call in run.call_args_list] + self.assertEqual(system[:2], ["sudo", sys.executable]) + self.assertEqual( + system[system.index("--providers") + 1 :], ["native", "python"] + ) + self.assertIn("/usr/local/bin", system) + self.assertIn("/usr/local/share/uv/tools", system) + self.assertEqual(npm, ["npm", "install", "-g", "@devcontainers/cli"]) + native.assert_not_called() + python.assert_not_called() + + def test_root_image_build_does_not_delegate_or_repeat_installation(self) -> None: + with ( + patch.object(installer.os, "geteuid", return_value=0), + patch.object(installer, "_cmd_install") as native, + patch.object(installer, "_cmd_install_python") as python, + patch.object(installer.subprocess, "run") as run, + ): + result = installer.main( + [ + "install-environment", + "repository-devcontainer", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + self.assertEqual(result, 0) + native.assert_called_once() + python.assert_called_once() + run.assert_called_once_with( + ["npm", "install", "-g", "@devcontainers/cli"], check=True + ) + + def test_failed_system_installation_stops_before_npm(self) -> None: + with ( + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer, "_is_writable", return_value=False), + patch.object( + installer.subprocess, + "run", + side_effect=subprocess.CalledProcessError(1, "sudo"), + ) as run, + ): + with self.assertRaises(subprocess.CalledProcessError): + installer.main( + [ + "install-environment", + "repository-devcontainer", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + run.assert_called_once() + + def test_writable_native_destination_does_not_require_sudo(self) -> None: + with tempfile.TemporaryDirectory() as directory: + args = installer._build_parser().parse_args( + [ + "install-environment", + "fixture", + "--destination", + str(Path(directory) / "new" / "bin"), + ] + ) + with ( + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer.subprocess, "run") as run, + ): + self.assertFalse( + installer._delegate_system_installation(args, {"native": ["ruff"]}) + ) + run.assert_not_called() + + def test_default_lifecycle_stops_if_runtime_installation_fails(self) -> None: + with patch.object( + installer, + "_apply_environment", + side_effect=[0, subprocess.CalledProcessError(1, "install")], + ) as apply: + with self.assertRaises(subprocess.CalledProcessError): + installer.main(["install-environment", "repository-devcontainer"]) + self.assertEqual( + [call.args[0].phase for call in apply.call_args_list], ["build", "runtime"] + ) + + def test_shell_metadata_includes_every_native_and_python_command_version( + self, + ) -> None: + # Feature tests should get all expected versions in one shell-metadata call. + metadata = installer._metadata_values() + for name, version in catalog.load_catalog_versions().items(): + self.assertEqual(metadata[name.replace("-", "_") + "_version"], version) + def test_python_dependency_and_build_only_phase_are_validated(self) -> None: with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "environments.json" @@ -467,6 +597,9 @@ def test_volume_setup_prepares_writable_user_parent_directories(self) -> None: fake_bin.mkdir() for name in (".config/opencode", ".config/gh", ".local/share/opencode"): (home / name).mkdir(parents=True, exist_ok=True) + private_directories = (home / ".config", home / ".local") + for path in private_directories: + path.chmod(0o700) sudo = fake_bin / "sudo" sudo.write_text( '#!/bin/sh\nprintf "%s\\n" "$@" >> "$SCORE_TEST_LOG"\nexec "$@"\n' @@ -486,9 +619,10 @@ def test_volume_setup_prepares_writable_user_parent_directories(self) -> None: ) self.assertEqual(result.returncode, 0, result.stderr) calls = log.read_text().splitlines() - self.assertEqual(calls[0], "install") - self.assertIn("-o", calls) - self.assertIn("-g", calls) + self.assertEqual(calls[0], "mkdir") + self.assertIn("chown", calls) + for path in private_directories: + self.assertEqual(path.stat().st_mode & 0o777, 0o700) for name in (".config", ".local", ".local/share"): self.assertIn(str(home / name), calls) @@ -506,6 +640,23 @@ def _run( *arguments, env=None, ): + # Simulate sudo for isolated integration tests: child commands may write + # only fixture paths and use fake apt/npm executables, never host packages. + fake_bin = cwd / "sudo-bin" + fake_bin.mkdir(exist_ok=True) + sudo = fake_bin / "sudo" + sudo.write_text( + f"#!{sys.executable}\n" + "import os, runpy, sys\n" + "from unittest.mock import patch\n" + "os.environ['SCORE_TEST_PRIVILEGED'] = '1'\n" + "sys.argv = sys.argv[2:]\n" + "with patch('os.geteuid', return_value=0):\n" + " runpy.run_path(sys.argv[0], run_name='__main__')\n" + ) + sudo.chmod(0o755) + env = dict(os.environ if env is None else env) + env["PATH"] = str(fake_bin) + os.pathsep + env["PATH"] return subprocess.run( [ sys.executable, From 3a00981f7034a1b52f2f8e2d0cbaf5ac0b5a9ad6 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 4 Nov 2026 13:17:28 +0100 Subject: [PATCH 5/6] fix(tools): preserve setup and respect provider filters --- tools/BUILD.bazel | 10 +- tools/internal/README.md | 152 +++--------------- tools/internal/devcontainer/README.md | 101 ++++++++++++ tools/internal/devcontainer/install.py | 11 +- tools/internal/devcontainer/setup/bazel.sh | 4 +- .../setup/bazel_compile_commands.sh | 4 +- tools/internal/devcontainer/setup/codeql.sh | 6 +- tools/internal/devcontainer/setup/java.sh | 4 + tools/lockfiles/python_tools.bzl | 9 +- tools/python_tool.bzl | 9 ++ tools/tests/environment_test.py | 73 ++++++++- 11 files changed, 235 insertions(+), 148 deletions(-) create mode 100644 tools/internal/devcontainer/README.md diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index c1a267ab..4cb3e1b4 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -13,7 +13,7 @@ load("@rules_shell//shell:sh_test.bzl", "sh_test") load(":multitool.bzl", "multitool_aliases") -load(":python_tool.bzl", "python_tool") +load(":python_tool.bzl", "python_tools") package(default_visibility = ["//visibility:public"]) @@ -49,11 +49,9 @@ multitool_aliases("opencode") # Python distributions are resolved through pinned uvx because they do not # provide the platform-specific release binaries consumed by rules_multitool. -python_tool("pre-commit") - -python_tool("basedpyright") - -python_tool("pytest") +# Export every catalogued Python command. environments.json chooses which of +# these are installed in each container; it does not restrict the module API. +python_tools() # This contract test keeps Bazel's shell launcher and the container installer # aligned with their shared catalog. diff --git a/tools/internal/README.md b/tools/internal/README.md index 0ef85ee9..f3059c5c 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -93,25 +93,11 @@ Other direct packages and special setup are defined in Bazel preinstallation and CodeQL metadata. The selections reference these names without copying their versions. -Python command-line tool metadata lives in -[`tools/lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl). The file is -a data-only Starlark dictionary because Bazel must read the pin during analysis, -before the command can run. Its restricted literal form is also valid Python -syntax. During DevContainer setup, `install.py` runs as root so it can install -tools in system locations such as `/usr/local/bin`. The shared reader parses -the catalog with `ast.literal_eval` instead of executing it as Python, so -catalog changes can provide only literal data and cannot run code with the -installer's privileges. -This shared format keeps the package, console entrypoint, version, and -description in one source of truth while Bazel execution remains independent -of host Python. - -APT `distribution` policies intentionally follow the distribution, `prefix` -constraints include Debian epochs, and `package-name` selects a versioned package -name (for example `python3.12`). Compatibility expectations for Git, Git LFS and -gh remain separate from their unpinned APT requests. The installer passes version variables directly to its setup subprocesses. -Feature tests use `shell-metadata` to read quoted assignments in their own shell; -there is no separate loader script or downloaded parser. +Python package definitions live in +[`lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl). Bazel and the +DevContainer installer read this same catalog; its introductory comments explain +the fields and literal data format. A container selects Python commands by name +in `environments.json`, just as it selects native commands. The catalogs have distinct ownership: `devcontainer-lock.json` records external DevContainer features, `python_tools.bzl` records Python package releases, and @@ -121,104 +107,15 @@ by the component that resolves it. The published `score_devcontainer` Bazel module and DevContainer image share a release version. Consumers using both must pin the same release. -## Tool selections and image variants +## DevContainer installation -Definitions answer **which version and installation method?** Named selections -in [`tools/environments.json`](../environments.json) answer **which tools belong -here?** Image and feature setup determines the base OS and lifecycle. -Selections contain names, not another copy of versions or download metadata. - -| Selection | Existing caller | -| --- | --- | -| `s-core-local` | The published image's `s-core-local` feature | -| `bazel-feature` | Its `bazel-feature` feature | -| `repository-devcontainer` | This repository's post-create setup | - -These names identify the current callers and their historical tool membership; -they are not newly designed tool groups. The published image combines the first -two through its existing features. A future lean CI image can define its own -selection based on the jobs it needs to run. This does not add another image. - -Callers request the selection with one command, for example -`install.py install-environment repository-devcontainer`. The installer checks -the complete selection before starting installation. It uses `sudo` for system -installation when needed and keeps npm in the calling user's Node environment. -Image builds already running as root install directly. Native and Python tools -can also be installed without `sudo` when their destination directories are -writable by the caller. - -Each selection groups its tools by installation method: - -| Group | Definition owner | Applied by | -| --- | --- | --- | -| `native` | `lockfiles/*.lock.json` | The native downloader | -| `python` | `lockfiles/python_tools.bzl` | Catalogued uv, with system-wide entrypoints | -| `apt` | `providers.json` | apt, preserving package/version policy and recommendations | -| `build-apt` | `providers.json` | apt during build; removed during cleanup | -| `npm` | `providers.json` | npm in the appropriate user's environment | -| `setup` | `providers.json` | Setup scripts declared by `setup_script`, gated by membership | - -For example, the local feature applies its selected groups during setup: - -```console -$ python3 tools/internal/devcontainer/install.py install-environment s-core-local -``` - -By default, the installer adds temporary build packages, installs the tools and -runs their setup, then removes the temporary packages. `--phase` can select one -phase explicitly; `--providers` can restrict installation to a provider group. -The repository setup uses the same single invocation. The installer keeps npm -in the development user's environment and elevates system installation as needed. -This preserves the Node base image's npm access while putting Python commands -in system paths. The feature's pre-commit, -basedpyright and pytest use `/usr/local/bin` and shared uv tool environments; -all container users can access them. Package versions are unchanged. - -`--destination`, `--os` and `--cpu` retain their native-install meanings. -APT/setup platform declarations are also checked; `--os` and `--cpu` select -metadata, not an emulated runtime for apt or uv. The whole selection is checked -before applying any phase: unknown names, wrong provider groups, missing direct -dependencies and unsupported platform artifacts fail before installation. -CodeQL's existing Linux ARM64 omission is explicit. Download or installation -failure can still leave earlier tools installed; application is not transactional. - -Feature entry scripts request their selection once; the installer handles the -build, runtime and cleanup phases. Tool-specific steps live in `internal/devcontainer/setup/`; definitions -name the required script with `setup_script`. For example, `openjdk` selects an -APT package and declares Java home configuration. Its name carries no version; -`packages` and `version` select the actual OpenJDK release. - -The installer gives setup scripts version variables directly as their process -environment. Python can set a child's environment, but cannot change the shell -that invoked it. Java and CodeQL setup therefore also write profile files for -later terminals. The feature tests read quoted `shell-metadata` output directly. - -Setup scripts can query `selected ENVIRONMENT TOOL` for optional bundle members. -CodeQL pack names come from the owned bundle through `items`, not a second list. -CodeQL declares `zstd` as a build dependency, so missing decompression support is -rejected before the build phase. Removing membership stops new setup from -offering the tool; it does not uninstall existing host copies. - -For APT definitions, `packages` and `platforms` are independent lists. Every -listed package is installed on each supported host; apt selects its host build. -For example, both QEMU guest emulators are installed on both Linux host CPUs. -Their order does not map packages to architectures. - -A distribution policy is not an exact pin: Black and the other APT-provided -Python CLIs retain the current Ubuntu version selection. Changing their provider -to pinned uv distributions requires explicit version choices and removal of the -APT copies. It is a separate change from recording existing selections. - -`ca-certificates-java` builds the Java trust store from system certificates. -Pinning that helper alone does not freeze the trust data or imported corporate -CAs. These continue to receive distribution updates. Reproducible base/APT and -certificate snapshots, and rebuilding releases for trust updates, are outside -this change's guarantee. - -This file covers the direct tools managed by our three setup scripts. Base-image -contents, external Dev Container features (such as LLVM, common-utils and DinD), -and their options and locks retain their existing configuration owners. It is -not an inventory of transitive packages, editor extensions or image contents. +Each setup caller selects its tools in +[`environments.json`](../environments.json). Definitions contain versions and +installation metadata; selections contain tool names. See the +[DevContainer installer guide](devcontainer/README.md) for selections, +installation phases, permissions and package policies. The scripts in +[`devcontainer/setup/`](devcontainer/setup) explain tool-specific configuration +and their environment-variable inputs beside the implementation. ## Adding or updating a tool @@ -235,8 +132,8 @@ changes together: 4. Add the command to the applicable native selection in [`tools/environments.json`](../environments.json): `s-core-local` for the `s-core-local` feature or `bazel-feature` for the `bazel-feature` feature. -5. Read the lockfile version and test the installed command in the matching - feature test: +5. Read expected versions with `install.py shell-metadata` and test the installed + command in the matching feature test: [`s-core-local/tests/test_default.sh`](../../src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh) or [`bazel-feature/tests/test_default.sh`](../../src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh). @@ -249,19 +146,12 @@ changes together: 7. If this repository's own DevContainer needs the command, add it to the `repository-devcontainer` selection in `tools/environments.json`. -For a Python command-line tool, add an entry to -`tools/lockfiles/python_tools.bzl`, add `python_tool("")` to -`tools/BUILD.bazel`, and add its name to the `python` group of each applicable -selection in `tools/environments.json`. Keep the catalog as a single literal -`PYTHON_TOOLS` assignment: this is what makes it safe for the installer and -directly loadable by Bazel. The `package` selects the distribution passed to -`uv`, while `entrypoint` names the console command because those names need not -be identical. - -Bazel invokes Python tools on demand with the pinned `uvx`; DevContainers -install the same package with the pinned `uv`. A Python selection must also -select native `uv`; Bazel already carries pinned uvx in its launcher runfiles. Add launcher and installer -coverage to `python_tool_runner_test` when either delivery contract changes. +For a Python command-line tool, add its package definition to +[`lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl) and its name to the +`python` group of the applicable selections in `environments.json`. Bazel targets +are created for every Python catalog entry automatically; no second command list +is needed in `BUILD.bazel`. Select native `uv` alongside Python tools for +container installation; Bazel supplies its own pinned `uvx`. Keep related commands such as `uv` and `uvx` in one lockfile when upstream publishes them together. The installer can locate a command in a differently diff --git a/tools/internal/devcontainer/README.md b/tools/internal/devcontainer/README.md new file mode 100644 index 00000000..a3b7ca91 --- /dev/null +++ b/tools/internal/devcontainer/README.md @@ -0,0 +1,101 @@ + + +# Installing tools in DevContainers + +The feature entry scripts and repository post-create script each request one +selection from [`environments.json`](../../environments.json): + +| Selection | Caller | +| --- | --- | +| `s-core-local` | The published image's `s-core-local` feature | +| `bazel-feature` | The published image's `bazel-feature` feature | +| `repository-devcontainer` | This repository's post-create setup | + +These lists record the existing callers' tool choices. A future image can add +its own selection. Base-image contents and external Dev Container features keep +their existing configuration owners; transitive packages are not listed here. +Bazel's public command targets are independent of these image selections. + +```console +$ python3 tools/internal/devcontainer/install.py install-environment s-core-local +``` + +## Installation and permissions + +One call validates the complete selection, adds temporary build packages, +installs tools and runs their setup, then removes the temporary packages. +Unknown names, missing direct dependencies and unavailable platform artifacts +fail before installation begins. A later download or installation failure stops +the sequence; installation is not transactional. + +The installer uses `sudo` for system installation when needed and keeps npm in +the calling user's Node environment. Image builds already running as root +install directly. Python tools use the pinned uv and shared tool environments; +entrypoints in `/usr/local/bin` are available to all container users. + +`--phase build|runtime|cleanup` requests an individual phase. `--providers` +restricts the installation methods applied. Native-, Python- and npm-only calls +skip APT build and cleanup steps. Writable native/Python destinations do not +require `sudo`. `--destination` and `--tool-dir` select these directories. +`--os` and `--cpu` select artifact metadata; they do not emulate apt or uv. + +## Definitions and setup + +| Group in a selection | Definition owner | Installation | +| --- | --- | --- | +| `native` | `lockfiles/*.lock.json` | Download and verify the release artifact | +| `python` | `lockfiles/python_tools.bzl` | Install the pinned Python package with uv | +| `apt` | `providers.json` | Install the declared packages using their version policy | +| `build-apt` | `providers.json` | Add temporary packages for build/setup; remove afterward | +| `npm` | `providers.json` | Install npm packages in the caller's environment | +| `setup` | `providers.json` | Run the declared tool-specific setup | + +Definitions can name a script in [`setup/`](setup) with `setup_script`. This also +applies to APT tools: OpenJDK needs `java.sh` to configure `JAVA_HOME` after +installation. Each script explains its purpose and inputs at the top. +`install.py` passes metadata as environment variables, not command-line flags. +Scripts can query `selected ENVIRONMENT TOOL` and `items TOOL FIELD` for optional +bundle membership. CodeQL, for example, selects its C/C++ packs from the catalog +and declares zstd as a direct build dependency. + +A child process cannot set variables in the shell that launched it. Java and +CodeQL therefore write profile files for later terminals. Feature tests use +`eval "$(.../install.py shell-metadata)"` to read quoted version assignments from +all catalogs in a single call. + +## APT policies and platform limits + +APT `distribution` follows Ubuntu's selected version; `prefix` includes any +Debian epoch (for example `1:3.22.0`); `package-name` chooses a versioned package +such as `python3.12`. Compatibility expectations for Git, Git LFS and gh are +separate from their unpinned APT requests. Black and other APT-provided Python +CLIs remain distribution-selected; migrating them to uv needs deliberate pins +and removal of the APT copies. + +`packages` and `platforms` are independent lists. All listed packages are +installed on each supported host; apt selects the host build. Both QEMU guest +emulators, for example, are installed on both Linux host CPUs. CodeQL explicitly +omits Linux ARM64 because its selected bundle has no artifact for that platform. + +`ca-certificates-java` populates Java's trust store from system certificates. +Pinning the helper alone would not freeze CA data or imported company CAs. +These inputs continue to follow distribution updates; this catalog does not +provide base-image, APT or certificate snapshots. + +Removing a tool from a selection stops offering it in new setups. The installer +does not uninstall previous host copies. Return to the +[tool maintenance guide](../README.md) for adding commands and Bazel targets. diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index dd4a225f..ab532c77 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -276,7 +276,7 @@ def _is_writable(path: str) -> bool: def _delegate_system_installation( - args: argparse.Namespace, selection: dict[str, list[str]] + args: argparse.Namespace, selection: dict[str, list[str]], setup_scripts: list[Path] ) -> bool: """Return whether system installation was completed by a privileged child.""" if os.geteuid() == 0: @@ -284,7 +284,8 @@ def _delegate_system_installation( providers = [ provider for provider in args.providers - if provider != "npm" and selection.get(provider) + if provider != "npm" + and (selection.get(provider) or (provider == "setup" and setup_scripts)) ] if args.phase != "runtime": providers = ["apt"] @@ -367,7 +368,7 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: if args.phase == "all": phases = ("build", "runtime", "cleanup") - if set(args.providers) == {"npm"}: + if not {"apt", "setup"}.intersection(args.providers): phases = ("runtime",) for phase in phases: phase_args = argparse.Namespace(**vars(args)) @@ -379,7 +380,9 @@ def _cmd_install_environment(args: argparse.Namespace) -> int: def _apply_environment(args, selection, definitions, setup_scripts) -> int: """Apply one validated installation phase, elevating system steps when needed.""" - if _delegate_system_installation(args, selection): + if args.phase != "runtime" and not {"apt", "setup"}.intersection(args.providers): + return 0 + if _delegate_system_installation(args, selection, setup_scripts): if args.phase != "runtime": return 0 # System groups are complete; npm still runs with the caller's Node setup. diff --git a/tools/internal/devcontainer/setup/bazel.sh b/tools/internal/devcontainer/setup/bazel.sh index dba36496..0597bde1 100755 --- a/tools/internal/devcontainer/setup/bazel.sh +++ b/tools/internal/devcontainer/setup/bazel.sh @@ -12,7 +12,9 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# install.py passes version and checksum variables from providers.json. +# Make Bazel available through Bazelisk, pre-download the chosen release for the +# development user, and configure completion and the Java trust store. +# install.py starts this script with bazel_version as an environment variable. # shellcheck disable=SC2154 set -euo pipefail diff --git a/tools/internal/devcontainer/setup/bazel_compile_commands.sh b/tools/internal/devcontainer/setup/bazel_compile_commands.sh index 5b6a1a7e..12f7a232 100755 --- a/tools/internal/devcontainer/setup/bazel_compile_commands.sh +++ b/tools/internal/devcontainer/setup/bazel_compile_commands.sh @@ -12,7 +12,9 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# install.py passes version and checksum variables from providers.json. +# Install the helper that generates compile_commands.json for Bazel projects. +# install.py supplies the release version and per-CPU checksums as environment +# variables: bazel_compile_commands_version and bazel_compile_commands_*_sha256. # shellcheck disable=SC2154 set -euo pipefail diff --git a/tools/internal/devcontainer/setup/codeql.sh b/tools/internal/devcontainer/setup/codeql.sh index 42c76bfa..5c54ae9c 100755 --- a/tools/internal/devcontainer/setup/codeql.sh +++ b/tools/internal/devcontainer/setup/codeql.sh @@ -12,7 +12,11 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# install.py passes version and checksum variables from providers.json. +# Install the CodeQL bundle and selected C/C++ query packs, configure CODEQL_HOME +# for later terminals, and remove unused language tools to reduce image size. +# install.py supplies codeql_version, checksums and query-pack versions as +# environment variables. SCORE_TOOLS_INSTALLER and SCORE_TOOL_ENVIRONMENT let +# this script query the selected pack bundle from the same tool definitions. # shellcheck disable=SC2154 set -euo pipefail diff --git a/tools/internal/devcontainer/setup/java.sh b/tools/internal/devcontainer/setup/java.sh index 0d6e9e15..e7ad931c 100755 --- a/tools/internal/devcontainer/setup/java.sh +++ b/tools/internal/devcontainer/setup/java.sh @@ -12,6 +12,10 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +# Configure JAVA_HOME for later terminals after APT has installed the JDK. +# install.py runs this script for the selected OpenJDK definition; no arguments +# or version variables are needed because javac identifies the installed JDK. + set -euo pipefail # Make JAVA_HOME available in later terminals so tools such as Bazel can find diff --git a/tools/lockfiles/python_tools.bzl b/tools/lockfiles/python_tools.bzl index 94ff046a..11964648 100644 --- a/tools/lockfiles/python_tools.bzl +++ b/tools/lockfiles/python_tools.bzl @@ -11,9 +11,12 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# A literal Starlark dictionary lets Bazel consume the pins during analysis -# while ast.literal_eval preserves a data-only boundary for the root-running -# DevContainer installer. +# Define each Python tool's package, pinned version and console command here. +# Both Bazel and the DevContainer installer read these definitions directly; +# environments.json selects which commands to install in each container. +# providers.json holds the APT, npm and special setup definitions instead. +# Keep this a literal dictionary: Bazel loads it as Starlark, while Python +# reads it with ast.literal_eval without executing repository code as root. PYTHON_TOOLS = { # The key is the public command and Bazel target name. "pre-commit": { diff --git a/tools/python_tool.bzl b/tools/python_tool.bzl index 0ff4c56b..5e197059 100644 --- a/tools/python_tool.bzl +++ b/tools/python_tool.bzl @@ -51,3 +51,12 @@ def python_tool(name): # its version and platform selection. data = [":uvx_binary"], ) + +def python_tools(): + """Expose every catalogued Python command as a public Bazel target.""" + + # Environment selections control image contents. The module API includes + # every command in the catalog so removing a tool from one image does not + # remove a public target used by other repositories. + for name in PYTHON_TOOLS: + python_tool(name) diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py index 1a694ae1..d7f9a75a 100644 --- a/tools/tests/environment_test.py +++ b/tools/tests/environment_test.py @@ -406,7 +406,9 @@ def test_writable_native_destination_does_not_require_sudo(self) -> None: patch.object(installer.subprocess, "run") as run, ): self.assertFalse( - installer._delegate_system_installation(args, {"native": ["ruff"]}) + installer._delegate_system_installation( + args, {"native": ["ruff"]}, [] + ) ) run.assert_not_called() @@ -430,6 +432,75 @@ def test_shell_metadata_includes_every_native_and_python_command_version( for name, version in catalog.load_catalog_versions().items(): self.assertEqual(metadata[name.replace("-", "_") + "_version"], version) + def test_apt_tool_setup_is_retained_when_installation_is_delegated(self) -> None: + # Java is selected through APT, but also needs java.sh to set JAVA_HOME. + # Both an image build and a normal user must execute that setup. + for uid in (0, 1000): + with ( + self.subTest(uid=uid), + patch.object(installer.os, "geteuid", return_value=uid), + patch.object( + installer, + "load_environments", + return_value={"java-only": {"apt": ["openjdk"]}}, + ), + patch.object(installer.subprocess, "run") as run, + ): + installer.main( + [ + "install-environment", + "java-only", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + commands = [call.args[0] for call in run.call_args_list] + if uid == 0: + self.assertEqual(commands[0][0], "apt-get") + self.assertEqual(commands[1][0], "bash") + self.assertEqual(Path(commands[1][1]).name, "java.sh") + else: + self.assertEqual(len(commands), 1) + self.assertEqual(commands[0][0], "sudo") + self.assertEqual( + commands[0][commands[0].index("--providers") + 1 :], + ["apt", "setup"], + ) + + def test_native_only_installation_skips_apt_even_if_selection_has_build_packages( + self, + ) -> None: + # s-core-local includes APT build dependencies for special setup. A user + # requesting only native tools must not install or purge those packages. + with tempfile.TemporaryDirectory() as directory: + for phase in ("all", "build", "cleanup"): + with ( + self.subTest(phase=phase), + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer, "_cmd_install") as native, + patch.object(installer.subprocess, "run") as run, + ): + installer.main( + [ + "install-environment", + "s-core-local", + "--destination", + directory, + "--os", + "linux", + "--cpu", + "arm64", + "--phase", + phase, + "--providers", + "native", + ] + ) + run.assert_not_called() + self.assertEqual(native.call_count, 1 if phase == "all" else 0) + def test_python_dependency_and_build_only_phase_are_validated(self) -> None: with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "environments.json" From f016017180860fb43ef2aa78ed46439ced31ada5 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 4 Nov 2026 13:51:29 +0100 Subject: [PATCH 6/6] refactor(tools): consolidate Python packages in provider definitions --- .pre-commit-config.yaml | 4 +- MODULE.bazel | 10 +++ tools/BUILD.bazel | 16 ++--- tools/catalog_repository.bzl | 48 +++++++++++++ tools/internal/README.md | 27 ++++--- tools/internal/catalog.py | 97 ++++++-------------------- tools/internal/devcontainer/README.md | 2 +- tools/internal/devcontainer/install.py | 2 +- tools/lockfiles/python_tools.bzl | 45 ------------ tools/providers.json | 24 +++++++ tools/python_tool.bzl | 4 +- tools/tests/catalog_test.py | 74 ++++++++++++-------- tools/tests/environment_test.py | 1 + tools/tests/python_tool_runner_test.sh | 18 +++++ 14 files changed, 191 insertions(+), 181 deletions(-) create mode 100644 tools/catalog_repository.bzl delete mode 100644 tools/lockfiles/python_tools.bzl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fec82e4d..3d2081ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,8 +72,8 @@ repos: entry: python3 tools/internal/sync_readme.py language: system pass_filenames: false - # Watch the shared reader as well as both catalog formats. - files: ^tools/(README\.md|(environments|providers)\.json|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/(.*\.lock\.json|python_tools\.bzl))$ + # Watch the shared reader, package definitions and native lockfiles. + files: ^tools/(README\.md|(environments|providers)\.json|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/.*\.lock\.json)$ - repo: https://github.com/eclipse-score/tooling rev: 31ff8eee214e4e97ef8f5cb46e443273515b63ec hooks: diff --git a/MODULE.bazel b/MODULE.bazel index e11b42e6..a93f9cca 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,16 @@ bazel_dep(name = "rules_multitool", version = "1.11.1") # Bazel versions. bazel_dep(name = "rules_shell", version = "0.4.1") +# Python packages live beside the other provider definitions in JSON. Convert +# that data to a loadable Starlark value in Bazel's cache, without a second +# maintained package list or a host Python dependency. +catalog_repository = use_repo_rule("//tools:catalog_repository.bzl", "catalog_repository") + +catalog_repository( + name = "score_tool_catalog", + definitions = "//tools:providers.json", +) + multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool") multitool.hub(lockfile = "//tools:lockfiles/actionlint.lock.json") multitool.hub(lockfile = "//tools:lockfiles/apm.lock.json") diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 4cb3e1b4..a3635edc 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -17,13 +17,9 @@ load(":python_tool.bzl", "python_tools") package(default_visibility = ["//visibility:public"]) -# Native lockfiles are consumed by rules_multitool; the Starlark catalog is -# loaded directly by python_tool.bzl. Export both because they are public -# metadata shipped with the score_devcontainer module. -exports_files(glob([ - "lockfiles/*.bzl", - "lockfiles/*.lock.json", -])) +# Native lockfiles are read by rules_multitool. The shared provider definitions +# also feed Bazel's generated Python package data; all remain module metadata. +exports_files(glob(["lockfiles/*.lock.json"]) + ["providers.json"]) multitool_aliases("actionlint") @@ -58,11 +54,15 @@ python_tools() sh_test( name = "python_tool_runner_test", srcs = ["tests/python_tool_runner_test.sh"], + # Compare the generated analysis-time catalog with the JSON used by the + # installer. rootpath resolves the generated file inside test runfiles. + args = ["$(rootpath @score_tool_catalog//:python_tools.bzl)"], data = [ "internal/catalog.py", "internal/devcontainer/install.py", "internal/run_python_tool.sh", - "lockfiles/python_tools.bzl", + "providers.json", + "@score_tool_catalog//:python_tools.bzl", ], ) diff --git a/tools/catalog_repository.bzl b/tools/catalog_repository.bzl new file mode 100644 index 00000000..20e3eba8 --- /dev/null +++ b/tools/catalog_repository.bzl @@ -0,0 +1,48 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Let Bazel read the same JSON package definitions as the container installer.""" + +def _catalog_repository_impl(repository_ctx): + definitions = json.decode(repository_ctx.read(repository_ctx.attr.definitions)) + if type(definitions) != "dict": + fail("Tool providers must be a dictionary") + tools = {} + for name, definition in definitions.items(): + if type(definition) != "dict": + fail("Invalid provider definition for '{}'".format(name)) + if definition.get("provider") != "python": + continue + tool = {} + for field in ("package", "version", "entrypoint", "description"): + value = definition.get(field) + if type(value) != "string" or not value: + fail("Malformed Python package definition for '{}'".format(name)) + tool[field] = value + dependencies = definition.get("requires", []) + if type(dependencies) != "list" or "uv" not in dependencies: + fail("Python tool '{}' must declare its uv dependency".format(name)) + tools[name] = tool + + # BUILD macros cannot read JSON source files. Produce their Starlark input + # inside Bazel's repository cache, directly from the tracked JSON file. + # repr quotes every string as data; catalog values cannot become code. + repository_ctx.file("python_tools.bzl", "PYTHON_TOOLS = " + repr(tools) + "\n") + repository_ctx.file("BUILD.bazel", 'exports_files(["python_tools.bzl"])\n') + +# The label is an explicit input: changing providers.json refreshes the generated +# data. No Python interpreter, network download or checked-in copy is needed. +catalog_repository = repository_rule( + implementation = _catalog_repository_impl, + attrs = {"definitions": attr.label(mandatory = True, allow_single_file = True)}, +) diff --git a/tools/internal/README.md b/tools/internal/README.md index f3059c5c..7ed220df 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -28,10 +28,10 @@ One catalog is delivered through two execution paths: | Component | Responsibility | | --- | --- | -| `tools/lockfiles/*.lock.json` and `tools/lockfiles/python_tools.bzl` | Define versions and delivery metadata. | +| `tools/lockfiles/*.lock.json` | Define native tool versions and release artifacts. | | `MODULE.bazel` | Makes native tool lockfiles available to `rules_multitool`. | | `tools/BUILD.bazel` | Exposes public Bazel targets for each command. | -| `tools/providers.json` | Define APT policies, npm packages and special feature setup metadata. | +| `tools/providers.json` | Define Python and npm packages, APT policies and special setup metadata. | | `tools/environments.json` | Record the complete tool selection of each existing setup caller. | | Feature installers | Apply selections and install the commands exposed on the DevContainer's `PATH`. | | `tools/run-tool` | Uses the command on `PATH` in a container or locally when available; otherwise uses the public Bazel alias (`--strict` forces Bazel). | @@ -89,20 +89,16 @@ supported platforms, download URLs, checksums, and archive layouts. The reader has no installation side effects; download, platform selection, and installation remain in the container installer. Other direct packages and special setup are defined in -[`providers.json`](../providers.json): APT constraints, npm package identities, -Bazel preinstallation and CodeQL metadata. The selections reference these names +[`providers.json`](../providers.json): Python package pins, APT constraints, +npm package identities, Bazel preinstallation and CodeQL metadata. The selections reference these names without copying their versions. -Python package definitions live in -[`lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl). Bazel and the -DevContainer installer read this same catalog; its introductory comments explain -the fields and literal data format. A container selects Python commands by name -in `environments.json`, just as it selects native commands. - -The catalogs have distinct ownership: `devcontainer-lock.json` records external -DevContainer features, `python_tools.bzl` records Python package releases, and -`uv.lock.json` records the installer runtime. Each version is therefore owned -by the component that resolves it. +Bazel reads the Python entries from this same JSON file through +[`catalog_repository.bzl`](../catalog_repository.bzl). It creates loadable +Starlark data in Bazel's cache; contributors maintain only `providers.json`. +The container installer reads the JSON directly. `devcontainer-lock.json` +continues to record external Dev Container features, while `uv.lock.json` +records the native installer runtime. The published `score_devcontainer` Bazel module and DevContainer image share a release version. Consumers using both must pin the same release. @@ -147,7 +143,8 @@ changes together: `repository-devcontainer` selection in `tools/environments.json`. For a Python command-line tool, add its package definition to -[`lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl) and its name to the +[`providers.json`](../providers.json) with `provider: "python"`, its package, +version, entrypoint, description and `requires: ["uv"]`. Add its name to the `python` group of the applicable selections in `environments.json`. Bazel targets are created for every Python catalog entry automatically; no second command list is needed in `BUILD.bazel`. Select native `uv` alongside Python tools for diff --git a/tools/internal/catalog.py b/tools/internal/catalog.py index e18b8124..ec29735f 100644 --- a/tools/internal/catalog.py +++ b/tools/internal/catalog.py @@ -16,7 +16,6 @@ from __future__ import annotations -import ast import json from collections.abc import Iterator from pathlib import Path @@ -45,7 +44,6 @@ class ToolData(TypedDict): LOCKFILE_ROOT = Path(__file__).resolve().parents[1] / "lockfiles" -PYTHON_TOOL_CATALOG = LOCKFILE_ROOT / "python_tools.bzl" ENVIRONMENTS_PATH = LOCKFILE_ROOT.parent / "environments.json" @@ -74,77 +72,18 @@ def _iter_catalog() -> Iterator[tuple[str, str, ToolData]]: yield tool, path.name, definition -def _parse_python_tool_catalog() -> object: - """Return the literal assigned to ``PYTHON_TOOLS``. - - The feature installer runs as root, so importing or executing a repository - file would cross a privilege boundary. Parsing one assignment and accepting - only literal data lets Bazel and the installer share a catalog safely. - """ - try: - module = ast.parse( - PYTHON_TOOL_CATALOG.read_text(encoding="utf-8"), - filename=str(PYTHON_TOOL_CATALOG), - ) - except SyntaxError as exc: - raise SystemExit(f"Malformed Python tool catalog: {exc}") from exc - - # Exactly one statement preserves the catalog's data-only contract. - if len(module.body) != 1: - raise SystemExit( - "Python tool catalog must contain only a PYTHON_TOOLS assignment" - ) - - assignment = module.body[0] - if not isinstance(assignment, ast.Assign) or len(assignment.targets) != 1: - raise SystemExit( - "Python tool catalog must contain only a PYTHON_TOOLS assignment" - ) - - target = assignment.targets[0] - if not isinstance(target, ast.Name) or target.id != "PYTHON_TOOLS": - raise SystemExit("Python tool catalog must assign its data to PYTHON_TOOLS") - - try: - return ast.literal_eval(assignment.value) - except (TypeError, ValueError) as exc: - raise SystemExit("Python tool catalog must contain only literal data") from exc - - def load_python_tools() -> dict[str, PythonTool]: - """Validate the shared catalog and return typed Python tool metadata.""" - catalog = _parse_python_tool_catalog() - if not isinstance(catalog, dict): - raise SystemExit("PYTHON_TOOLS must be a dictionary") - - tools: dict[str, PythonTool] = {} - for command, metadata in catalog.items(): - if ( - not isinstance(command, str) - or not command - or not isinstance(metadata, dict) - ): - raise SystemExit(f"Malformed Python tool catalog entry for '{command}'") - - package = metadata.get("package") - version = metadata.get("version") - entrypoint = metadata.get("entrypoint") - description = metadata.get("description") - # Central validation gives Bazel, installation, and documentation the - # same required-field contract. - if not all( - isinstance(value, str) and value - for value in (package, version, entrypoint, description) - ): - raise SystemExit(f"Malformed Python tool catalog entry for '{command}'") - - tools[command] = { - "package": package, - "version": version, - "entrypoint": entrypoint, - "description": description, + """Read Python package definitions from the shared provider catalog.""" + return { + name: { + "package": definition["package"], + "version": definition["version"], + "entrypoint": definition["entrypoint"], + "description": definition["description"], } - return tools + for name, definition in load_providers().items() + if definition["provider"] == "python" + } def load_catalog_versions() -> dict[str, str]: @@ -240,7 +179,7 @@ def resolve_lockfile(tool: str, lockfile: str | None = None) -> str: def load_providers() -> dict: - """Read and check installation details for APT, npm and tools with custom setup.""" + """Read and check package definitions and tool-specific setup metadata.""" providers = json.loads(PROVIDERS_PATH.read_text(encoding="utf-8")) if not isinstance(providers, dict): raise SystemExit("Tool providers must be a dictionary") @@ -248,6 +187,7 @@ def load_providers() -> dict: if not isinstance(definition, dict) or definition.get("provider") not in ( "apt", "npm", + "python", "release", "bazelisk", ): @@ -273,6 +213,15 @@ def load_providers() -> dict: raise SystemExit(f"Invalid APT packages for '{name}'") if definition["policy"] != "distribution" and not definition.get("version"): raise SystemExit(f"Missing APT version constraint for '{name}'") + elif definition["provider"] == "python": + if not all( + isinstance(definition.get(field), str) and definition[field] + for field in ("package", "version", "entrypoint", "description") + ): + raise SystemExit(f"Malformed Python package definition for '{name}'") + dependencies = definition.get("requires", []) + if not isinstance(dependencies, list) or "uv" not in dependencies: + raise SystemExit(f"Python tool '{name}' must declare its uv dependency") elif definition["provider"] == "npm": if ( not isinstance(definition.get("package"), str) @@ -289,10 +238,6 @@ def all_definitions() -> dict: tools = {} sources = ( ((name, {**data, "provider": "native"}) for name, _, data in _iter_catalog()), - ( - (name, {**data, "provider": "python", "requires": ["uv"]}) - for name, data in load_python_tools().items() - ), load_providers().items(), ) for source in sources: diff --git a/tools/internal/devcontainer/README.md b/tools/internal/devcontainer/README.md index a3b7ca91..f1dd0352 100644 --- a/tools/internal/devcontainer/README.md +++ b/tools/internal/devcontainer/README.md @@ -58,7 +58,7 @@ require `sudo`. `--destination` and `--tool-dir` select these directories. | Group in a selection | Definition owner | Installation | | --- | --- | --- | | `native` | `lockfiles/*.lock.json` | Download and verify the release artifact | -| `python` | `lockfiles/python_tools.bzl` | Install the pinned Python package with uv | +| `python` | `providers.json` | Install the pinned Python package with uv | | `apt` | `providers.json` | Install the declared packages using their version policy | | `build-apt` | `providers.json` | Add temporary packages for build/setup; remove afterward | | `npm` | `providers.json` | Install npm packages in the caller's environment | diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index ab532c77..60f11a05 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -562,7 +562,7 @@ def _build_parser() -> argparse.ArgumentParser: install_python_parser = subparsers.add_parser( "install-python", - help="Install Python command-line tools declared in python_tools.bzl.", + help="Install Python command-line tools declared in providers.json.", ) install_python_parser.add_argument( "tools", diff --git a/tools/lockfiles/python_tools.bzl b/tools/lockfiles/python_tools.bzl deleted file mode 100644 index 11964648..00000000 --- a/tools/lockfiles/python_tools.bzl +++ /dev/null @@ -1,45 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# Define each Python tool's package, pinned version and console command here. -# Both Bazel and the DevContainer installer read these definitions directly; -# environments.json selects which commands to install in each container. -# providers.json holds the APT, npm and special setup definitions instead. -# Keep this a literal dictionary: Bazel loads it as Starlark, while Python -# reads it with ast.literal_eval without executing repository code as root. -PYTHON_TOOLS = { - # The key is the public command and Bazel target name. - "pre-commit": { - # `package` is resolved by uv; `entrypoint` is the executable exposed to - # users. They are separate because Python distributions may name them - # differently. - "package": "pre-commit", - "version": "4.7.0", - "entrypoint": "pre-commit", - # Documentation is generated from the catalog so version and purpose - # are reviewed together. - "description": "Run repository pre-commit hooks", - }, - "basedpyright": { - "package": "basedpyright", - "version": "1.35.0", - "entrypoint": "basedpyright", - "description": "Type check Python and provide its language server", - }, - "pytest": { - "package": "pytest", - "version": "9.0.2", - "entrypoint": "pytest", - "description": "Run Python tests", - }, -} diff --git a/tools/providers.json b/tools/providers.json index 0f5516c1..a21b73fe 100644 --- a/tools/providers.json +++ b/tools/providers.json @@ -300,5 +300,29 @@ "packages": ["python3-venv"], "policy": "distribution", "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pre-commit": { + "provider": "python", + "package": "pre-commit", + "version": "4.7.0", + "entrypoint": "pre-commit", + "description": "Run repository pre-commit hooks", + "requires": ["uv"] + }, + "basedpyright": { + "provider": "python", + "package": "basedpyright", + "version": "1.35.0", + "entrypoint": "basedpyright", + "description": "Type check Python and provide its language server", + "requires": ["uv"] + }, + "pytest": { + "provider": "python", + "package": "pytest", + "version": "9.0.2", + "entrypoint": "pytest", + "description": "Run Python tests", + "requires": ["uv"] } } diff --git a/tools/python_tool.bzl b/tools/python_tool.bzl index 5e197059..9e0969dd 100644 --- a/tools/python_tool.bzl +++ b/tools/python_tool.bzl @@ -18,7 +18,7 @@ shell and the pinned uvx executable supplied by rules_multitool. """ load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load(":lockfiles/python_tools.bzl", "PYTHON_TOOLS") +load("@score_tool_catalog//:python_tools.bzl", "PYTHON_TOOLS") def python_tool(name): """Creates a runnable target from the shared Python tool catalog.""" @@ -26,7 +26,7 @@ def python_tool(name): # Fail during analysis so a misspelled target cannot degrade into an # unpinned uvx invocation at runtime. if name not in PYTHON_TOOLS: - fail("Python tool '{}' is not defined in python_tools.bzl".format(name)) + fail("Python tool '{}' is not defined in providers.json".format(name)) tool = PYTHON_TOOLS[name] diff --git a/tools/tests/catalog_test.py b/tools/tests/catalog_test.py index 0d74c7dd..c747ca21 100644 --- a/tools/tests/catalog_test.py +++ b/tools/tests/catalog_test.py @@ -14,7 +14,6 @@ from __future__ import annotations -import ast import json import shutil import subprocess @@ -49,8 +48,15 @@ def test_metadata_matches_definitions(self) -> None: if "description" in metadata: descriptions[name] = metadata["description"] - source = (TOOLS_DIR / "lockfiles/python_tools.bzl").read_text(encoding="utf-8") - python_tools = ast.literal_eval(ast.parse(source).body[0].value) + providers = json.loads((TOOLS_DIR / "providers.json").read_text()) + python_tools = { + name: { + field: definition[field] + for field in ("package", "version", "entrypoint", "description") + } + for name, definition in providers.items() + if definition["provider"] == "python" + } self.assertEqual(catalog.load_python_tools(), python_tools) self.assertEqual(set(python_tools), {"pre-commit", "basedpyright", "pytest"}) versions.update( @@ -64,38 +70,42 @@ def test_metadata_matches_definitions(self) -> None: self.assertEqual(catalog.resolve_lockfile("uvx"), "uv") self.assertEqual(catalog.resolve_lockfile("uvx", "explicit"), "explicit") - def test_rejects_malformed_and_executable_python_catalogs(self) -> None: + def test_rejects_python_packages_with_missing_fields_or_uv_dependency(self) -> None: + valid = { + "provider": "python", + "package": "example", + "version": "1", + "entrypoint": "example", + "description": "Example tool", + "requires": ["uv"], + } with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - path = root / "python_tools.bzl" - marker = root / "executed" - payload = f"__import__('pathlib').Path({str(marker)!r}).touch()" - cases = [ - ("PYTHON_TOOLS = {", "Malformed Python tool catalog"), - ("OTHER = {}", "must assign its data to PYTHON_TOOLS"), - ("PYTHON_TOOLS = []", "PYTHON_TOOLS must be a dictionary"), - ( - "PYTHON_TOOLS = {'bad': {'version': '1'}}", - "Malformed Python tool catalog entry", - ), - (f"PYTHON_TOOLS = {payload}", "must contain only literal data"), - ( - f"PYTHON_TOOLS = {{}}\n{payload}", - "must contain only a PYTHON_TOOLS assignment", - ), - ] - with patch.object(catalog, "PYTHON_TOOL_CATALOG", path): - for source, message in cases: - with self.subTest(source=source): - path.write_text(source, encoding="utf-8") + path = Path(directory) / "providers.json" + with patch.object(catalog, "PROVIDERS_PATH", path): + for field in ( + "package", + "version", + "entrypoint", + "description", + "requires", + ): + with self.subTest(field=field): + definition = { + key: value for key, value in valid.items() if key != field + } + path.write_text(json.dumps({"example": definition})) + message = ( + "uv dependency" + if field == "requires" + else "Malformed Python package definition" + ) with self.assertRaisesRegex(SystemExit, message): catalog.load_python_tools() - self.assertFalse(marker.exists()) def test_duplicate_definitions_retain_rejection(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - python_path = root / "python_tools.bzl" + python_path = root / "providers.json" metadata = { "version": "1", "description": "Example tool", @@ -105,6 +115,8 @@ def test_duplicate_definitions_retain_rejection(self) -> None: native_path.write_text(json.dumps({"example": metadata}), encoding="utf-8") duplicate_path = root / "two.lock.json" python_metadata = { + "provider": "python", + "requires": ["uv"], "package": "example", "version": "1", "entrypoint": "example", @@ -112,16 +124,16 @@ def test_duplicate_definitions_retain_rejection(self) -> None: } with ( patch.object(catalog, "LOCKFILE_ROOT", root), - patch.object(catalog, "PYTHON_TOOL_CATALOG", python_path), + patch.object(catalog, "PROVIDERS_PATH", python_path), ): for duplicate in ("lockfiles", "catalogs"): if duplicate == "lockfiles": duplicate_path.write_bytes(native_path.read_bytes()) - python_path.write_text("PYTHON_TOOLS = {}", encoding="utf-8") + python_path.write_text("{}", encoding="utf-8") else: duplicate_path.unlink() python_path.write_text( - f"PYTHON_TOOLS = {{'example': {python_metadata!r}}}", + json.dumps({"example": python_metadata}), encoding="utf-8", ) for query in ( diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py index d7f9a75a..f614d863 100644 --- a/tools/tests/environment_test.py +++ b/tools/tests/environment_test.py @@ -225,6 +225,7 @@ def test_complete_selection_preserves_provider_order_and_package_policies( }, "cli": {"provider": "npm", "package": "@fixture/cli"}, } + providers["pre-commit"] = catalog.load_providers()["pre-commit"] (tools / "providers.json").write_text(json.dumps(providers)) selection = { "native": ["uv"], diff --git a/tools/tests/python_tool_runner_test.sh b/tools/tests/python_tool_runner_test.sh index 74f1f91a..089c5cd5 100755 --- a/tools/tests/python_tool_runner_test.sh +++ b/tools/tests/python_tool_runner_test.sh @@ -20,6 +20,24 @@ runfiles_root="${TEST_SRCDIR}/${TEST_WORKSPACE}" installer="${runfiles_root}/tools/internal/devcontainer/install.py" runner="${runfiles_root}/tools/internal/run_python_tool.sh" +# Bazel and the container installer must consume the same package definitions. +# Only the cached Starlark representation is generated; JSON remains the source. +python3 - "${runfiles_root}/tools/providers.json" "${runfiles_root}/$1" <<'PYTHON' +import ast +import json +import sys +from pathlib import Path + +providers = json.loads(Path(sys.argv[1]).read_text()) +expected = { + name: {field: definition[field] for field in ("package", "version", "entrypoint", "description")} + for name, definition in providers.items() + if definition["provider"] == "python" +} +generated = ast.literal_eval(ast.parse(Path(sys.argv[2]).read_text()).body[0].value) +assert generated == expected, (generated, expected) +PYTHON + fake_uvx="${TEST_TMPDIR}/uvx" output="${TEST_TMPDIR}/uvx.args" working_directory="${TEST_TMPDIR}/working-directory"