diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh index d1cc20f2..e09b98d3 100755 --- a/.devcontainer/post_create_command.sh +++ b/.devcontainer/post_create_command.sh @@ -13,20 +13,10 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -npm install -g @devcontainers/cli - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" REPOSITORY_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -P)" -# Install uv and uvx from the native lockfile first. The pinned uv release then -# resolves every Python distribution declared in the shared catalog. -sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install actionlint bazelisk buildifier ruff shellcheck uv uvx yamlfmt - -# Install the catalogued Python distribution system-wide. The explicit -# directories avoid root-specific uv defaults and make pre-commit available to -# the non-root development user. -sudo "${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-python pre-commit \ - --bin-dir /usr/local/bin --tool-dir /usr/local/share/uv/tools +"${REPOSITORY_ROOT}/tools/internal/devcontainer/install.py" install-environment repository-devcontainer # Hooks can only be registered after the catalogued executable is on PATH. pre-commit install diff --git a/.github/renovate.json5 b/.github/renovate.json5 index a1e81c80..2ea8bc55 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -81,30 +81,6 @@ "datasourceTemplate": "github-releases", "depNameTemplate": "astral-sh/uv", "versioningTemplate": "semver" - }, - { - "customType": "regex", - "fileMatch": [ - "versions\\.yaml$" - ], - "matchStrings": [ - "apm:\\n\\s+version:\\s+(?\\d+\\.\\d+\\.\\d+)" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "microsoft/apm", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "fileMatch": [ - "versions\\.yaml$" - ], - "matchStrings": [ - "opencode:\\n\\s+version:\\s+(?\\d+\\.\\d+\\.\\d+)" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "anomalyco/opencode", - "versioningTemplate": "semver" } ], "enabledManagers": [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c56e0800..3d2081ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,8 +72,8 @@ repos: entry: python3 tools/internal/sync_readme.py language: system pass_filenames: false - # Watch the shared reader as well as both catalog formats. - files: ^tools/(README\.md|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/(.*\.lock\.json|python_tools\.bzl))$ + # Watch the shared reader, package definitions and native lockfiles. + files: ^tools/(README\.md|(environments|providers)\.json|internal/(catalog\.py|sync_readme\.py|devcontainer/install\.py)|lockfiles/.*\.lock\.json)$ - repo: https://github.com/eclipse-score/tooling rev: 31ff8eee214e4e97ef8f5cb46e443273515b63ec hooks: diff --git a/MODULE.bazel b/MODULE.bazel index e11b42e6..a93f9cca 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,16 @@ bazel_dep(name = "rules_multitool", version = "1.11.1") # Bazel versions. bazel_dep(name = "rules_shell", version = "0.4.1") +# Python packages live beside the other provider definitions in JSON. Convert +# that data to a loadable Starlark value in Bazel's cache, without a second +# maintained package list or a host Python dependency. +catalog_repository = use_repo_rule("//tools:catalog_repository.bzl", "catalog_repository") + +catalog_repository( + name = "score_tool_catalog", + definitions = "//tools:providers.json", +) + multitool = use_extension("@rules_multitool//multitool:extension.bzl", "multitool") multitool.hub(lockfile = "//tools:lockfiles/actionlint.lock.json") multitool.hub(lockfile = "//tools:lockfiles/apm.lock.json") diff --git a/REUSE.toml b/REUSE.toml index f4012b00..5bb950db 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -27,6 +27,8 @@ SPDX-License-Identifier = "Apache-2.0" path = ["resources/reopen_in_container.png", "resources/devcontainer_success.png", "tools/lockfiles/*.lock.json", + "tools/environments.json", + "tools/providers.json", "MODULE.bazel.lock", ".bazelversion", ] diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh index ea34c32c..3ac1d62d 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh @@ -26,54 +26,13 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe # shellcheck disable=SC2034 # used by apt-get only inside this script -DEBIAN_FRONTEND=noninteractive - -# Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/bazel/versions.yaml - -ARCHITECTURE=$(dpkg --print-architecture) +export DEBIAN_FRONTEND=noninteractive apt-get update -# INSTALL CONTAINER BUILD DEPENDENCIES -# Container build dependencies are not pinned, since they are removed anyway after container creation. -apt-get install apt-transport-https -y - -# Lockfile-managed Bazel tooling -/usr/local/share/score-tools/internal/devcontainer/install.py install bazelisk buildifier starpls - -# Bazelisk + Bazel -ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel - -# Pre-install a fixed Bazel version, setup the bash command completion -export USE_BAZEL_VERSION=${bazel_version} -# bazelisk downloads Bazel into the home directory of the user running the command -# lets assume there is only one user in the devcontainer -su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" -mkdir -p /etc/bash_completion.d -mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash -sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" - -# Configure Bazel to use system trust store for SSL/TLS connections -# This is required for corporate environments with custom CA certificates -echo 'startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit' >> /etc/bazel.bazelrc - -# Code completion for C++ code of Bazel projects -# (see https://github.com/kiron1/bazel-compile-commands) -source /etc/lsb-release -curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb -# Extract correct sha256 for current DISTRIB_CODENAME and check -SHA256SUM="${bazel_compile_commands_amd64_sha256}" -if [ "${ARCHITECTURE}" = "arm64" ]; then - SHA256SUM="${bazel_compile_commands_arm64_sha256}" -fi -echo "${SHA256SUM} /tmp/bazel-compile-commands.deb" | sha256sum -c - || exit 1 -apt-get install -y --no-install-recommends --fix-broken /tmp/bazel-compile-commands.deb -rm /tmp/bazel-compile-commands.deb +# environments.json selects the tools; providers.json names their extra setup scripts. +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment bazel-feature -# Cleanup -# REMOVE CONTAINER BUILD DEPENDENCIES -apt-get remove --purge -y apt-transport-https apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh index c6447abc..d971b776 100755 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh @@ -16,20 +16,17 @@ set -euo pipefail # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/bazel/versions.yaml +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -bazelisk_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version bazelisk)" -buildifier_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version buildifier)" -starpls_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version starpls)" # Bazel-related tools ## This is the bazel version preinstalled in the devcontainer. ## A solid test would disable the network interface first to prevent a different version from being downloaded, ## but that requires CAP_NET_ADMIN, which is not yet added. export USE_BAZEL_VERSION=${bazel_version} -check "validate bazelisk is working and has the correct version" bash -c "bazelisk version | grep '${bazelisk_lockfile_version}'" +check "validate bazelisk is working and has the correct version" bash -c "bazelisk version | grep '${bazelisk_version}'" check "validate bazel is working and has the correct version" bash -c "bazel version | grep '${bazel_version}'" unset USE_BAZEL_VERSION -check "validate buildifier is working and has the correct version" bash -c "buildifier --version | grep '${buildifier_lockfile_version}'" -check "validate starpls is working and has the correct version" bash -c "starpls version | grep '${starpls_lockfile_version}'" +check "validate buildifier is working and has the correct version" bash -c "buildifier --version | grep '${buildifier_version}'" +check "validate starpls is working and has the correct version" bash -c "starpls version | grep '${starpls_version}'" check "validate bazel-compile-commands is working and has the correct version" bash -c "bazel-compile-commands --version 2>&1 | grep '${bazel_compile_commands_version}'" diff --git a/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml b/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml deleted file mode 100644 index c72910ee..00000000 --- a/src/s-core-devcontainer/.devcontainer/bazel-feature/versions.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -bazel: - version: 8.6.0 - # no need to define sha256 here, as bazel is installed via bazelisk -bazel_compile_commands: - version: 0.18.0 - amd64: - # The following sha256sums are for the deb package bazel-compile-commands_-noble_amd64.deb - # It is generated by running 'sha256sum bazel-compile-commands_-noble_amd64.deb' - sha256: 6735ea846241497094719792ad3d4f67b1d3123048693d34fcdf7190f8c2da4e - arm64: - # The following sha256sums are for the deb package bazel-compile-commands_-noble_arm64.deb - # It is generated by running 'sha256sum bazel-compile-commands_-noble_arm64.deb' - sha256: d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88 diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh index ea3362df..87337d05 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh @@ -36,6 +36,14 @@ ensure_owner() { fi } -ensure_owner "/home/vscode/.config/opencode" -ensure_owner "/home/vscode/.local/share/opencode" -ensure_owner "/home/vscode/.config/gh" +# Docker may create .local and .local/share as root when mounting a volume +# below them. Give the user ownership of these parents too, so tools can create +# directories such as .local/state. Leave existing contents alone. +# mkdir preserves existing directory permissions, including private mode 0700. +sudo mkdir -p "${HOME}/.config" "${HOME}/.local" "${HOME}/.local/share" +sudo chown "${current_user_group}" \ + "${HOME}/.config" "${HOME}/.local" "${HOME}/.local/share" + +ensure_owner "${HOME}/.config/opencode" +ensure_owner "${HOME}/.local/share/opencode" +ensure_owner "${HOME}/.config/gh" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh index 8e3ded87..f5a6ec65 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/install.sh @@ -12,9 +12,6 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# shellcheck disable=SC2016 -# Variables in strings with single quotes are expected to be evaluated later by `eval` - set -euo pipefail # Copy feature sources and tests to expected location @@ -28,159 +25,16 @@ rm -f "${COPY_TARGET}/devcontainer-features.env" "${COPY_TARGET}/devcontainer-fe # shellcheck disable=SC2034 # used by apt-get only inside this script -DEBIAN_FRONTEND=noninteractive - -# Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/s-core-local/versions.yaml - -ARCHITECTURE=$(dpkg --print-architecture) -KERNEL=$(uname -s) +export DEBIAN_FRONTEND=noninteractive apt-get update -# Unminimize the image to include standard packages like man pages +# Restore standard OS documentation in the development image. bash -c "yes || true" | unminimize -apt-get install -y man-db manpages manpages-dev manpages-posix manpages-posix-dev - -# INSTALL CONTAINER BUILD DEPENDENCIES -# Container build dependencies are not pinned, since they are removed anyway after container creation. -apt-get install apt-transport-https -y - -# Python, via APT -apt-get install -y "python${python_version}" python3-pip python3-venv -# The following packages correspond to the list of packages installed by the -# devcontainer feature "python" (cf. https://github.com/devcontainers/features/tree/main/src/python ) -apt-get install -y flake8 python3-autopep8 black python3-yapf mypy pydocstyle pycodestyle bandit pipenv virtualenv pylint - -# uv must be installed from the native lockfile before it can install the -# Python distributions declared in the shared Python tool catalog below. -/usr/local/share/score-tools/internal/devcontainer/install.py install shellcheck ruff actionlint yamlfmt uv uvx apm opencode - -# Keep Python CLIs in system directories so every DevContainer user sees the -# same entrypoint and isolated environment. Bazel reads the same package pin -# from this catalog but executes it on demand with uvx. -/usr/local/share/score-tools/internal/devcontainer/install.py install-python pre-commit \ - --bin-dir /usr/local/bin --tool-dir /usr/local/share/uv/tools - -# GraphViz -# The Ubuntu Noble package of GraphViz -apt-get install -y graphviz="${graphviz_version}*" - -# Protobuf compiler, via APT (needed by FEO) -apt-get install -y protobuf-compiler="${protobuf_compiler_version}*" - -# Git and Git LFS, via APT -apt-get install -y git -apt-get install -y git-lfs -apt-get install -y gh - -# OpenJDK 21, via APT -# Set JAVA_HOME environment variable system-wide, since some tools rely on it (e.g., Bazel's rules_java) -apt-get install -y ca-certificates-java openjdk-21-jdk-headless="${openjdk_21_version}*" -JAVA_HOME="$(dirname $(dirname $(realpath $(command -v javac))))" -export JAVA_HOME -echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh - -# QEMU system emulators for ARM and x86 -apt-get install -y --no-install-recommends --fix-broken \ - qemu-system-arm="${qemu_system_version}*" \ - qemu-system-x86="${qemu_system_version}*" - -# basedpyright -su $(ls /home) -c "uv tool install basedpyright@\"${basedpyright_version}\"" - -# pytest -su $(ls /home) -c "uv tool install pytest@\"${pytest_version}\"" - -# sshpass -apt-get install -y sshpass="${sshpass_version}*" - -# pycairo build dependencies (pkg-config + cairo headers needed when pip builds pycairo from source) -apt-get install -y pkg-config="${pkg_config_version}*" libcairo2-dev="${libcairo2_dev_version}*" - -# additional developer tools -apt-get install -y gdb="${gdb_version}*" - -apt-get install -y valgrind="1:${valgrind_version}*" - -apt-get install -y ripgrep="${ripgrep_version}*" - -# lcov - code coverage reporting for C/C++ -apt-get install -y lcov="${lcov_version}*" - -# CodeQL -apt-get install -y zstd -if [ "${ARCHITECTURE}" = "amd64" ]; then - VARIANT=linux64 - SHA256SUM="${codeql_amd64_sha256}" -elif [ "${ARCHITECTURE}" = "arm64" ]; then - if [ "${KERNEL}" = "Darwin" ]; then - VARIANT=osx64 - SHA256SUM="${codeql_arm64_sha256}" - else - echo "CodeQl unsupported architecture/os: ${ARCHITECTURE} on ${KERNEL}, skipping installation" - VARIANT=noinstall - fi -else - echo "Unsupported architecture: ${ARCHITECTURE} for CodeQL" - exit 1 -fi - -if [ "${VARIANT}" != "noinstall" ]; then - codeql_install_dir="/usr/local" - curl -L "https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-${VARIANT}.tar.zst" -o /tmp/codeql.tar.zst - echo "${SHA256SUM} /tmp/codeql.tar.zst" | sha256sum -c - || exit 1 - tar -I zstd -xf /tmp/codeql.tar.zst -C "${codeql_install_dir}" - ln -s "${codeql_install_dir}"/codeql/codeql /usr/local/bin/codeql - rm /tmp/codeql.tar.zst - export CODEQL_HOME=${codeql_install_dir}/codeql - echo "export CODEQL_HOME=${codeql_install_dir}/codeql" > /etc/profile.d/codeql.sh - - codeql pack download codeql/misra-cpp-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - codeql pack download codeql/misra-c-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - codeql pack download codeql/cert-cpp-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - codeql pack download codeql/cert-c-coding-standards@"${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" - # slim down codeql bundle (1.7GB -> 1.1 GB) by removing unnecessary language extractors and qlpacks - codeql_purge_dirs=( - "${codeql_install_dir}/codeql/csharp" - "${codeql_install_dir}/codeql/go" - "${codeql_install_dir}/codeql/java" - "${codeql_install_dir}/codeql/javascript" - "${codeql_install_dir}/codeql/python" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/go-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/java-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/python-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-examples" - "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-queries" - "${codeql_install_dir}/codeql/qlpacks/codeql/swift-all" - "${codeql_install_dir}/codeql/qlpacks/codeql/swift-queries" - "${codeql_install_dir}/codeql/ruby" - "${codeql_install_dir}/codeql/swift" - ) - for dir in "${codeql_purge_dirs[@]}"; do - if [ -d "${dir}" ]; then - rm -rf "${dir}" - fi - done -fi +# environments.json selects the tools; providers.json names their extra setup scripts. +/usr/local/share/score-tools/internal/devcontainer/install.py install-environment s-core-local -# Cleanup -# REMOVE CONTAINER BUILD DEPENDENCIES -apt-get remove --purge -y apt-transport-https zstd apt-get autoremove -y apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh index f83eb4ad..ca88e87e 100755 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh +++ b/src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh @@ -19,27 +19,15 @@ ARCHITECTURE=$(dpkg --print-architecture) KERNEL=$(uname -s) # Read tool versions + metadata into environment variables -. /usr/local/share/score-tools/internal/devcontainer/load_feature_versions.sh /devcontainer/features/s-core-local/versions.yaml +eval "$(/usr/local/share/score-tools/internal/devcontainer/install.py shell-metadata)" -shellcheck_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version shellcheck)" -ruff_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version ruff)" -actionlint_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version actionlint)" -yamlfmt_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version yamlfmt)" -uv_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version uv)" -uvx_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version uvx)" -apm_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version apm)" -opencode_lockfile_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version opencode)" - -# The shared catalog is the assertion source, so installation metadata and the -# executable exposed on PATH are checked against the same release. -pre_commit_catalog_version="$(/usr/local/share/score-tools/internal/devcontainer/install.py version pre-commit)" # Verify the system-wide uv installation exposes the catalogued release to the # non-root user running feature tests. -check "validate pre-commit is working and has the correct version" bash -c "pre-commit --version | grep '${pre_commit_catalog_version}'" +check "validate pre-commit is working and has the correct version" bash -c "pre-commit --version | grep '${pre_commit_version}'" # Common tooling -check "validate shellcheck is working and has the correct version" bash -c "shellcheck --version | grep '${shellcheck_lockfile_version}'" +check "validate shellcheck is working and has the correct version" bash -c "shellcheck --version | grep '${shellcheck_version}'" # For an unknown reason, dot -V reports on Ubuntu Noble a version 2.43.0, while the package has a different version. # Hence, we have to work around that. @@ -66,27 +54,29 @@ check "validate pytest is working" bash -c "pytest --version" check "validate pylint is working" bash -c "pylint --version" # OpenJDK -check "validate java is working and has the correct version" bash -c "java -version 2>&1 | grep '${openjdk_21_version}'" -check "validate JAVA_HOME is set correctly" bash -c "echo ${JAVA_HOME} | xargs readlink -f | grep \"java-21-openjdk\"" +check "validate java is working and has the correct version" bash -c "java -version 2>&1 | grep '${openjdk_version}'" +# Expansion belongs to the child shell that performs the check. +# shellcheck disable=SC2016 +check "validate JAVA_HOME matches javac" bash -c 'test "$(readlink -f "$JAVA_HOME")" = "$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")")"' # ruff -check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_lockfile_version}'" +check "validate ruff is working and has the correct version" bash -c "ruff --version | grep '${ruff_version}'" # actionlint -check "validate actionlint is working and has the correct version" bash -c "actionlint --version | grep '${actionlint_lockfile_version}'" +check "validate actionlint is working and has the correct version" bash -c "actionlint --version | grep '${actionlint_version}'" # yamlfmt -check "validate yamlfmt is working and has the correct version" bash -c "yamlfmt --version | grep '${yamlfmt_lockfile_version}'" +check "validate yamlfmt is working and has the correct version" bash -c "yamlfmt --version | grep '${yamlfmt_version}'" # apm -check "validate apm is working and has the correct version" bash -c "apm --version | grep '${apm_lockfile_version}'" +check "validate apm is working and has the correct version" bash -c "apm --version | grep '${apm_version}'" # opencode -check "validate opencode is working and has the correct version" bash -c "opencode --version | grep '${opencode_lockfile_version}'" +check "validate opencode is working and has the correct version" bash -c "opencode --version | grep '${opencode_version}'" # uv -check "validate uv is working and has the correct version" bash -c "uv --version | grep '${uv_lockfile_version}'" -check "validate uvx is working and has the correct version" bash -c "uvx --version | grep '${uvx_lockfile_version}'" +check "validate uv is working and has the correct version" bash -c "uv --version | grep '${uv_version}'" +check "validate uvx is working and has the correct version" bash -c "uvx --version | grep '${uvx_version}'" # pycairo build from source (requires pkg-config and libcairo2-dev) check "validate pycairo can be built and installed from source" bash -c "pip install --break-system-packages pycairo==1.29.0" @@ -95,7 +85,9 @@ check "validate pycairo can be built and installed from source" bash -c "pip ins check "validate gdb is working and has the correct version" bash -c "gdb --version | grep '${gdb_version}'" check "validate pkg-config is working and has the correct version" bash -c "pkg-config --version | grep '${pkg_config_version}'" check "validate gh is working and has the correct version" bash -c "gh --version | grep '${gh_version}'" -check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version}'" +# Debian package versions may start with an epoch, e.g. 1:3.22.0. +# ${value#*:} removes that prefix; the executable reports only 3.22.0. +check "validate valgrind is working and has the correct version" bash -c "valgrind --version | grep '${valgrind_version#*:}'" check "validate ripgrep is working and has the correct version" bash -c "rg --version | grep '${ripgrep_version}'" if [ "${ARCHITECTURE}" = "amd64" ] || { [ "${ARCHITECTURE}" = "arm64" ] && [ "${KERNEL}" = "Darwin" ]; }; then check "validate codeql is working and has the correct version" bash -c "codeql --version | grep '${codeql_version}'" @@ -105,7 +97,7 @@ fi # lcov check "validate lcov is working and has the correct version" bash -c "lcov --version | grep '${lcov_version}'" -# Qemu target-related tools -check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version}'" -check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version}'" +# Qemu target-related tools (strip the Debian epoch here too). +check "validate qemu-system-aarch64 is working and has the correct version" bash -c "qemu-system-aarch64 --version | grep '${qemu_system_version#*:}'" +check "validate qemu-system-x86_64 is working and has the correct version" bash -c "qemu-system-x86_64 --version | grep '${qemu_system_version#*:}'" check "validate sshpass is working and has the correct version" bash -c "sshpass -V | grep '${sshpass_version}'" diff --git a/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml b/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml deleted file mode 100644 index 279bad08..00000000 --- a/src/s-core-devcontainer/.devcontainer/s-core-local/versions.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -basedpyright: - version: 1.35.0 -graphviz: - version: 2.42.2 -protobuf_compiler: - version: 3.21.12 -qemu_system: - version: 1:8.2.2 -sshpass: - version: 1.09 -gdb: - version: 15.0.50 -git: - version: 2.43.0 -git_lfs: - version: 3.4.1 -python: - version: '3.12' -pytest: - version: 9.0.2 -gh: - version: 2.45.0 -openjdk_21: - version: 21.0.12 -codeql: - # the coding_standards_version below dictates the codeql version - version: 2.21.4 - amd64: - # see https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-linux64.tar.zst.checksum.txt - sha256: 6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545 - arm64: - # see https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-osx64.tar.zst.checksum.txt - sha256: 810a1fff48aeb081be754e46e255edd90d8695966e78431a65edb00e9e6cc399 -codeql_coding_standards: - version: 2.54.0 -valgrind: - version: 3.22.0 -ripgrep: - version: 14.1.0 -lcov: - version: 2.0 -apm: - version: 0.35.0 -opencode: - version: 2.2.3 -pkg_config: - version: 1.8.1 -libcairo2_dev: - version: 1.18.0 diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 77238405..a3635edc 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -13,17 +13,13 @@ load("@rules_shell//shell:sh_test.bzl", "sh_test") load(":multitool.bzl", "multitool_aliases") -load(":python_tool.bzl", "python_tool") +load(":python_tool.bzl", "python_tools") package(default_visibility = ["//visibility:public"]) -# Native lockfiles are consumed by rules_multitool; the Starlark catalog is -# loaded directly by python_tool.bzl. Export both because they are public -# metadata shipped with the score_devcontainer module. -exports_files(glob([ - "lockfiles/*.bzl", - "lockfiles/*.lock.json", -])) +# Native lockfiles are read by rules_multitool. The shared provider definitions +# also feed Bazel's generated Python package data; all remain module metadata. +exports_files(glob(["lockfiles/*.lock.json"]) + ["providers.json"]) multitool_aliases("actionlint") @@ -49,18 +45,24 @@ multitool_aliases("opencode") # Python distributions are resolved through pinned uvx because they do not # provide the platform-specific release binaries consumed by rules_multitool. -python_tool("pre-commit") +# Export every catalogued Python command. environments.json chooses which of +# these are installed in each container; it does not restrict the module API. +python_tools() # This contract test keeps Bazel's shell launcher and the container installer # aligned with their shared catalog. sh_test( name = "python_tool_runner_test", srcs = ["tests/python_tool_runner_test.sh"], + # Compare the generated analysis-time catalog with the JSON used by the + # installer. rootpath resolves the generated file inside test runfiles. + args = ["$(rootpath @score_tool_catalog//:python_tools.bzl)"], data = [ "internal/catalog.py", "internal/devcontainer/install.py", "internal/run_python_tool.sh", - "lockfiles/python_tools.bzl", + "providers.json", + "@score_tool_catalog//:python_tools.bzl", ], ) diff --git a/tools/README.md b/tools/README.md index 27437a02..973dc0fc 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,10 +41,12 @@ host-side Bazel invocation may require network access to download the binary. | --- | --- | --- | | `actionlint` | `1.7.7` | Check GitHub Actions workflow files | | `apm` | `0.35.0` | Manage APM packages | +| `basedpyright` | `1.35.0` | Type check Python and provide its language server | | `bazelisk` | `1.27.0` | Run the Bazel version selected by a repository | | `buildifier` | `8.2.1` | Format and lint Bazel files | | `opencode` | `2.2.3` | Run the OpenCode CLI | | `pre-commit` | `4.7.0` | Run repository pre-commit hooks | +| `pytest` | `9.0.2` | Run Python tests | | `ruff` | `0.11.13` | Check and format Python code | | `shellcheck` | `0.10.0` | Check shell scripts | | `starpls` | `0.1.22` | Provide language-server support for Starlark | diff --git a/tools/catalog_repository.bzl b/tools/catalog_repository.bzl new file mode 100644 index 00000000..20e3eba8 --- /dev/null +++ b/tools/catalog_repository.bzl @@ -0,0 +1,48 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Let Bazel read the same JSON package definitions as the container installer.""" + +def _catalog_repository_impl(repository_ctx): + definitions = json.decode(repository_ctx.read(repository_ctx.attr.definitions)) + if type(definitions) != "dict": + fail("Tool providers must be a dictionary") + tools = {} + for name, definition in definitions.items(): + if type(definition) != "dict": + fail("Invalid provider definition for '{}'".format(name)) + if definition.get("provider") != "python": + continue + tool = {} + for field in ("package", "version", "entrypoint", "description"): + value = definition.get(field) + if type(value) != "string" or not value: + fail("Malformed Python package definition for '{}'".format(name)) + tool[field] = value + dependencies = definition.get("requires", []) + if type(dependencies) != "list" or "uv" not in dependencies: + fail("Python tool '{}' must declare its uv dependency".format(name)) + tools[name] = tool + + # BUILD macros cannot read JSON source files. Produce their Starlark input + # inside Bazel's repository cache, directly from the tracked JSON file. + # repr quotes every string as data; catalog values cannot become code. + repository_ctx.file("python_tools.bzl", "PYTHON_TOOLS = " + repr(tools) + "\n") + repository_ctx.file("BUILD.bazel", 'exports_files(["python_tools.bzl"])\n') + +# The label is an explicit input: changing providers.json refreshes the generated +# data. No Python interpreter, network download or checked-in copy is needed. +catalog_repository = repository_rule( + implementation = _catalog_repository_impl, + attrs = {"definitions": attr.label(mandatory = True, allow_single_file = True)}, +) diff --git a/tools/environments.json b/tools/environments.json new file mode 100644 index 00000000..d1b7c0cb --- /dev/null +++ b/tools/environments.json @@ -0,0 +1,96 @@ +{ + "s-core-local": { + "native": [ + "shellcheck", + "ruff", + "actionlint", + "yamlfmt", + "uv", + "uvx", + "apm", + "opencode" + ], + "python": [ + "pre-commit", + "basedpyright", + "pytest" + ], + "apt": [ + "graphviz", + "protobuf_compiler", + "qemu_system", + "sshpass", + "gdb", + "git", + "git_lfs", + "gh", + "openjdk", + "valgrind", + "ripgrep", + "lcov", + "pkg_config", + "libcairo2_dev", + "python", + "python3-pip", + "python3-venv", + "flake8", + "python3-autopep8", + "black", + "python3-yapf", + "mypy", + "pydocstyle", + "pycodestyle", + "bandit", + "pipenv", + "virtualenv", + "pylint", + "ca-certificates-java", + "man-db", + "manpages", + "manpages-dev", + "manpages-posix", + "manpages-posix-dev" + ], + "setup": [ + "codeql", + "codeql_coding_standards" + ], + "build-apt": [ + "apt-transport-https", + "zstd" + ] + }, + "bazel-feature": { + "native": [ + "bazelisk", + "buildifier", + "starpls" + ], + "python": [], + "setup": [ + "bazel", + "bazel_compile_commands" + ], + "build-apt": [ + "apt-transport-https" + ] + }, + "repository-devcontainer": { + "native": [ + "actionlint", + "bazelisk", + "buildifier", + "ruff", + "shellcheck", + "uv", + "uvx", + "yamlfmt" + ], + "python": [ + "pre-commit" + ], + "npm": [ + "devcontainers-cli" + ] + } +} diff --git a/tools/internal/README.md b/tools/internal/README.md index cfae24e4..7ed220df 100644 --- a/tools/internal/README.md +++ b/tools/internal/README.md @@ -28,10 +28,12 @@ One catalog is delivered through two execution paths: | Component | Responsibility | | --- | --- | -| `tools/lockfiles/*.lock.json` and `tools/lockfiles/python_tools.bzl` | Define versions and delivery metadata. | +| `tools/lockfiles/*.lock.json` | Define native tool versions and release artifacts. | | `MODULE.bazel` | Makes native tool lockfiles available to `rules_multitool`. | | `tools/BUILD.bazel` | Exposes public Bazel targets for each command. | -| Feature installers | Install the commands exposed on the DevContainer's `PATH`. | +| `tools/providers.json` | Define Python and npm packages, APT policies and special setup metadata. | +| `tools/environments.json` | Record the complete tool selection of each existing setup caller. | +| Feature installers | Apply selections and install the commands exposed on the DevContainer's `PATH`. | | `tools/run-tool` | Uses the command on `PATH` in a container or locally when available; otherwise uses the public Bazel alias (`--strict` forces Bazel). | | `tools/README.md` | Documents every command and its version from the catalog. | @@ -86,28 +88,31 @@ supported platforms, download URLs, checksums, and archive layouts. [`devcontainer/install.py`](devcontainer/install.py) and README generation. The reader has no installation side effects; download, platform selection, and installation remain in the container installer. - -Python command-line tool metadata lives in -[`tools/lockfiles/python_tools.bzl`](../lockfiles/python_tools.bzl). The file is -a data-only Starlark dictionary because Bazel must read the pin during analysis, -before the command can run. Its restricted literal form is also valid Python -syntax. During DevContainer setup, `install.py` runs as root so it can install -tools in system locations such as `/usr/local/bin`. The shared reader parses -the catalog with `ast.literal_eval` instead of executing it as Python, so -catalog changes can provide only literal data and cannot run code with the -installer's privileges. -This shared format keeps the package, console entrypoint, version, and -description in one source of truth while Bazel execution remains independent -of host Python. - -The catalogs have distinct ownership: `devcontainer-lock.json` records external -DevContainer features, `python_tools.bzl` records Python package releases, and -`uv.lock.json` records the installer runtime. Each version is therefore owned -by the component that resolves it. +Other direct packages and special setup are defined in +[`providers.json`](../providers.json): Python package pins, APT constraints, +npm package identities, Bazel preinstallation and CodeQL metadata. The selections reference these names +without copying their versions. + +Bazel reads the Python entries from this same JSON file through +[`catalog_repository.bzl`](../catalog_repository.bzl). It creates loadable +Starlark data in Bazel's cache; contributors maintain only `providers.json`. +The container installer reads the JSON directly. `devcontainer-lock.json` +continues to record external Dev Container features, while `uv.lock.json` +records the native installer runtime. The published `score_devcontainer` Bazel module and DevContainer image share a release version. Consumers using both must pin the same release. +## DevContainer installation + +Each setup caller selects its tools in +[`environments.json`](../environments.json). Definitions contain versions and +installation metadata; selections contain tool names. See the +[DevContainer installer guide](devcontainer/README.md) for selections, +installation phases, permissions and package policies. The scripts in +[`devcontainer/setup/`](devcontainer/setup) explain tool-specific configuration +and their environment-variable inputs beside the implementation. + ## Adding or updating a tool For a tool distributed as a native release artifact, make the following @@ -120,13 +125,11 @@ changes together: [`MODULE.bazel`](../../MODULE.bazel). 3. Add `multitool_aliases("")` to [`tools/BUILD.bazel`](../BUILD.bazel). -4. Add the command to the appropriate feature installer: - [`s-core-local/install.sh`](../../src/s-core-devcontainer/.devcontainer/s-core-local/install.sh) - for general tools or - [`bazel-feature/install.sh`](../../src/s-core-devcontainer/.devcontainer/bazel-feature/install.sh) - for Bazel-specific tools. -5. Read the lockfile version and test the installed command in the matching - feature test: +4. Add the command to the applicable native selection in + [`tools/environments.json`](../environments.json): `s-core-local` for the + `s-core-local` feature or `bazel-feature` for the `bazel-feature` feature. +5. Read expected versions with `install.py shell-metadata` and test the installed + command in the matching feature test: [`s-core-local/tests/test_default.sh`](../../src/s-core-devcontainer/.devcontainer/s-core-local/tests/test_default.sh) or [`bazel-feature/tests/test_default.sh`](../../src/s-core-devcontainer/.devcontainer/bazel-feature/tests/test_default.sh). @@ -136,21 +139,16 @@ changes together: $ python3 tools/internal/sync_readme.py ``` -7. If this repository's own DevContainer needs the command, add it to - [`.devcontainer/post_create_command.sh`](../../.devcontainer/post_create_command.sh). - -For a Python command-line tool, add an entry to -`tools/lockfiles/python_tools.bzl`, add `python_tool("")` to -`tools/BUILD.bazel`, and install it through `install.py install-python` in -each applicable DevContainer installer. Keep the catalog as a single literal -`PYTHON_TOOLS` assignment: this is what makes it safe for the installer and -directly loadable by Bazel. The `package` selects the distribution passed to -`uv`, while `entrypoint` names the console command because those names need not -be identical. +7. If this repository's own DevContainer needs the command, add it to the + `repository-devcontainer` selection in `tools/environments.json`. -Bazel invokes Python tools on demand with the pinned `uvx`; DevContainers -install the same package with the pinned `uv`. Add launcher and installer -coverage to `python_tool_runner_test` when either delivery contract changes. +For a Python command-line tool, add its package definition to +[`providers.json`](../providers.json) with `provider: "python"`, its package, +version, entrypoint, description and `requires: ["uv"]`. Add its name to the +`python` group of the applicable selections in `environments.json`. Bazel targets +are created for every Python catalog entry automatically; no second command list +is needed in `BUILD.bazel`. Select native `uv` alongside Python tools for +container installation; Bazel supplies its own pinned `uvx`. Keep related commands such as `uv` and `uvx` in one lockfile when upstream publishes them together. The installer can locate a command in a differently @@ -170,7 +168,9 @@ description: $ python3 tools/internal/sync_readme.py ``` -The pre-commit hook runs the same command; pre-commit rejects the commit if +The pre-commit hook also validates selection names, provider groups, direct +dependencies and tool references. +It runs the same command; pre-commit rejects the commit if running it changes `tools/README.md`, so a stale table cannot be committed. Run the feature test for every installer changed. The feature tests read their diff --git a/tools/internal/catalog.py b/tools/internal/catalog.py index 69d07aec..ec29735f 100644 --- a/tools/internal/catalog.py +++ b/tools/internal/catalog.py @@ -10,13 +10,12 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Read pinned native and Python tool metadata without executing catalog code.""" +"""Read tool definitions and environment selections without running installation code.""" # pyright: reportAny=false, reportUnusedCallResult=false, reportExplicitAny=false from __future__ import annotations -import ast import json from collections.abc import Iterator from pathlib import Path @@ -45,7 +44,7 @@ class ToolData(TypedDict): LOCKFILE_ROOT = Path(__file__).resolve().parents[1] / "lockfiles" -PYTHON_TOOL_CATALOG = LOCKFILE_ROOT / "python_tools.bzl" +ENVIRONMENTS_PATH = LOCKFILE_ROOT.parent / "environments.json" class PythonTool(TypedDict): @@ -73,77 +72,18 @@ def _iter_catalog() -> Iterator[tuple[str, str, ToolData]]: yield tool, path.name, definition -def _parse_python_tool_catalog() -> object: - """Return the literal assigned to ``PYTHON_TOOLS``. - - The feature installer runs as root, so importing or executing a repository - file would cross a privilege boundary. Parsing one assignment and accepting - only literal data lets Bazel and the installer share a catalog safely. - """ - try: - module = ast.parse( - PYTHON_TOOL_CATALOG.read_text(encoding="utf-8"), - filename=str(PYTHON_TOOL_CATALOG), - ) - except SyntaxError as exc: - raise SystemExit(f"Malformed Python tool catalog: {exc}") from exc - - # Exactly one statement preserves the catalog's data-only contract. - if len(module.body) != 1: - raise SystemExit( - "Python tool catalog must contain only a PYTHON_TOOLS assignment" - ) - - assignment = module.body[0] - if not isinstance(assignment, ast.Assign) or len(assignment.targets) != 1: - raise SystemExit( - "Python tool catalog must contain only a PYTHON_TOOLS assignment" - ) - - target = assignment.targets[0] - if not isinstance(target, ast.Name) or target.id != "PYTHON_TOOLS": - raise SystemExit("Python tool catalog must assign its data to PYTHON_TOOLS") - - try: - return ast.literal_eval(assignment.value) - except (TypeError, ValueError) as exc: - raise SystemExit("Python tool catalog must contain only literal data") from exc - - def load_python_tools() -> dict[str, PythonTool]: - """Validate the shared catalog and return typed Python tool metadata.""" - catalog = _parse_python_tool_catalog() - if not isinstance(catalog, dict): - raise SystemExit("PYTHON_TOOLS must be a dictionary") - - tools: dict[str, PythonTool] = {} - for command, metadata in catalog.items(): - if ( - not isinstance(command, str) - or not command - or not isinstance(metadata, dict) - ): - raise SystemExit(f"Malformed Python tool catalog entry for '{command}'") - - package = metadata.get("package") - version = metadata.get("version") - entrypoint = metadata.get("entrypoint") - description = metadata.get("description") - # Central validation gives Bazel, installation, and documentation the - # same required-field contract. - if not all( - isinstance(value, str) and value - for value in (package, version, entrypoint, description) - ): - raise SystemExit(f"Malformed Python tool catalog entry for '{command}'") - - tools[command] = { - "package": package, - "version": version, - "entrypoint": entrypoint, - "description": description, + """Read Python package definitions from the shared provider catalog.""" + return { + name: { + "package": definition["package"], + "version": definition["version"], + "entrypoint": definition["entrypoint"], + "description": definition["description"], } - return tools + for name, definition in load_providers().items() + if definition["provider"] == "python" + } def load_catalog_versions() -> dict[str, str]: @@ -233,3 +173,153 @@ def resolve_lockfile(tool: str, lockfile: str | None = None) -> str: if _lockfile_path(tool).exists(): return tool return _find_lockfile(tool) + + +PROVIDERS_PATH = LOCKFILE_ROOT.parent / "providers.json" + + +def load_providers() -> dict: + """Read and check package definitions and tool-specific setup metadata.""" + providers = json.loads(PROVIDERS_PATH.read_text(encoding="utf-8")) + if not isinstance(providers, dict): + raise SystemExit("Tool providers must be a dictionary") + for name, definition in providers.items(): + if not isinstance(definition, dict) or definition.get("provider") not in ( + "apt", + "npm", + "python", + "release", + "bazelisk", + ): + raise SystemExit(f"Invalid provider definition for '{name}'") + if definition["provider"] == "apt": + if definition.get("policy") not in ( + "distribution", + "prefix", + "package-name", + ): + raise SystemExit(f"Invalid APT policy for '{name}'") + packages = definition.get("packages") + if ( + not isinstance(packages, list) + or not packages + or any( + not isinstance(package, str) + or not package + or package.startswith("-") + for package in packages + ) + ): + raise SystemExit(f"Invalid APT packages for '{name}'") + if definition["policy"] != "distribution" and not definition.get("version"): + raise SystemExit(f"Missing APT version constraint for '{name}'") + elif definition["provider"] == "python": + if not all( + isinstance(definition.get(field), str) and definition[field] + for field in ("package", "version", "entrypoint", "description") + ): + raise SystemExit(f"Malformed Python package definition for '{name}'") + dependencies = definition.get("requires", []) + if not isinstance(dependencies, list) or "uv" not in dependencies: + raise SystemExit(f"Python tool '{name}' must declare its uv dependency") + elif definition["provider"] == "npm": + if ( + not isinstance(definition.get("package"), str) + or not definition["package"] + ): + raise SystemExit(f"Missing npm package for '{name}'") + return providers + + +def all_definitions() -> dict: + """Look up every defined tool by name, regardless of how it is installed. + + Only the copied tools/ directory is needed, so this also works inside images.""" + tools = {} + sources = ( + ((name, {**data, "provider": "native"}) for name, _, data in _iter_catalog()), + load_providers().items(), + ) + for source in sources: + for name, definition in source: + if name in tools: + raise SystemExit(f"Duplicate tool definition: '{name}'") + tools[name] = definition + return tools + + +def load_environments() -> dict[str, dict[str, list[str]]]: + """Read environments.json and check tool names, installation groups and dependencies.""" + data = json.loads(ENVIRONMENTS_PATH.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SystemExit("Tool environments must be a dictionary") + tools = all_definitions() + # Build-only packages use the same APT definitions as regular packages, + # but are removed once image setup is complete. + categories = { + "native": {"native"}, + "python": {"python"}, + "apt": {"apt"}, + "build-apt": {"apt"}, + "npm": {"npm"}, + "setup": {"release", "bazelisk"}, + } + for name, selection in data.items(): + if not isinstance(name, str) or not name: + raise SystemExit("Tool environment names must be nonempty strings") + if not isinstance(selection, dict) or selection.keys() - categories.keys(): + raise SystemExit(f"Invalid provider categories in environment '{name}'") + for category, names in selection.items(): + if not isinstance(names, list) or any( + not isinstance(tool, str) or not tool for tool in names + ): + raise SystemExit(f"Environment '{name}/{category}' must be a tool list") + if len(names) != len(set(names)): + raise SystemExit( + f"Environment '{name}/{category}' lists a tool more than once" + ) + for tool in names: + if tool not in tools: + raise SystemExit(f"Unknown tool '{tool}' in environment '{name}'") + if tools[tool]["provider"] not in categories[category]: + raise SystemExit( + f"Wrong provider for '{tool}' in environment '{name}/{category}'" + ) + if (category == "build-apt") != ( + tools[tool].get("phase") == "build-only" + ): + raise SystemExit( + f"Wrong installation phase for '{tool}' in environment '{name}'" + ) + # Dependencies can be in another group: CodeQL needs zstd during the + # build, while Python tools need uv from the native tool selection. + selected = {tool for names in selection.values() for tool in names} + for tool in selected: + for dependency in tools[tool].get("requires", []): + if dependency not in selected: + raise SystemExit( + f"Tool '{tool}' requires '{dependency}' in environment '{name}'" + ) + return data + + +def apt_requirements( + selection: dict, definitions: dict, *, recommends: bool +) -> list[str]: + """Build apt package arguments for the chosen recommendation setting. + + Keep version strings intact, including Debian's optional prefix such as "1:".""" + requirements = [] + for name in selection.get("apt", []): + tool = definitions[name] + if tool.get("install_recommends", True) != recommends: + continue + for package in tool["packages"]: + # Some releases are selected by package name, such as python3.12. + if tool["policy"] == "package-name": + package = package.format(version=tool["version"]) + # Include Ubuntu's package revisions, e.g. graphviz=2.42.2*. + elif tool["policy"] == "prefix": + package += "=" + tool["version"] + "*" + requirements.append(package) + return requirements diff --git a/tools/internal/devcontainer/README.md b/tools/internal/devcontainer/README.md new file mode 100644 index 00000000..f1dd0352 --- /dev/null +++ b/tools/internal/devcontainer/README.md @@ -0,0 +1,101 @@ + + +# Installing tools in DevContainers + +The feature entry scripts and repository post-create script each request one +selection from [`environments.json`](../../environments.json): + +| Selection | Caller | +| --- | --- | +| `s-core-local` | The published image's `s-core-local` feature | +| `bazel-feature` | The published image's `bazel-feature` feature | +| `repository-devcontainer` | This repository's post-create setup | + +These lists record the existing callers' tool choices. A future image can add +its own selection. Base-image contents and external Dev Container features keep +their existing configuration owners; transitive packages are not listed here. +Bazel's public command targets are independent of these image selections. + +```console +$ python3 tools/internal/devcontainer/install.py install-environment s-core-local +``` + +## Installation and permissions + +One call validates the complete selection, adds temporary build packages, +installs tools and runs their setup, then removes the temporary packages. +Unknown names, missing direct dependencies and unavailable platform artifacts +fail before installation begins. A later download or installation failure stops +the sequence; installation is not transactional. + +The installer uses `sudo` for system installation when needed and keeps npm in +the calling user's Node environment. Image builds already running as root +install directly. Python tools use the pinned uv and shared tool environments; +entrypoints in `/usr/local/bin` are available to all container users. + +`--phase build|runtime|cleanup` requests an individual phase. `--providers` +restricts the installation methods applied. Native-, Python- and npm-only calls +skip APT build and cleanup steps. Writable native/Python destinations do not +require `sudo`. `--destination` and `--tool-dir` select these directories. +`--os` and `--cpu` select artifact metadata; they do not emulate apt or uv. + +## Definitions and setup + +| Group in a selection | Definition owner | Installation | +| --- | --- | --- | +| `native` | `lockfiles/*.lock.json` | Download and verify the release artifact | +| `python` | `providers.json` | Install the pinned Python package with uv | +| `apt` | `providers.json` | Install the declared packages using their version policy | +| `build-apt` | `providers.json` | Add temporary packages for build/setup; remove afterward | +| `npm` | `providers.json` | Install npm packages in the caller's environment | +| `setup` | `providers.json` | Run the declared tool-specific setup | + +Definitions can name a script in [`setup/`](setup) with `setup_script`. This also +applies to APT tools: OpenJDK needs `java.sh` to configure `JAVA_HOME` after +installation. Each script explains its purpose and inputs at the top. +`install.py` passes metadata as environment variables, not command-line flags. +Scripts can query `selected ENVIRONMENT TOOL` and `items TOOL FIELD` for optional +bundle membership. CodeQL, for example, selects its C/C++ packs from the catalog +and declares zstd as a direct build dependency. + +A child process cannot set variables in the shell that launched it. Java and +CodeQL therefore write profile files for later terminals. Feature tests use +`eval "$(.../install.py shell-metadata)"` to read quoted version assignments from +all catalogs in a single call. + +## APT policies and platform limits + +APT `distribution` follows Ubuntu's selected version; `prefix` includes any +Debian epoch (for example `1:3.22.0`); `package-name` chooses a versioned package +such as `python3.12`. Compatibility expectations for Git, Git LFS and gh are +separate from their unpinned APT requests. Black and other APT-provided Python +CLIs remain distribution-selected; migrating them to uv needs deliberate pins +and removal of the APT copies. + +`packages` and `platforms` are independent lists. All listed packages are +installed on each supported host; apt selects the host build. Both QEMU guest +emulators, for example, are installed on both Linux host CPUs. CodeQL explicitly +omits Linux ARM64 because its selected bundle has no artifact for that platform. + +`ca-certificates-java` populates Java's trust store from system certificates. +Pinning the helper alone would not freeze CA data or imported company CAs. +These inputs continue to follow distribution updates; this catalog does not +provide base-image, APT or certificate snapshots. + +Removing a tool from a selection stops offering it in new setups. The installer +does not uninstall previous host copies. Return to the +[tool maintenance guide](../README.md) for adding commands and Bazel targets. diff --git a/tools/internal/devcontainer/install.py b/tools/internal/devcontainer/install.py index ee3e2a28..60f11a05 100755 --- a/tools/internal/devcontainer/install.py +++ b/tools/internal/devcontainer/install.py @@ -19,6 +19,7 @@ Usage: install.py install shellcheck yamlfmt + install.py install-environment s-core-local install.py install-python pre-commit install.py version shellcheck """ @@ -46,8 +47,12 @@ from catalog import ( Binary, ToolData, + all_definitions, + apt_requirements, load_catalog_versions, + load_environments, load_python_tools, + load_providers, load_tool, resolve_lockfile, ) @@ -209,11 +214,19 @@ def _cmd_install(args: argparse.Namespace) -> int: """Download, verify, and install tools from the lockfile catalog.""" dest_dir = Path(args.destination) + # Check every requested tool has a download for this OS and CPU first. + # A bad name late in the list must not leave earlier tools installed. + binaries: list[tuple[str, Binary]] = [] for tool in args.tools: lockfile = resolve_lockfile(tool, args.lockfile) tool_data = load_tool(lockfile, tool) - binary = _select_binary(tool_data, args.os, args.cpu) + try: + binary = _select_binary(tool_data, args.os, args.cpu) + except SystemExit as exc: + raise SystemExit(f"Tool '{tool}': {exc}") from exc + binaries.append((tool, binary)) + for tool, binary in binaries: kind = binary["kind"] url = binary["url"] expected_sha256 = binary["sha256"] @@ -254,12 +267,264 @@ def _cmd_install(args: argparse.Namespace) -> int: return 0 +def _is_writable(path: str) -> bool: + """Check the nearest existing parent when an install directory is still missing.""" + directory = Path(path).resolve() + while not directory.exists(): + directory = directory.parent + return os.access(directory, os.W_OK) + + +def _delegate_system_installation( + args: argparse.Namespace, selection: dict[str, list[str]], setup_scripts: list[Path] +) -> bool: + """Return whether system installation was completed by a privileged child.""" + if os.geteuid() == 0: + return False + providers = [ + provider + for provider in args.providers + if provider != "npm" + and (selection.get(provider) or (provider == "setup" and setup_scripts)) + ] + if args.phase != "runtime": + providers = ["apt"] + needs_root = bool(selection.get("build-apt")) + else: + needs_root = ( + any(provider in providers for provider in ("apt", "setup")) + or ("native" in providers and not _is_writable(args.destination)) + or ( + "python" in providers + and not all(_is_writable(p) for p in (args.destination, args.tool_dir)) + ) + ) + if not needs_root: + return False + + # Reuse the same installer and explicit paths under sudo. Do not elevate the + # whole invocation: npm must retain the user's PATH and global package prefix. + subprocess.run( + [ + "sudo", + sys.executable, + str(Path(__file__).resolve()), + "install-environment", + args.environment, + "--phase", + args.phase, + "--destination", + str(Path(args.destination).resolve()), + "--tool-dir", + str(Path(args.tool_dir).resolve()), + "--os", + args.os, + "--cpu", + args.cpu, + "--providers", + *providers, + ], + check=True, + ) + return True + + +def _cmd_install_environment(args: argparse.Namespace) -> int: + """Install an environment's tools, or add/remove its temporary build packages.""" + selection = load_environments().get(args.environment) + if selection is None: + raise SystemExit(f"Unknown tool environment '{args.environment}'") + definitions = all_definitions() + system = args.os + "/" + args.cpu + # Even the build phase checks tools needed by the finished image. Report + # missing downloads before installing packages used only during setup. + for names in selection.values(): + for name in names: + definition = definitions[name] + if definition["provider"] == "native": + try: + _select_binary(definition, args.os, args.cpu) + except SystemExit as exc: + raise SystemExit(f"Tool '{name}': {exc}") from exc + elif definition["provider"] in ("apt", "release", "bazelisk"): + if system not in definition[ + "platforms" + ] and system not in definition.get("omissions", {}): + raise SystemExit(f"Tool '{name}' is unsupported on {system}") + + # Some tools need extra configuration, such as setting JAVA_HOME. Only + # setup scripts shipped next to this installer can be selected. + setup_scripts = [] + setup_root = Path(__file__).resolve().parent / "setup" + for name in selection.get("apt", []) + selection.get("setup", []): + definition = definitions[name] + script = definition.get("setup_script") + if script is None: + continue + if Path(script).name != script or not (setup_root / script).is_file(): + raise SystemExit(f"Invalid setup script for '{name}': {script}") + if system not in definition.get("omissions", {}): + setup_scripts.append(setup_root / script) + + if args.phase == "all": + phases = ("build", "runtime", "cleanup") + if not {"apt", "setup"}.intersection(args.providers): + phases = ("runtime",) + for phase in phases: + phase_args = argparse.Namespace(**vars(args)) + phase_args.phase = phase + _apply_environment(phase_args, selection, definitions, setup_scripts) + return 0 + return _apply_environment(args, selection, definitions, setup_scripts) + + +def _apply_environment(args, selection, definitions, setup_scripts) -> int: + """Apply one validated installation phase, elevating system steps when needed.""" + if args.phase != "runtime" and not {"apt", "setup"}.intersection(args.providers): + return 0 + if _delegate_system_installation(args, selection, setup_scripts): + if args.phase != "runtime": + return 0 + # System groups are complete; npm still runs with the caller's Node setup. + args.providers = [p for p in args.providers if p == "npm"] + + # Build packages, such as zstd for unpacking CodeQL, are added before tool + # installation and removed after all setup steps have finished. + if args.phase != "runtime": + packages = [ + package + for name in selection.get("build-apt", []) + for package in definitions[name]["packages"] + ] + if packages: + action = ( + ["install", "-y"] + if args.phase == "build" + else ["remove", "--purge", "-y"] + ) + subprocess.run(["apt-get", *action, *packages], check=True) + return 0 + + # APT normally installs recommended extra packages. QEMU opts out; the + # other packages keep that default, including Python build dependencies. + for recommends in (True, False) if "apt" in args.providers else (): + packages = apt_requirements(selection, definitions, recommends=recommends) + if packages: + flags = [] if recommends else ["--no-install-recommends", "--fix-broken"] + subprocess.run(["apt-get", "install", "-y", *flags, *packages], check=True) + # Install uv as a native tool before using it to install Python tools. + if "native" in args.providers: + args.tools = selection.get("native", []) + args.lockfile = None + _cmd_install(args) + if "python" in args.providers and selection.get("python"): + args.tools = selection["python"] + args.uv = str(Path(args.destination).resolve() / "uv") + args.bin_dir = args.destination + _cmd_install_python(args) + for name in selection.get("npm", []) if "npm" in args.providers else (): + subprocess.run( + ["npm", "install", "-g", definitions[name]["package"]], check=True + ) + # Pass variables to the setup processes. Python cannot change the + # environment of the shell that launched this installer. + if "setup" in args.providers and setup_scripts: + environment = {**os.environ, **_metadata_values()} + environment["SCORE_TOOL_ENVIRONMENT"] = args.environment + environment["SCORE_TOOLS_INSTALLER"] = str(Path(__file__).resolve()) + for script in setup_scripts: + subprocess.run(["bash", str(script)], check=True, env=environment) + return 0 + + +def _metadata_values() -> dict[str, str]: + """Turn definition fields into variables such as codeql_version for setup and tests.""" + import re + + values: dict[str, str] = {} + + def emit(name, value): + if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name): + raise SystemExit(f"Invalid metadata variable: {name}") + if isinstance(value, dict): + for key, child in value.items(): + emit(name + "_" + key, child) + elif isinstance(value, str): + values[name] = value + + for name, definition in load_providers().items(): + # Use only version and checksum fields. Other data, such as platform + # names containing slashes, cannot be used as shell variable names. + for field in ("version", "compatibility_version", "amd64", "arm64"): + if field in definition: + emit(name.replace("-", "_") + "_" + field, definition[field]) + # Unpinned APT tools can still have expected versions for feature tests. + # Give these expectations the same _version names as pinned tools. + if "compatibility_version" in definition: + emit( + name.replace("-", "_") + "_version", definition["compatibility_version"] + ) + # Native and Python command versions use the same variable names as the + # other definitions, so feature tests need just one metadata query. + for name, version in load_catalog_versions().items(): + emit(name.replace("-", "_") + "_version", version) + # Python package and entrypoint metadata is also available to setup scripts. + for name, definition in load_python_tools().items(): + emit(name.replace("-", "_"), definition) + return values + + +def _cmd_shell_metadata(args: argparse.Namespace) -> int: + """Print shell-quoted exports so feature tests can read versions in their shell.""" + import shlex + + for name, value in _metadata_values().items(): + print("export " + name + "=" + shlex.quote(value)) + return 0 + + +def _cmd_selected(args: argparse.Namespace) -> int: + """Exit with 0 if the environment selects this tool, or 1 if it does not.""" + selections = load_environments()[args.environment] + return 0 if any(args.tool in names for names in selections.values()) else 1 + + +def _cmd_items(args: argparse.Namespace) -> int: + """Print a tool definition's list, such as CodeQL packs, one item per line.""" + value = load_providers()[args.tool][args.field] + if ( + not isinstance(value, list) + or not value + or any(not isinstance(item, str) or not item or "\n" in item for item in value) + ): + raise SystemExit( + f"{args.tool}/{args.field} must be a non-empty list of single-line strings" + ) + print("\n".join(value)) + return 0 + + def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Install pinned tools from multitool-compatible lockfiles.", ) subparsers = parser.add_subparsers(dest="command", required=True) + metadata_parser = subparsers.add_parser( + "shell-metadata", help="Read feature setup metadata." + ) + metadata_parser.set_defaults(func=_cmd_shell_metadata) + selected_parser = subparsers.add_parser( + "selected", help="Query membership for feature-specific setup." + ) + selected_parser.add_argument("environment") + selected_parser.add_argument("tool") + selected_parser.set_defaults(func=_cmd_selected) + items_parser = subparsers.add_parser("items", help="Read a setup bundle's members.") + items_parser.add_argument("tool") + items_parser.add_argument("field") + items_parser.set_defaults(func=_cmd_items) + install_parser = subparsers.add_parser( "install", help="Download, verify, and install tools.", @@ -271,9 +536,33 @@ def _build_parser() -> argparse.ArgumentParser: install_parser.add_argument("--cpu", default=_detect_cpu()) install_parser.set_defaults(func=_cmd_install) + environment_parser = subparsers.add_parser( + "install-environment", + help="Apply a tool selection from tools/environments.json.", + ) + environment_parser.add_argument("environment") + environment_parser.add_argument( + "--providers", + nargs="+", + choices=("apt", "native", "python", "npm", "setup"), + default=("apt", "native", "python", "npm", "setup"), + help="Apply only these provider groups; installation rights are handled automatically.", + ) + environment_parser.add_argument( + "--phase", + choices=("all", "runtime", "build", "cleanup"), + default="all", + help="Run all installation phases by default, or apply just one phase.", + ) + environment_parser.add_argument("--tool-dir", default="/usr/local/share/uv/tools") + environment_parser.add_argument("--destination", default="/usr/local/bin") + environment_parser.add_argument("--os", default=_detect_os()) + environment_parser.add_argument("--cpu", default=_detect_cpu()) + environment_parser.set_defaults(func=_cmd_install_environment) + install_python_parser = subparsers.add_parser( "install-python", - help="Install Python command-line tools declared in python_tools.bzl.", + help="Install Python command-line tools declared in providers.json.", ) install_python_parser.add_argument( "tools", diff --git a/tools/internal/devcontainer/load_feature_versions.sh b/tools/internal/devcontainer/load_feature_versions.sh deleted file mode 100755 index bc2ce333..00000000 --- a/tools/internal/devcontainer/load_feature_versions.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -set -euo pipefail - -ARCHITECTURE=$(dpkg --print-architecture) -VERSION="v4.47.2" - -SHA256_FIELD="1bb99e1019e23de33c7e6afc23e93dad72aad6cf2cb03c797f068ea79814ddb0" # Default to amd64 -if [ "${ARCHITECTURE}" = "arm64" ]; then - SHA256_FIELD="05df1f6aed334f223bb3e6a967db259f7185e33650c3b6447625e16fea0ed31f" -fi - -yq_binary="$(mktemp)" -trap 'rm -f "${yq_binary}"' EXIT - -curl --fail --location "https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_linux_${ARCHITECTURE}" -o "${yq_binary}" -echo "${SHA256_FIELD} ${yq_binary}" | sha256sum -c - || exit 1 -chmod +x "${yq_binary}" - -# Read tool versions and metadata into environment variables -export $("${yq_binary}" eval '.. | select((tag == "!!map" or tag == "!!seq") | not) | (path | join("_")) + "=" + .' "$1" | awk '!/=$/{print }' | xargs) diff --git a/tools/internal/devcontainer/setup/bazel.sh b/tools/internal/devcontainer/setup/bazel.sh new file mode 100755 index 00000000..0597bde1 --- /dev/null +++ b/tools/internal/devcontainer/setup/bazel.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Make Bazel available through Bazelisk, pre-download the chosen release for the +# development user, and configure completion and the Java trust store. +# install.py starts this script with bazel_version as an environment variable. +# shellcheck disable=SC2154 +set -euo pipefail + +# Let the bazel command use Bazelisk to select and download the requested version. +ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel + +# Download the chosen Bazel release into the development user's cache and save +# shell completion for later terminals. +export USE_BAZEL_VERSION=${bazel_version} +# Run as the development user so their first Bazel call can reuse this download. +su $(ls /home) -c "bazel help completion bash > /tmp/bazel-complete.bash" +mkdir -p /etc/bash_completion.d +mv /tmp/bazel-complete.bash /etc/bash_completion.d/bazel-complete.bash +sh -c "echo 'INSTALLED_BAZEL_VERSION=${bazel_version}' >> /devcontainer/features/bazel/bazel_setup.sh" + +# Make Bazel trust the same certificates as the OS, including company CAs. +echo 'startup --host_jvm_args=-Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts --host_jvm_args=-Djavax.net.ssl.trustStorePassword=changeit' >> /etc/bazel.bazelrc diff --git a/tools/internal/devcontainer/setup/bazel_compile_commands.sh b/tools/internal/devcontainer/setup/bazel_compile_commands.sh new file mode 100755 index 00000000..12f7a232 --- /dev/null +++ b/tools/internal/devcontainer/setup/bazel_compile_commands.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Install the helper that generates compile_commands.json for Bazel projects. +# install.py supplies the release version and per-CPU checksums as environment +# variables: bazel_compile_commands_version and bazel_compile_commands_*_sha256. +# shellcheck disable=SC2154 +set -euo pipefail + +ARCHITECTURE=$(dpkg --print-architecture) + +# Install the helper that writes compile_commands.json for editors using Bazel. +# (see https://github.com/kiron1/bazel-compile-commands) +source /etc/lsb-release +curl -L "https://github.com/kiron1/bazel-compile-commands/releases/download/v${bazel_compile_commands_version}/bazel-compile-commands_${bazel_compile_commands_version}-${DISTRIB_CODENAME}_${ARCHITECTURE}.deb" -o /tmp/bazel-compile-commands.deb +# Check the checksum for this CPU before installing the downloaded package. +SHA256SUM="${bazel_compile_commands_amd64_sha256}" +if [ "${ARCHITECTURE}" = "arm64" ]; then + SHA256SUM="${bazel_compile_commands_arm64_sha256}" +fi +echo "${SHA256SUM} /tmp/bazel-compile-commands.deb" | sha256sum -c - || exit 1 +apt-get install -y --no-install-recommends --fix-broken /tmp/bazel-compile-commands.deb +rm /tmp/bazel-compile-commands.deb diff --git a/tools/internal/devcontainer/setup/codeql.sh b/tools/internal/devcontainer/setup/codeql.sh new file mode 100755 index 00000000..5c54ae9c --- /dev/null +++ b/tools/internal/devcontainer/setup/codeql.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Install the CodeQL bundle and selected C/C++ query packs, configure CODEQL_HOME +# for later terminals, and remove unused language tools to reduce image size. +# install.py supplies codeql_version, checksums and query-pack versions as +# environment variables. SCORE_TOOLS_INSTALLER and SCORE_TOOL_ENVIRONMENT let +# this script query the selected pack bundle from the same tool definitions. +# shellcheck disable=SC2154 +set -euo pipefail + +ARCHITECTURE=$(dpkg --print-architecture) +if [ "${ARCHITECTURE}" != "amd64" ]; then + echo "No CodeQL bundle selected for Linux ${ARCHITECTURE}" >&2 + exit 1 +fi +VARIANT=linux64 +SHA256SUM="${codeql_amd64_sha256}" + +codeql_install_dir="/usr/local" +curl -L "https://github.com/github/codeql-action/releases/download/codeql-bundle-v${codeql_version}/codeql-bundle-${VARIANT}.tar.zst" -o /tmp/codeql.tar.zst +echo "${SHA256SUM} /tmp/codeql.tar.zst" | sha256sum -c - || exit 1 +tar -I zstd -xf /tmp/codeql.tar.zst -C "${codeql_install_dir}" +ln -s "${codeql_install_dir}"/codeql/codeql /usr/local/bin/codeql +rm /tmp/codeql.tar.zst +export CODEQL_HOME=${codeql_install_dir}/codeql +echo "export CODEQL_HOME=${codeql_install_dir}/codeql" > /etc/profile.d/codeql.sh + +if "${SCORE_TOOLS_INSTALLER}" selected "${SCORE_TOOL_ENVIRONMENT}" codeql_coding_standards; then + codeql_packs="$("${SCORE_TOOLS_INSTALLER}" items codeql_coding_standards packs)" + while IFS= read -r pack; do + codeql pack download "${pack}@${codeql_coding_standards_version}" -d "${codeql_install_dir}/codeql/qlpacks/" + done <<< "${codeql_packs}" +fi + +# Remove unused language tools and query packs to keep the image smaller. +codeql_purge_dirs=( + "${codeql_install_dir}/codeql/csharp" + "${codeql_install_dir}/codeql/go" + "${codeql_install_dir}/codeql/java" + "${codeql_install_dir}/codeql/javascript" + "${codeql_install_dir}/codeql/python" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/csharp-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/go-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/java-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/javascript-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/python-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-examples" + "${codeql_install_dir}/codeql/qlpacks/codeql/ruby-queries" + "${codeql_install_dir}/codeql/qlpacks/codeql/swift-all" + "${codeql_install_dir}/codeql/qlpacks/codeql/swift-queries" + "${codeql_install_dir}/codeql/ruby" + "${codeql_install_dir}/codeql/swift" +) +for dir in "${codeql_purge_dirs[@]}"; do + if [ -d "${dir}" ]; then + rm -rf "${dir}" + fi +done diff --git a/tools/internal/devcontainer/setup/java.sh b/tools/internal/devcontainer/setup/java.sh new file mode 100755 index 00000000..e7ad931c --- /dev/null +++ b/tools/internal/devcontainer/setup/java.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Configure JAVA_HOME for later terminals after APT has installed the JDK. +# install.py runs this script for the selected OpenJDK definition; no arguments +# or version variables are needed because javac identifies the installed JDK. + +set -euo pipefail + +# Make JAVA_HOME available in later terminals so tools such as Bazel can find +# Java. Derive the path from the installed javac, not a release number. +JAVA_HOME="$(dirname "$(dirname "$(realpath "$(command -v javac)")")")" +export JAVA_HOME +echo -e "JAVA_HOME=${JAVA_HOME}\nexport JAVA_HOME" > /etc/profile.d/java_home.sh diff --git a/tools/internal/sync_readme.py b/tools/internal/sync_readme.py index 3f7aae77..b89edf90 100755 --- a/tools/internal/sync_readme.py +++ b/tools/internal/sync_readme.py @@ -11,7 +11,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Synchronize the documented tool table with the lockfile catalog.""" +"""Update the documented tool table and check environment selections.""" from __future__ import annotations @@ -21,7 +21,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) -from catalog import load_catalog_descriptions, load_catalog_versions +from catalog import load_catalog_descriptions, load_catalog_versions, load_environments TOOLS_DIR = Path(__file__).resolve().parent.parent @@ -67,6 +67,9 @@ def main(argv: list[str] | None = None) -> int: ) parser.parse_args(argv) + # The pre-commit documentation check also catches unknown tools and missing + # dependencies in environments.json, even when the table does not change. + load_environments() current = README_PATH.read_text(encoding="utf-8") table = _render_table(load_catalog_versions(), load_catalog_descriptions()) expected = _updated_readme(current, table) diff --git a/tools/lockfiles/python_tools.bzl b/tools/lockfiles/python_tools.bzl deleted file mode 100644 index 262d40b0..00000000 --- a/tools/lockfiles/python_tools.bzl +++ /dev/null @@ -1,30 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -# A literal Starlark dictionary lets Bazel consume the pins during analysis -# while ast.literal_eval preserves a data-only boundary for the root-running -# DevContainer installer. -PYTHON_TOOLS = { - # The key is the public command and Bazel target name. - "pre-commit": { - # `package` is resolved by uv; `entrypoint` is the executable exposed to - # users. They are separate because Python distributions may name them - # differently. - "package": "pre-commit", - "version": "4.7.0", - "entrypoint": "pre-commit", - # Documentation is generated from the catalog so version and purpose - # are reviewed together. - "description": "Run repository pre-commit hooks", - }, -} diff --git a/tools/providers.json b/tools/providers.json new file mode 100644 index 00000000..a21b73fe --- /dev/null +++ b/tools/providers.json @@ -0,0 +1,328 @@ +{ + "graphviz": { + "provider": "apt", + "packages": ["graphviz"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "2.42.2" + }, + "protobuf_compiler": { + "provider": "apt", + "packages": ["protobuf-compiler"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "3.21.12" + }, + "qemu_system": { + "provider": "apt", + "packages": ["qemu-system-arm", "qemu-system-x86"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1:8.2.2", + "install_recommends": false, + "description": "Both ARM and x86 guest emulators are installed on each supported host platform." + }, + "sshpass": { + "provider": "apt", + "packages": ["sshpass"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.09" + }, + "gdb": { + "provider": "apt", + "packages": ["gdb"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "15.0.50" + }, + "git": { + "provider": "apt", + "packages": ["git"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "2.43.0" + }, + "git_lfs": { + "provider": "apt", + "packages": ["git-lfs"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "3.4.1" + }, + "gh": { + "provider": "apt", + "packages": ["gh"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "compatibility_version": "2.45.0" + }, + "openjdk": { + "provider": "apt", + "packages": ["openjdk-21-jdk-headless"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "21.0.12", + "setup_script": "java.sh" + }, + "valgrind": { + "provider": "apt", + "packages": ["valgrind"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1:3.22.0" + }, + "ripgrep": { + "provider": "apt", + "packages": ["ripgrep"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "14.1.0" + }, + "lcov": { + "provider": "apt", + "packages": ["lcov"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "2.0" + }, + "pkg_config": { + "provider": "apt", + "packages": ["pkg-config"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.8.1" + }, + "libcairo2_dev": { + "provider": "apt", + "packages": ["libcairo2-dev"], + "policy": "prefix", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "1.18.0" + }, + "python": { + "provider": "apt", + "description": "Distribution Python interpreter", + "packages": ["python{version}"], + "policy": "package-name", + "version": "3.12", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "flake8": { + "provider": "apt", + "packages": ["flake8"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-autopep8": { + "provider": "apt", + "packages": ["python3-autopep8"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "black": { + "provider": "apt", + "packages": ["black"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "description": "Ubuntu-provided Black; distribution policy does not pin an exact version." + }, + "python3-yapf": { + "provider": "apt", + "packages": ["python3-yapf"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "mypy": { + "provider": "apt", + "packages": ["mypy"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pydocstyle": { + "provider": "apt", + "packages": ["pydocstyle"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pycodestyle": { + "provider": "apt", + "packages": ["pycodestyle"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "bandit": { + "provider": "apt", + "packages": ["bandit"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pipenv": { + "provider": "apt", + "packages": ["pipenv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "virtualenv": { + "provider": "apt", + "packages": ["virtualenv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pylint": { + "provider": "apt", + "packages": ["pylint"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "ca-certificates-java": { + "provider": "apt", + "packages": ["ca-certificates-java"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "description": "Populate the Java trust store from system CA certificates; follow distribution updates." + }, + "man-db": { + "provider": "apt", + "packages": ["man-db"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages": { + "provider": "apt", + "packages": ["manpages"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-dev": { + "provider": "apt", + "packages": ["manpages-dev"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-posix": { + "provider": "apt", + "packages": ["manpages-posix"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "manpages-posix-dev": { + "provider": "apt", + "packages": ["manpages-posix-dev"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "codeql": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64"], + "version": "2.21.4", + "amd64": { + "sha256": "6da9f3df228d4e162af877569240a627c1af22f6adbd26352b22357fb8bd7545" + }, + "omissions": { + "linux/arm64": "The selected bundle has no supported Linux arm64 artifact" + }, + "setup_script": "codeql.sh", + "requires": ["zstd"] + }, + "codeql_coding_standards": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64"], + "version": "2.54.0", + "requires": ["codeql"], + "omissions": { + "linux/arm64": "Not installed without the CodeQL bundle" + }, + "packs": [ + "codeql/misra-cpp-coding-standards", + "codeql/misra-c-coding-standards", + "codeql/cert-cpp-coding-standards", + "codeql/cert-c-coding-standards" + ] + }, + "bazel": { + "provider": "bazelisk", + "policy": "exact", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "8.6.0", + "requires": ["bazelisk"], + "setup_script": "bazel.sh" + }, + "bazel_compile_commands": { + "provider": "release", + "policy": "exact", + "platforms": ["linux/x86_64", "linux/arm64"], + "version": "0.18.0", + "amd64": { + "sha256": "6735ea846241497094719792ad3d4f67b1d3123048693d34fcdf7190f8c2da4e" + }, + "arm64": { + "sha256": "d73998efa01cbd501b82ad6266642464b78ecd9fc6224a60c9cb558182d52d88" + }, + "setup_script": "bazel_compile_commands.sh" + }, + "apt-transport-https": { + "provider": "apt", + "description": "Temporary container build dependency (removed after setup)", + "packages": ["apt-transport-https"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "phase": "build-only" + }, + "zstd": { + "provider": "apt", + "description": "Temporary container build dependency (removed after setup)", + "packages": ["zstd"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"], + "phase": "build-only" + }, + "devcontainers-cli": { + "provider": "npm", + "description": "Build and test Dev Containers", + "package": "@devcontainers/cli", + "policy": "latest", + "platforms": ["Node.js-supported platforms"], + "phase": "repository setup" + }, + "python3-pip": { + "provider": "apt", + "description": "Distribution pip (independent of the interpreter package-name constraint)", + "packages": ["python3-pip"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "python3-venv": { + "provider": "apt", + "description": "Distribution Python venv support", + "packages": ["python3-venv"], + "policy": "distribution", + "platforms": ["linux/x86_64", "linux/arm64"] + }, + "pre-commit": { + "provider": "python", + "package": "pre-commit", + "version": "4.7.0", + "entrypoint": "pre-commit", + "description": "Run repository pre-commit hooks", + "requires": ["uv"] + }, + "basedpyright": { + "provider": "python", + "package": "basedpyright", + "version": "1.35.0", + "entrypoint": "basedpyright", + "description": "Type check Python and provide its language server", + "requires": ["uv"] + }, + "pytest": { + "provider": "python", + "package": "pytest", + "version": "9.0.2", + "entrypoint": "pytest", + "description": "Run Python tests", + "requires": ["uv"] + } +} diff --git a/tools/python_tool.bzl b/tools/python_tool.bzl index 0ff4c56b..9e0969dd 100644 --- a/tools/python_tool.bzl +++ b/tools/python_tool.bzl @@ -18,7 +18,7 @@ shell and the pinned uvx executable supplied by rules_multitool. """ load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load(":lockfiles/python_tools.bzl", "PYTHON_TOOLS") +load("@score_tool_catalog//:python_tools.bzl", "PYTHON_TOOLS") def python_tool(name): """Creates a runnable target from the shared Python tool catalog.""" @@ -26,7 +26,7 @@ def python_tool(name): # Fail during analysis so a misspelled target cannot degrade into an # unpinned uvx invocation at runtime. if name not in PYTHON_TOOLS: - fail("Python tool '{}' is not defined in python_tools.bzl".format(name)) + fail("Python tool '{}' is not defined in providers.json".format(name)) tool = PYTHON_TOOLS[name] @@ -51,3 +51,12 @@ def python_tool(name): # its version and platform selection. data = [":uvx_binary"], ) + +def python_tools(): + """Expose every catalogued Python command as a public Bazel target.""" + + # Environment selections control image contents. The module API includes + # every command in the catalog so removing a tool from one image does not + # remove a public target used by other repositories. + for name in PYTHON_TOOLS: + python_tool(name) diff --git a/tools/tests/catalog_test.py b/tools/tests/catalog_test.py index 66310650..c747ca21 100644 --- a/tools/tests/catalog_test.py +++ b/tools/tests/catalog_test.py @@ -14,7 +14,6 @@ from __future__ import annotations -import ast import json import shutil import subprocess @@ -49,10 +48,17 @@ def test_metadata_matches_definitions(self) -> None: if "description" in metadata: descriptions[name] = metadata["description"] - source = (TOOLS_DIR / "lockfiles/python_tools.bzl").read_text(encoding="utf-8") - python_tools = ast.literal_eval(ast.parse(source).body[0].value) + providers = json.loads((TOOLS_DIR / "providers.json").read_text()) + python_tools = { + name: { + field: definition[field] + for field in ("package", "version", "entrypoint", "description") + } + for name, definition in providers.items() + if definition["provider"] == "python" + } self.assertEqual(catalog.load_python_tools(), python_tools) - self.assertEqual(set(python_tools), {"pre-commit"}) + self.assertEqual(set(python_tools), {"pre-commit", "basedpyright", "pytest"}) versions.update( (name, metadata["version"]) for name, metadata in python_tools.items() ) @@ -64,38 +70,42 @@ def test_metadata_matches_definitions(self) -> None: self.assertEqual(catalog.resolve_lockfile("uvx"), "uv") self.assertEqual(catalog.resolve_lockfile("uvx", "explicit"), "explicit") - def test_rejects_malformed_and_executable_python_catalogs(self) -> None: + def test_rejects_python_packages_with_missing_fields_or_uv_dependency(self) -> None: + valid = { + "provider": "python", + "package": "example", + "version": "1", + "entrypoint": "example", + "description": "Example tool", + "requires": ["uv"], + } with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - path = root / "python_tools.bzl" - marker = root / "executed" - payload = f"__import__('pathlib').Path({str(marker)!r}).touch()" - cases = [ - ("PYTHON_TOOLS = {", "Malformed Python tool catalog"), - ("OTHER = {}", "must assign its data to PYTHON_TOOLS"), - ("PYTHON_TOOLS = []", "PYTHON_TOOLS must be a dictionary"), - ( - "PYTHON_TOOLS = {'bad': {'version': '1'}}", - "Malformed Python tool catalog entry", - ), - (f"PYTHON_TOOLS = {payload}", "must contain only literal data"), - ( - f"PYTHON_TOOLS = {{}}\n{payload}", - "must contain only a PYTHON_TOOLS assignment", - ), - ] - with patch.object(catalog, "PYTHON_TOOL_CATALOG", path): - for source, message in cases: - with self.subTest(source=source): - path.write_text(source, encoding="utf-8") + path = Path(directory) / "providers.json" + with patch.object(catalog, "PROVIDERS_PATH", path): + for field in ( + "package", + "version", + "entrypoint", + "description", + "requires", + ): + with self.subTest(field=field): + definition = { + key: value for key, value in valid.items() if key != field + } + path.write_text(json.dumps({"example": definition})) + message = ( + "uv dependency" + if field == "requires" + else "Malformed Python package definition" + ) with self.assertRaisesRegex(SystemExit, message): catalog.load_python_tools() - self.assertFalse(marker.exists()) def test_duplicate_definitions_retain_rejection(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) - python_path = root / "python_tools.bzl" + python_path = root / "providers.json" metadata = { "version": "1", "description": "Example tool", @@ -105,6 +115,8 @@ def test_duplicate_definitions_retain_rejection(self) -> None: native_path.write_text(json.dumps({"example": metadata}), encoding="utf-8") duplicate_path = root / "two.lock.json" python_metadata = { + "provider": "python", + "requires": ["uv"], "package": "example", "version": "1", "entrypoint": "example", @@ -112,16 +124,16 @@ def test_duplicate_definitions_retain_rejection(self) -> None: } with ( patch.object(catalog, "LOCKFILE_ROOT", root), - patch.object(catalog, "PYTHON_TOOL_CATALOG", python_path), + patch.object(catalog, "PROVIDERS_PATH", python_path), ): for duplicate in ("lockfiles", "catalogs"): if duplicate == "lockfiles": duplicate_path.write_bytes(native_path.read_bytes()) - python_path.write_text("PYTHON_TOOLS = {}", encoding="utf-8") + python_path.write_text("{}", encoding="utf-8") else: duplicate_path.unlink() python_path.write_text( - f"PYTHON_TOOLS = {{'example': {python_metadata!r}}}", + json.dumps({"example": python_metadata}), encoding="utf-8", ) for query in ( diff --git a/tools/tests/environment_test.py b/tools/tests/environment_test.py new file mode 100644 index 00000000..f614d863 --- /dev/null +++ b/tools/tests/environment_test.py @@ -0,0 +1,756 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Check selected tools are installed and invalid selections fail before installation.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from tools.internal import catalog +from tools.internal.devcontainer import install as installer + +TOOLS_DIR = Path(__file__).resolve().parents[1] + + +class EnvironmentTest(unittest.TestCase): + def test_rejects_invalid_selection_data_and_wrong_providers(self) -> None: + cases = ( + ([], "must be a dictionary"), + ({"": {"native": []}}, "names must be nonempty"), + ({"ci": []}, "Invalid provider categories"), + ({"ci": {"unknown": []}}, "Invalid provider categories"), + ({"ci": {"native": "ruff"}}, "must be a tool list"), + ({"ci": {"native": [7]}}, "must be a tool list"), + ({"ci": {"native": ["ruff", "ruff"]}}, "more than once"), + ({"ci": {"native": ["missing"]}}, "Unknown tool 'missing'"), + ({"ci": {"native": ["pre-commit"]}}, "Wrong provider for 'pre-commit'"), + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "environments.json" + with patch.object(catalog, "ENVIRONMENTS_PATH", path): + for data, message in cases: + with self.subTest(data=data): + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(SystemExit, message): + catalog.load_environments() + + def test_copied_installer_applies_an_independent_ci_selection(self) -> None: + # A new image can declare its own tool list without changing the + # installer. Local files exercise the real download/checksum/install path. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + artifacts = root / "artifacts" + artifacts.mkdir() + metadata = {} + for name in ("lint", "check"): + source = artifacts / name + source.write_text(f"#!/bin/sh\nprintf '%s\\n' '{name} 1.0'\n") + metadata[name] = { + "version": "1.0", + "binaries": [ + { + "os": "linux", + "cpu": "arm64", + "kind": "file", + "url": source.as_uri(), + "sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + } + ], + } + (tools / "lockfiles/ci.lock.json").write_text(json.dumps(metadata)) + (tools / "environments.json").write_text( + json.dumps( + { + "ci": {"native": ["lint", "check"]}, + "unused": {"native": ["ruff"]}, + } + ) + ) + destination = root / "ci tools" + result = self._run(tools, root, "ci", destination) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual({p.name for p in destination.iterdir()}, {"lint", "check"}) + for name in ("lint", "check"): + self.assertEqual( + (destination / name).read_bytes(), (artifacts / name).read_bytes() + ) + self.assertTrue((destination / name).stat().st_mode & 0o111) + + def test_invalid_later_tool_or_platform_prevents_any_download(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + # The first URL deliberately does not exist. If installation starts + # before validating the second entry, this produces the wrong error. + first = { + "version": "1.0", + "binaries": [ + { + "os": "linux", + "cpu": "arm64", + "kind": "file", + "url": (root / "must-not-be-downloaded").as_uri(), + "sha256": "0" * 64, + } + ], + } + unsupported = {"version": "1.0", "binaries": []} + (tools / "lockfiles/fixture.lock.json").write_text( + json.dumps( + { + "first": first, + "unsupported": unsupported, + } + ) + ) + for second, message in ( + ("missing", "Unknown tool 'missing'"), + ("unsupported", "Tool 'unsupported': No binary defined"), + ): + with self.subTest(second=second): + (tools / "environments.json").write_text( + json.dumps( + { + "ci": {"native": ["first", second]}, + } + ) + ) + destination = root / "not-created" + result = self._run(tools, root, "ci", destination) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + self.assertFalse(destination.exists()) + + def test_unknown_environment_is_reported_without_installing(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + destination = root / "not-created" + result = self._run(tools, root, "no-such-environment", destination) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "Unknown tool environment 'no-such-environment'", result.stderr + ) + self.assertFalse(destination.exists()) + + def test_complete_selection_preserves_provider_order_and_package_policies( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + fake_bin = root / "fake-bin" + fake_bin.mkdir() + # Fake apt/npm/uv commands record what would be installed without + # changing host packages. uv itself comes from a checksum-checked file. + recorder = ( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "with open(os.environ['SCORE_TEST_LOG'], 'a') as log:\n" + " log.write(json.dumps({'command': Path(sys.argv[0]).name, " + "'args': sys.argv[1:], 'bin_dir': os.environ.get('UV_TOOL_BIN_DIR'), " + "'tool_dir': os.environ.get('UV_TOOL_DIR'), " + "'privileged': os.environ.get('SCORE_TEST_PRIVILEGED')}) + '\\n')\n" + ) + for name in ("apt-get", "npm"): + executable = fake_bin / name + executable.write_text(recorder) + executable.chmod(0o755) + uv_artifact = root / "uv-artifact" + uv_artifact.write_text(recorder) + (tools / "lockfiles/uv.lock.json").write_text( + json.dumps( + { + "uv": { + "version": "test", + "binaries": [ + { + "os": "linux", + "cpu": "arm64", + "kind": "file", + "url": uv_artifact.as_uri(), + "sha256": hashlib.sha256( + uv_artifact.read_bytes() + ).hexdigest(), + } + ], + }, + } + ) + ) + providers = { + "regular": { + "provider": "apt", + "packages": ["regular"], + "policy": "prefix", + "version": "1:2.3", + "platforms": ["linux/arm64"], + }, + "minimal": { + "provider": "apt", + "packages": ["minimal"], + "policy": "distribution", + "install_recommends": False, + "platforms": ["linux/arm64"], + }, + "build-helper": { + "provider": "apt", + "packages": ["build-helper"], + "policy": "distribution", + "phase": "build-only", + "platforms": ["linux/arm64"], + }, + "cli": {"provider": "npm", "package": "@fixture/cli"}, + } + providers["pre-commit"] = catalog.load_providers()["pre-commit"] + (tools / "providers.json").write_text(json.dumps(providers)) + selection = { + "native": ["uv"], + "python": ["pre-commit"], + "apt": ["regular", "minimal"], + "build-apt": ["build-helper"], + "npm": ["cli"], + } + (tools / "environments.json").write_text(json.dumps({"ci": selection})) + destination = root / "bin" + tool_dir = root / "python tools" + log = root / "calls.jsonl" + environment = dict( + os.environ, + PATH=str(fake_bin) + os.pathsep + os.environ["PATH"], + SCORE_TEST_LOG=str(log), + ) + # One ordinary invocation installs build dependencies, applies tools, + # and removes the temporary dependencies after successful setup. + result = self._run( + tools, + root, + "ci", + destination, + "--tool-dir", + str(tool_dir), + env=environment, + ) + self.assertEqual(result.returncode, 0, result.stderr) + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual( + [(c["command"], c["args"]) for c in calls], + [ + ("apt-get", ["install", "-y", "build-helper"]), + ("apt-get", ["install", "-y", "regular=1:2.3*"]), + ( + "apt-get", + [ + "install", + "-y", + "--no-install-recommends", + "--fix-broken", + "minimal", + ], + ), + ("uv", ["tool", "install", "--force", "pre-commit==4.7.0"]), + ("npm", ["install", "-g", "@fixture/cli"]), + ("apt-get", ["remove", "--purge", "-y", "build-helper"]), + ], + ) + self.assertEqual(calls[3]["bin_dir"], str(destination)) + self.assertEqual(calls[3]["tool_dir"], str(tool_dir)) + # The installer elevates system tools but keeps npm in the user's + # environment. This fixture's sudo records that boundary without root. + expected_privilege = "1" if os.geteuid() != 0 else None + self.assertTrue( + all( + c["privileged"] == expected_privilege + for c in calls + if c["command"] != "npm" + ) + ) + self.assertIsNone(calls[4]["privileged"]) + # A provider filter still permits installing just one group. + log.unlink() + result = self._run( + tools, root, "ci", destination, "--providers", "npm", env=environment + ) + self.assertEqual(result.returncode, 0, result.stderr) + npm_only = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual( + [(c["command"], c["args"]) for c in npm_only], + [("npm", ["install", "-g", "@fixture/cli"])], + ) + # Removing a group from the selection must stop its install commands, + # not just change the displayed tool list. + (tools / "environments.json").write_text( + json.dumps({"ci": {"apt": ["minimal"]}}) + ) + log.unlink() + result = self._run(tools, root, "ci", destination, env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + remaining = [json.loads(line) for line in log.read_text().splitlines()] + self.assertEqual([c["command"] for c in remaining], ["apt-get"]) + self.assertEqual(remaining[0]["args"][-1], "minimal") + + def test_repository_setup_elevates_system_tools_and_keeps_npm_in_caller( + self, + ) -> None: + with ( + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer, "_is_writable", return_value=False), + patch.object(installer, "_cmd_install") as native, + patch.object(installer, "_cmd_install_python") as python, + patch.object(installer.subprocess, "run") as run, + ): + result = installer.main( + [ + "install-environment", + "repository-devcontainer", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + self.assertEqual(result, 0) + self.assertEqual(run.call_count, 2) + system, npm = [call.args[0] for call in run.call_args_list] + self.assertEqual(system[:2], ["sudo", sys.executable]) + self.assertEqual( + system[system.index("--providers") + 1 :], ["native", "python"] + ) + self.assertIn("/usr/local/bin", system) + self.assertIn("/usr/local/share/uv/tools", system) + self.assertEqual(npm, ["npm", "install", "-g", "@devcontainers/cli"]) + native.assert_not_called() + python.assert_not_called() + + def test_root_image_build_does_not_delegate_or_repeat_installation(self) -> None: + with ( + patch.object(installer.os, "geteuid", return_value=0), + patch.object(installer, "_cmd_install") as native, + patch.object(installer, "_cmd_install_python") as python, + patch.object(installer.subprocess, "run") as run, + ): + result = installer.main( + [ + "install-environment", + "repository-devcontainer", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + self.assertEqual(result, 0) + native.assert_called_once() + python.assert_called_once() + run.assert_called_once_with( + ["npm", "install", "-g", "@devcontainers/cli"], check=True + ) + + def test_failed_system_installation_stops_before_npm(self) -> None: + with ( + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer, "_is_writable", return_value=False), + patch.object( + installer.subprocess, + "run", + side_effect=subprocess.CalledProcessError(1, "sudo"), + ) as run, + ): + with self.assertRaises(subprocess.CalledProcessError): + installer.main( + [ + "install-environment", + "repository-devcontainer", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + run.assert_called_once() + + def test_writable_native_destination_does_not_require_sudo(self) -> None: + with tempfile.TemporaryDirectory() as directory: + args = installer._build_parser().parse_args( + [ + "install-environment", + "fixture", + "--destination", + str(Path(directory) / "new" / "bin"), + ] + ) + with ( + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer.subprocess, "run") as run, + ): + self.assertFalse( + installer._delegate_system_installation( + args, {"native": ["ruff"]}, [] + ) + ) + run.assert_not_called() + + def test_default_lifecycle_stops_if_runtime_installation_fails(self) -> None: + with patch.object( + installer, + "_apply_environment", + side_effect=[0, subprocess.CalledProcessError(1, "install")], + ) as apply: + with self.assertRaises(subprocess.CalledProcessError): + installer.main(["install-environment", "repository-devcontainer"]) + self.assertEqual( + [call.args[0].phase for call in apply.call_args_list], ["build", "runtime"] + ) + + def test_shell_metadata_includes_every_native_and_python_command_version( + self, + ) -> None: + # Feature tests should get all expected versions in one shell-metadata call. + metadata = installer._metadata_values() + for name, version in catalog.load_catalog_versions().items(): + self.assertEqual(metadata[name.replace("-", "_") + "_version"], version) + + def test_apt_tool_setup_is_retained_when_installation_is_delegated(self) -> None: + # Java is selected through APT, but also needs java.sh to set JAVA_HOME. + # Both an image build and a normal user must execute that setup. + for uid in (0, 1000): + with ( + self.subTest(uid=uid), + patch.object(installer.os, "geteuid", return_value=uid), + patch.object( + installer, + "load_environments", + return_value={"java-only": {"apt": ["openjdk"]}}, + ), + patch.object(installer.subprocess, "run") as run, + ): + installer.main( + [ + "install-environment", + "java-only", + "--os", + "linux", + "--cpu", + "arm64", + ] + ) + commands = [call.args[0] for call in run.call_args_list] + if uid == 0: + self.assertEqual(commands[0][0], "apt-get") + self.assertEqual(commands[1][0], "bash") + self.assertEqual(Path(commands[1][1]).name, "java.sh") + else: + self.assertEqual(len(commands), 1) + self.assertEqual(commands[0][0], "sudo") + self.assertEqual( + commands[0][commands[0].index("--providers") + 1 :], + ["apt", "setup"], + ) + + def test_native_only_installation_skips_apt_even_if_selection_has_build_packages( + self, + ) -> None: + # s-core-local includes APT build dependencies for special setup. A user + # requesting only native tools must not install or purge those packages. + with tempfile.TemporaryDirectory() as directory: + for phase in ("all", "build", "cleanup"): + with ( + self.subTest(phase=phase), + patch.object(installer.os, "geteuid", return_value=1000), + patch.object(installer, "_cmd_install") as native, + patch.object(installer.subprocess, "run") as run, + ): + installer.main( + [ + "install-environment", + "s-core-local", + "--destination", + directory, + "--os", + "linux", + "--cpu", + "arm64", + "--phase", + phase, + "--providers", + "native", + ] + ) + run.assert_not_called() + self.assertEqual(native.call_count, 1 if phase == "all" else 0) + + def test_python_dependency_and_build_only_phase_are_validated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "environments.json" + with patch.object(catalog, "ENVIRONMENTS_PATH", path): + for data, message in ( + ({"ci": {"python": ["pre-commit"]}}, "requires 'uv'"), + ({"ci": {"apt": ["zstd"]}}, "Wrong installation phase"), + ( + {"ci": {"setup": ["codeql_coding_standards"]}}, + "requires 'codeql'", + ), + ): + with self.subTest(data=data): + path.write_text(json.dumps(data)) + with self.assertRaisesRegex(SystemExit, message): + catalog.load_environments() + + def test_special_setup_uses_membership_and_owned_bundle_items(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + installer = tools / "internal/devcontainer/install.py" + + def run(*args): + return subprocess.run( + [sys.executable, "-I", "-B", str(installer), *args], + cwd=root, + capture_output=True, + text=True, + ) + + self.assertEqual(run("selected", "s-core-local", "codeql").returncode, 0) + selection_path = tools / "environments.json" + selections = json.loads(selection_path.read_text()) + selections["s-core-local"]["setup"] = [] + selection_path.write_text(json.dumps(selections)) + self.assertEqual(run("selected", "s-core-local", "codeql").returncode, 1) + packs = run("items", "codeql_coding_standards", "packs") + self.assertEqual(packs.returncode, 0, packs.stderr) + definitions = json.loads((tools / "providers.json").read_text()) + self.assertEqual( + packs.stdout.splitlines(), + definitions["codeql_coding_standards"]["packs"], + ) + + def test_shell_metadata_keeps_command_substitution_as_literal_data(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + marker = root / "must-not-exist" + value = "$(touch " + str(marker) + ") 'quoted value'" + providers = {"fixture": {"provider": "release", "version": value}} + (tools / "providers.json").write_text(json.dumps(providers)) + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(tools / "internal/devcontainer/install.py"), + "shell-metadata", + ], + cwd=root, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + check = subprocess.run( + [ + "bash", + "-c", + 'eval "$1"; test "$fixture_version" = "$2"', + "metadata-test", + result.stdout, + value, + ], + cwd=root, + capture_output=True, + text=True, + ) + self.assertEqual(check.returncode, 0, check.stderr) + self.assertFalse(marker.exists()) + + def test_missing_codeql_build_dependency_is_rejected_before_build(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + path = tools / "environments.json" + selections = json.loads(path.read_text()) + selections["s-core-local"]["build-apt"].remove("zstd") + path.write_text(json.dumps(selections)) + marker = root / "apt-was-called" + fake_bin = root / "fake-bin" + fake_bin.mkdir() + apt = fake_bin / "apt-get" + apt.write_text(f"#!/bin/sh\ntouch '{marker}'\n") + apt.chmod(0o755) + environment = dict( + os.environ, PATH=str(fake_bin) + os.pathsep + os.environ["PATH"] + ) + result = self._run( + tools, + root, + "s-core-local", + root / "bin", + "--phase", + "build", + env=environment, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Tool 'codeql' requires 'zstd'", result.stderr) + self.assertFalse(marker.exists()) + + def test_declared_setup_receives_metadata_without_changing_the_parent_shell( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tools = self._copy_tools(root) + provider = { + "provider": "release", + "version": "a 'literal' value", + "platforms": ["linux/arm64"], + "setup_script": "fixture.sh", + } + providers_path = tools / "providers.json" + providers_path.write_text(json.dumps({"fixture": provider})) + selections_path = tools / "environments.json" + selections_path.write_text(json.dumps({"ci": {"setup": ["fixture"]}})) + script = tools / "internal/devcontainer/setup/fixture.sh" + script.write_text( + 'printf "%s\\n" "$fixture_version" "$SCORE_TOOL_ENVIRONMENT" > "$SCORE_TEST_OUTPUT"\n' + ) + output = root / "setup-output" + environment = dict(os.environ, SCORE_TEST_OUTPUT=str(output)) + result = self._run(tools, root, "ci", root / "bin", env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + output.read_text().splitlines(), [provider["version"], "ci"] + ) + output.unlink() + # An explicitly unsupported platform must skip this setup script. + # Removing the tool from the selection must also stop it running. + provider["platforms"] = ["linux/x86_64"] + provider["omissions"] = {"linux/arm64": "Fixture has no ARM artifact"} + providers_path.write_text(json.dumps({"fixture": provider})) + result = self._run(tools, root, "ci", root / "bin", env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(output.exists()) + selections_path.write_text(json.dumps({"ci": {"setup": []}})) + result = self._run(tools, root, "ci", root / "bin", env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(output.exists()) + + def test_volume_setup_prepares_writable_user_parent_directories(self) -> None: + # Docker can create .local as root when mounting a volume below it. + # Its owner must be fixed so the user can create sibling state directories. + script = ( + TOOLS_DIR.parent + / "src/s-core-devcontainer/.devcontainer/s-core-local/fix_config_volume_ownership.sh" + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home = root / "home" + fake_bin = root / "bin" + fake_bin.mkdir() + for name in (".config/opencode", ".config/gh", ".local/share/opencode"): + (home / name).mkdir(parents=True, exist_ok=True) + private_directories = (home / ".config", home / ".local") + for path in private_directories: + path.chmod(0o700) + sudo = fake_bin / "sudo" + sudo.write_text( + '#!/bin/sh\nprintf "%s\\n" "$@" >> "$SCORE_TEST_LOG"\nexec "$@"\n' + ) + sudo.chmod(0o755) + log = root / "calls" + result = subprocess.run( + ["bash", str(script)], + env=dict( + os.environ, + HOME=str(home), + PATH=str(fake_bin) + os.pathsep + os.environ["PATH"], + SCORE_TEST_LOG=str(log), + ), + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + calls = log.read_text().splitlines() + self.assertEqual(calls[0], "mkdir") + self.assertIn("chown", calls) + for path in private_directories: + self.assertEqual(path.stat().st_mode & 0o777, 0o700) + for name in (".config", ".local", ".local/share"): + self.assertIn(str(home / name), calls) + + def _copy_tools(self, root: Path) -> Path: + tools = root / "copied-tools" + shutil.copytree(TOOLS_DIR, tools, ignore=shutil.ignore_patterns("__pycache__")) + return tools + + def _run( + self, + tools: Path, + cwd: Path, + environment: str, + destination: Path, + *arguments, + env=None, + ): + # Simulate sudo for isolated integration tests: child commands may write + # only fixture paths and use fake apt/npm executables, never host packages. + fake_bin = cwd / "sudo-bin" + fake_bin.mkdir(exist_ok=True) + sudo = fake_bin / "sudo" + sudo.write_text( + f"#!{sys.executable}\n" + "import os, runpy, sys\n" + "from unittest.mock import patch\n" + "os.environ['SCORE_TEST_PRIVILEGED'] = '1'\n" + "sys.argv = sys.argv[2:]\n" + "with patch('os.geteuid', return_value=0):\n" + " runpy.run_path(sys.argv[0], run_name='__main__')\n" + ) + sudo.chmod(0o755) + env = dict(os.environ if env is None else env) + env["PATH"] = str(fake_bin) + os.pathsep + env["PATH"] + return subprocess.run( + [ + sys.executable, + "-I", + "-B", + str(tools / "internal/devcontainer/install.py"), + "install-environment", + environment, + "--destination", + str(destination), + "--os", + "linux", + "--cpu", + "arm64", + *arguments, + ], + cwd=cwd, + env=env, + capture_output=True, + text=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/python_tool_runner_test.sh b/tools/tests/python_tool_runner_test.sh index 74f1f91a..089c5cd5 100755 --- a/tools/tests/python_tool_runner_test.sh +++ b/tools/tests/python_tool_runner_test.sh @@ -20,6 +20,24 @@ runfiles_root="${TEST_SRCDIR}/${TEST_WORKSPACE}" installer="${runfiles_root}/tools/internal/devcontainer/install.py" runner="${runfiles_root}/tools/internal/run_python_tool.sh" +# Bazel and the container installer must consume the same package definitions. +# Only the cached Starlark representation is generated; JSON remains the source. +python3 - "${runfiles_root}/tools/providers.json" "${runfiles_root}/$1" <<'PYTHON' +import ast +import json +import sys +from pathlib import Path + +providers = json.loads(Path(sys.argv[1]).read_text()) +expected = { + name: {field: definition[field] for field in ("package", "version", "entrypoint", "description")} + for name, definition in providers.items() + if definition["provider"] == "python" +} +generated = ast.literal_eval(ast.parse(Path(sys.argv[2]).read_text()).body[0].value) +assert generated == expected, (generated, expected) +PYTHON + fake_uvx="${TEST_TMPDIR}/uvx" output="${TEST_TMPDIR}/uvx.args" working_directory="${TEST_TMPDIR}/working-directory"