From 8ed504c1f49fc9645577ed25d4bf0cd76aa987ee Mon Sep 17 00:00:00 2001 From: baxyz Date: Wed, 9 Sep 2026 18:54:30 +0000 Subject: [PATCH 1/4] =?UTF-8?q?test(claude-dev):=20=E2=9C=85=20add=20a=20s?= =?UTF-8?q?cenario=20test=20for=20installCli:true?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prompted by the vite-plus VP_HOME regression: install.sh's installCli path already degrades gracefully (warns instead of aborting) if the CLI install fails, which is correct — but installCli defaults to false and no CI matrix entry ever set it to true, so that whole code path has never once run in CI. The exact same silent-degrade shape that let vite-plus's own regression through, just on an untested path this time instead of a buggy one. Verified directly: `devcontainer features test --features claude-dev --skip-autogenerated .` runs the real claude.ai installer with installCli:true and installs correctly — /usr/local/bin/claude present and runnable. Not a bug, just previously unverified. Co-Authored-By: Claude Sonnet 5 --- src/claude-dev/README.md | 3 +++ src/claude-dev/devcontainer-feature.json | 2 +- test/claude-dev/scenarios.json | 10 ++++++++ test/claude-dev/with_cli.sh | 29 ++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/claude-dev/scenarios.json create mode 100755 test/claude-dev/with_cli.sh diff --git a/src/claude-dev/README.md b/src/claude-dev/README.md index 69d6e8b..5a9bea0 100644 --- a/src/claude-dev/README.md +++ b/src/claude-dev/README.md @@ -93,6 +93,9 @@ without depending on that user's shell profile already including ## Version History +- **v1.2.5**: Test-only change, no behavior change — added a `with_cli` scenario test + (`installCli: true`), the default-off code path that plain `test.sh` never exercises. See + AGENTS.md "Testing non-default options". - **v1.2.4**: Internal cleanup, no behavior change — dropped dead `_BUILD_ARG_*` fallbacks in `install.sh` (`INSTALLCLI`, `USERNAME`). That prefix is only ever set for the legacy `internalVersion: "1"` manifest shape, which this feature (and every other one in this diff --git a/src/claude-dev/devcontainer-feature.json b/src/claude-dev/devcontainer-feature.json index 84ceb6a..ed453b7 100644 --- a/src/claude-dev/devcontainer-feature.json +++ b/src/claude-dev/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "claude-dev", - "version": "1.2.4", + "version": "1.2.5", "name": "Claude Code Development Environment", "description": "Installs the Claude Code IDE extension (anthropic.claude-code) for VS Code and Cursor, and optionally the claude CLI. Persists ~/.claude (credentials, config, memory) across rebuilds and Codespaces via a Docker named volume, shared per host OS user. Also self-heals your devcontainer setup.", "documentationURL": "https://github.com/helpers4/devcontainer/tree/main/src/claude-dev", diff --git a/test/claude-dev/scenarios.json b/test/claude-dev/scenarios.json new file mode 100644 index 0000000..e1562a9 --- /dev/null +++ b/test/claude-dev/scenarios.json @@ -0,0 +1,10 @@ +{ + "with_cli": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "claude-dev": { + "installCli": true + } + } + } +} diff --git a/test/claude-dev/with_cli.sh b/test/claude-dev/with_cli.sh new file mode 100755 index 0000000..74cf037 --- /dev/null +++ b/test/claude-dev/with_cli.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This file is part of helpers4. +# Copyright (C) 2026 baxyz +# SPDX-License-Identifier: LGPL-3.0-or-later +# +# Exercises installCli:true — the default-off code path that plain test.sh +# never runs, and the exact class of gap that let vite-plus's own installer +# regression go unnoticed: an install.sh that degrades gracefully (warns +# instead of aborting) on a failed CLI install looks identical in CI to one +# that succeeded, unless something actually asserts the binary landed. + +set -e + +echo "Testing claude-dev with installCli:true..." + +if [ ! -x /usr/local/bin/claude ]; then + echo "❌ FAIL: /usr/local/bin/claude missing — installCli:true did not install the CLI" + exit 1 +fi +echo "✅ PASS: /usr/local/bin/claude present" + +if ! /usr/local/bin/claude --version >/dev/null 2>&1; then + echo "❌ FAIL: /usr/local/bin/claude exists but does not run" + exit 1 +fi +echo "✅ PASS: claude --version runs" + +echo "🎉 Test passed." From 4e84d9a0a927db8c4c7e939b05897e1dd7b6e84e Mon Sep 17 00:00:00 2001 From: baxyz Date: Wed, 9 Sep 2026 18:54:30 +0000 Subject: [PATCH 2/4] =?UTF-8?q?ci(CI-CD):=20=F0=9F=91=B7=20add=20a=20test-?= =?UTF-8?q?scenarios=20job=20and=20document=20the=20scenarios.json=20patte?= =?UTF-8?q?rn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires claude-dev's new with_cli scenario into both pr-validation.yml and test.yml as a separate job (matching devcontainers/features' own convention: --skip-scenarios on the regular per-baseImage job, --skip-autogenerated on the scenario job — a scenario defines its own image, so running it once per baseImage matrix entry would be both wrong and wasteful). Documents the pattern in AGENTS.md under "Testing non-default options" so the next default-off, silently-degrading option gets a scenario instead of staying untested indefinitely. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/pr-validation.yml | 33 +++++++++++++++++++++++++++-- .github/workflows/test.yml | 29 ++++++++++++++++++++++++- AGENTS.md | 30 +++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index f3393b5..2932f3a 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -188,7 +188,35 @@ jobs: ~/.aws/config ~/.kube/config ~/.docker/config.json - name: "Test feature '${{ matrix.features }}' against '${{ matrix.baseImage }}'" - run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} . + # --skip-scenarios: scenario tests (test//scenarios.json) define + # their own image and run once per feature in the test-scenarios job + # below, not once per baseImage here — without this flag they'd + # silently re-run (wastefully, and against the wrong image) on every + # matrix entry for a feature that has any. + run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} --skip-scenarios . + + test-scenarios: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + # A scenario tests a non-default option combination that the + # autogenerated test.sh (default options only) never exercises — + # see test//scenarios.json's own comments for why each one + # exists. Add an entry here whenever a feature gains a + # scenarios.json; see AGENTS.md "Testing non-default options". + - features: claude-dev + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install latest devcontainer CLI + run: npm install -g @devcontainers/cli + + - name: "Test '${{ matrix.features }}' scenarios" + run: devcontainer features test --features ${{ matrix.features }} --skip-autogenerated . shellcheck: runs-on: ubuntu-latest @@ -231,7 +259,7 @@ jobs: pr-comment: runs-on: ubuntu-latest - needs: [conventional-commits, version-bump-check, test-features, shellcheck] + needs: [conventional-commits, version-bump-check, test-features, test-scenarios, shellcheck] if: always() steps: - name: Update PR comment @@ -242,5 +270,6 @@ jobs: "🧾 Conventional Commits": "${{ needs.conventional-commits.result }}", "🔖 Version Bump": "${{ needs.version-bump-check.result }}", "🧪 Feature Tests": "${{ needs.test-features.result }}", + "🧪 Scenario Tests": "${{ needs.test-scenarios.result }}", "🐚 ShellCheck": "${{ needs.shellcheck.result }}" } diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4db9d89..99f82d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,4 +145,31 @@ jobs: ~/.aws/config ~/.kube/config ~/.docker/config.json - name: Test feature '${{ matrix.features }}' against '${{ matrix.baseImage }}' - run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} . + # --skip-scenarios: scenario tests (test//scenarios.json) define + # their own image and run once per feature in the test-scenarios job + # below, not once per baseImage here. + run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} --skip-scenarios . + + test-scenarios: + runs-on: ubuntu-latest + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + # A scenario tests a non-default option combination the + # autogenerated test.sh (default options only) never exercises — + # see test//scenarios.json's own comments for why each one + # exists. Add an entry here whenever a feature gains a + # scenarios.json; see AGENTS.md "Testing non-default options". + - features: claude-dev + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install latest devcontainer CLI + run: npm install -g @devcontainers/cli + + - name: "Test '${{ matrix.features }}' scenarios" + run: devcontainer features test --features ${{ matrix.features }} --skip-autogenerated . diff --git a/AGENTS.md b/AGENTS.md index c1d5a46..788b36a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,34 @@ devcontainer features test --features . devcontainer features test . ``` +**Testing non-default options:** `test//test.sh` (the "autogenerated" test) only ever +runs with every option at its default — an option that defaults `false`/off is never actually +exercised by CI. This is exactly how a real regression shipped unnoticed once: `vite-plus`'s +install script degraded gracefully (warned instead of aborting) when the official installer's +own layout changed, so the failure was silent by design — but the default-on path was covered +by `test.sh`, so it was eventually caught. A default-*off* path with the same silent-degrade +shape (e.g. `claude-dev`'s `installCli`) has no equivalent coverage unless you add a scenario. + +Add a `test//scenarios.json` + one `test//.sh` per entry (same shape +as [devcontainers/features' own examples](https://github.com/devcontainers/features/blob/main/test/node/scenarios.json)): + +```json +{ + "with_cli": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { "": { "someOption": true } } + } +} +``` + +Run it locally with `devcontainer features test --features --skip-autogenerated .` +(scenarios define their own `image`, so no `--base-image` flag). Wire a matrix entry into the +`test-scenarios` job in both `pr-validation.yml` and `test.yml` — `--skip-scenarios` is on the +regular `test-features`/`test` job's own command specifically so a scenario doesn't silently +re-run once per that job's `baseImage` matrix entries. Not every option needs a scenario — +reserve it for an off-by-default path whose failure mode is silent (a warn-and-continue on +install failure, not a hard `exit 1`) rather than one that already fails loudly if broken. + **Available features:** | Feature | Ver | Description | @@ -31,7 +59,7 @@ devcontainer features test . | `essential-dev` | 1.2.4 | Git visualization, editor enhancements, Markdown | | `github-dev` | 1.0.7 | gh CLI, Copilot Chat, PR/Issues/Actions extensions | | `copilot-dev` | 1.0.4 | Copilot Chat + AI instructions (commits, PRs, code review) | -| `claude-dev` | 1.2.4 | Claude Code extension + CLI + `~/.claude` named-volume persistence (credentials + memory) | +| `claude-dev` | 1.2.5 | Claude Code extension + CLI + `~/.claude` named-volume persistence (credentials + memory) | | `mistral-dev` | 1.2.4 | Mistral Vibe extension + `~/.vibe` named-volume persistence | | `cline-dev` | 1.0.1 | Cline extension (`saoudrizwan.claude-dev`) + optional CLI, no credential persistence | | `nub` | 1.2.3 | Fast TS/JS/script runner on top of existing node+package-manager (dependsOn node) | From 0ecadc6467dbac33e576365665f80f4de77cb0ab Mon Sep 17 00:00:00 2001 From: baxyz Date: Wed, 9 Sep 2026 19:00:02 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix(CI-CD):=20=F0=9F=90=9B=20version-bump-c?= =?UTF-8?q?heck=20shouldn't=20flag=20test-only=20scenario=20additions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new test//scenarios.json or scenario script doesn't change what already-published consumers get — it only verifies a non-default option combination, the same "doesn't touch what ships" reasoning already applied to README-only changes. The check's own grep was broader than its documented policy ("test//test.sh" specifically, per this file's "Modifying an existing feature" section above), so it flagged claude-dev's new with_cli scenario as needing a bump. Narrowed it to match: src// (minus README) or test//test.sh specifically trigger the check, not every file under test//. Also drops the version bump and changelog entry this added to claude-dev for the scenario-only PR — no longer needed now that the check matches its own policy. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/pr-validation.yml | 16 ++++++++++------ AGENTS.md | 7 ++++++- src/claude-dev/README.md | 3 --- src/claude-dev/devcontainer-feature.json | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 2932f3a..c3b864d 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -53,12 +53,16 @@ jobs: new_version=$(jq -r '.version' "$manifest") old_version=$(git show "${MERGE_BASE}:${manifest}" 2>/dev/null | jq -r '.version' 2>/dev/null || echo "") - # README-only changes don't require a bump — the README is the - # only file where a diff can't change what actually ships. Every - # other file (install.sh, the manifest, test//test.sh, ...) - # still does. - non_readme_touched=$(echo "$touched_files" \ - | grep -E "^(src|test)/${name}/" | grep -v "^src/${name}/README\.md$" || true) + # A version bump is required for anything under src// except + # its README, or for test//test.sh specifically — the one + # test file CI runs against every consumer's build. A new + # test//scenarios.json or scenario script is test-only, like + # a README change: it verifies a non-default option combination + # but doesn't touch what already-published consumers get. + non_readme_touched=$({ \ + echo "$touched_files" | grep -E "^src/${name}/" | grep -v "^src/${name}/README\.md$"; \ + echo "$touched_files" | grep -E "^test/${name}/test\.sh$"; \ + } || true) if [ -z "$old_version" ]; then echo "✅ ${name}: new feature, no bump required" diff --git a/AGENTS.md b/AGENTS.md index 788b36a..8b10f54 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,11 @@ re-run once per that job's `baseImage` matrix entries. Not every option needs a reserve it for an off-by-default path whose failure mode is silent (a warn-and-continue on install failure, not a hard `exit 1`) rather than one that already fails loudly if broken. +Adding a `scenarios.json` or scenario script doesn't need a version bump — like a README, it +verifies an existing option combination without changing what already-published consumers get. +Only `test//test.sh` itself (the one CI runs against every consumer's build) triggers the +version-bump-check below. + **Available features:** | Feature | Ver | Description | @@ -59,7 +64,7 @@ install failure, not a hard `exit 1`) rather than one that already fails loudly | `essential-dev` | 1.2.4 | Git visualization, editor enhancements, Markdown | | `github-dev` | 1.0.7 | gh CLI, Copilot Chat, PR/Issues/Actions extensions | | `copilot-dev` | 1.0.4 | Copilot Chat + AI instructions (commits, PRs, code review) | -| `claude-dev` | 1.2.5 | Claude Code extension + CLI + `~/.claude` named-volume persistence (credentials + memory) | +| `claude-dev` | 1.2.4 | Claude Code extension + CLI + `~/.claude` named-volume persistence (credentials + memory) | | `mistral-dev` | 1.2.4 | Mistral Vibe extension + `~/.vibe` named-volume persistence | | `cline-dev` | 1.0.1 | Cline extension (`saoudrizwan.claude-dev`) + optional CLI, no credential persistence | | `nub` | 1.2.3 | Fast TS/JS/script runner on top of existing node+package-manager (dependsOn node) | diff --git a/src/claude-dev/README.md b/src/claude-dev/README.md index 5a9bea0..69d6e8b 100644 --- a/src/claude-dev/README.md +++ b/src/claude-dev/README.md @@ -93,9 +93,6 @@ without depending on that user's shell profile already including ## Version History -- **v1.2.5**: Test-only change, no behavior change — added a `with_cli` scenario test - (`installCli: true`), the default-off code path that plain `test.sh` never exercises. See - AGENTS.md "Testing non-default options". - **v1.2.4**: Internal cleanup, no behavior change — dropped dead `_BUILD_ARG_*` fallbacks in `install.sh` (`INSTALLCLI`, `USERNAME`). That prefix is only ever set for the legacy `internalVersion: "1"` manifest shape, which this feature (and every other one in this diff --git a/src/claude-dev/devcontainer-feature.json b/src/claude-dev/devcontainer-feature.json index ed453b7..84ceb6a 100644 --- a/src/claude-dev/devcontainer-feature.json +++ b/src/claude-dev/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "claude-dev", - "version": "1.2.5", + "version": "1.2.4", "name": "Claude Code Development Environment", "description": "Installs the Claude Code IDE extension (anthropic.claude-code) for VS Code and Cursor, and optionally the claude CLI. Persists ~/.claude (credentials, config, memory) across rebuilds and Codespaces via a Docker named volume, shared per host OS user. Also self-heals your devcontainer setup.", "documentationURL": "https://github.com/helpers4/devcontainer/tree/main/src/claude-dev", From 534dabae654f5ec936849741f169c77e79d77eab Mon Sep 17 00:00:00 2001 From: baxyz Date: Wed, 9 Sep 2026 19:07:48 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(CI-CD):=20=F0=9F=94=92=EF=B8=8F=20add?= =?UTF-8?q?=20minimal=20permissions=20block=20to=20test.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL flagged this workflow for not limiting the default GITHUB_TOKEN permissions. pr-validation.yml already scopes to contents:read + pull-requests:write; test.yml only checks out code and runs the devcontainer CLI locally, so contents:read alone covers it. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99f82d6..1e1d720 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,9 @@ on: branches: - main +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest