test: add scenarios.json testing pattern, close claude-dev's installCli gap

.github/workflows/pr-validation.yml

@@ -53,12 +53,16 @@ jobs:
             new_version=$(jq -r '.version' "$manifest")
             old_version=$(git show "${MERGE_BASE}:${manifest}" 2>/dev/null | jq -r '.version' 2>/dev/null || echo "")
 
-            # README-only changes don't require a bump — the README is the
-            # only file where a diff can't change what actually ships. Every
-            # other file (install.sh, the manifest, test/<name>/test.sh, ...)
-            # still does.
-            non_readme_touched=$(echo "$touched_files" \
-              | grep -E "^(src|test)/${name}/" | grep -v "^src/${name}/README\.md$" || true)
+            # A version bump is required for anything under src/<name>/ except
+            # its README, or for test/<name>/test.sh specifically — the one
+            # test file CI runs against every consumer's build. A new
+            # test/<name>/scenarios.json or scenario script is test-only, like
+            # a README change: it verifies a non-default option combination
+            # but doesn't touch what already-published consumers get.
+            non_readme_touched=$({ \
+              echo "$touched_files" | grep -E "^src/${name}/" | grep -v "^src/${name}/README\.md$"; \
+              echo "$touched_files" | grep -E "^test/${name}/test\.sh$"; \
+            } || true)
 
             if [ -z "$old_version" ]; then
               echo "✅ ${name}: new feature, no bump required"
@@ -188,7 +192,35 @@ jobs:
                 ~/.aws/config ~/.kube/config ~/.docker/config.json
 
       - name: "Test feature '${{ matrix.features }}' against '${{ matrix.baseImage }}'"
-        run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} .
+        # --skip-scenarios: scenario tests (test/<name>/scenarios.json) define
+        # their own image and run once per feature in the test-scenarios job
+        # below, not once per baseImage here — without this flag they'd
+        # silently re-run (wastefully, and against the wrong image) on every
+        # matrix entry for a feature that has any.
+        run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} --skip-scenarios .
+
+  test-scenarios:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # A scenario tests a non-default option combination that the
+          # autogenerated test.sh (default options only) never exercises —
+          # see test/<name>/scenarios.json's own comments for why each one
+          # exists. Add an entry here whenever a feature gains a
+          # scenarios.json; see AGENTS.md "Testing non-default options".
+          - features: claude-dev
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Install latest devcontainer CLI
+        run: npm install -g @devcontainers/cli
+
+      - name: "Test '${{ matrix.features }}' scenarios"
+        run: devcontainer features test --features ${{ matrix.features }} --skip-autogenerated .
 
   shellcheck:
     runs-on: ubuntu-latest
@@ -231,7 +263,7 @@ jobs:
 
   pr-comment:
     runs-on: ubuntu-latest
-    needs: [conventional-commits, version-bump-check, test-features, shellcheck]
+    needs: [conventional-commits, version-bump-check, test-features, test-scenarios, shellcheck]
     if: always()
     steps:
       - name: Update PR comment
@@ -242,5 +274,6 @@ jobs:
               "🧾 Conventional Commits": "${{ needs.conventional-commits.result }}",
               "🔖 Version Bump": "${{ needs.version-bump-check.result }}",
               "🧪 Feature Tests": "${{ needs.test-features.result }}",
+              "🧪 Scenario Tests": "${{ needs.test-scenarios.result }}",
               "🐚 ShellCheck": "${{ needs.shellcheck.result }}"
             }

.github/workflows/test.yml

@@ -4,6 +4,9 @@ on:
     branches:
       - main
 
+permissions:
+  contents: read
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -145,4 +148,31 @@ jobs:
                 ~/.aws/config ~/.kube/config ~/.docker/config.json
 
       - name: Test feature '${{ matrix.features }}' against '${{ matrix.baseImage }}'
-        run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} .
+        # --skip-scenarios: scenario tests (test/<name>/scenarios.json) define
+        # their own image and run once per feature in the test-scenarios job
+        # below, not once per baseImage here.
+        run: devcontainer features test --features ${{ matrix.features }} --base-image ${{ matrix.baseImage }} --skip-scenarios .
+
+  test-scenarios:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # A scenario tests a non-default option combination the
+          # autogenerated test.sh (default options only) never exercises —
+          # see test/<name>/scenarios.json's own comments for why each one
+          # exists. Add an entry here whenever a feature gains a
+          # scenarios.json; see AGENTS.md "Testing non-default options".
+          - features: claude-dev
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Install latest devcontainer CLI
+        run: npm install -g @devcontainers/cli
+
+      - name: "Test '${{ matrix.features }}' scenarios"
+        run: devcontainer features test --features ${{ matrix.features }} --skip-autogenerated .

AGENTS.md

@@ -23,6 +23,39 @@ devcontainer features test --features <name> .
 devcontainer features test .
 ```
 
+**Testing non-default options:** `test/<name>/test.sh` (the "autogenerated" test) only ever
+runs with every option at its default — an option that defaults `false`/off is never actually
+exercised by CI. This is exactly how a real regression shipped unnoticed once: `vite-plus`'s
+install script degraded gracefully (warned instead of aborting) when the official installer's
+own layout changed, so the failure was silent by design — but the default-on path was covered
+by `test.sh`, so it was eventually caught. A default-*off* path with the same silent-degrade
+shape (e.g. `claude-dev`'s `installCli`) has no equivalent coverage unless you add a scenario.
+
+Add a `test/<name>/scenarios.json` + one `test/<name>/<scenario-name>.sh` per entry (same shape
+as [devcontainers/features' own examples](https://github.com/devcontainers/features/blob/main/test/node/scenarios.json)):
+
+```json
+{
+  "with_cli": {
+    "image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+    "features": { "<name>": { "someOption": true } }
+  }
+}
+```
+
+Run it locally with `devcontainer features test --features <name> --skip-autogenerated .`
+(scenarios define their own `image`, so no `--base-image` flag). Wire a matrix entry into the
+`test-scenarios` job in both `pr-validation.yml` and `test.yml` — `--skip-scenarios` is on the
+regular `test-features`/`test` job's own command specifically so a scenario doesn't silently
+re-run once per that job's `baseImage` matrix entries. Not every option needs a scenario —
+reserve it for an off-by-default path whose failure mode is silent (a warn-and-continue on
+install failure, not a hard `exit 1`) rather than one that already fails loudly if broken.
+
+Adding a `scenarios.json` or scenario script doesn't need a version bump — like a README, it
+verifies an existing option combination without changing what already-published consumers get.
+Only `test/<name>/test.sh` itself (the one CI runs against every consumer's build) triggers the
+version-bump-check below.
+
 **Available features:**
 
 | Feature | Ver | Description |

test/claude-dev/scenarios.json

@@ -0,0 +1,10 @@
+{
+  "with_cli": {
+    "image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+    "features": {
+      "claude-dev": {
+        "installCli": true
+      }
+    }
+  }
+}

test/claude-dev/with_cli.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This file is part of helpers4.
+# Copyright (C) 2026 baxyz
+# SPDX-License-Identifier: LGPL-3.0-or-later
+#
+# Exercises installCli:true — the default-off code path that plain test.sh
+# never runs, and the exact class of gap that let vite-plus's own installer
+# regression go unnoticed: an install.sh that degrades gracefully (warns
+# instead of aborting) on a failed CLI install looks identical in CI to one
+# that succeeded, unless something actually asserts the binary landed.
+
+set -e
+
+echo "Testing claude-dev with installCli:true..."
+
+if [ ! -x /usr/local/bin/claude ]; then
+    echo "❌ FAIL: /usr/local/bin/claude missing — installCli:true did not install the CLI"
+    exit 1
+fi
+echo "✅ PASS: /usr/local/bin/claude present"
+
+if ! /usr/local/bin/claude --version >/dev/null 2>&1; then
+    echo "❌ FAIL: /usr/local/bin/claude exists but does not run"
+    exit 1
+fi
+echo "✅ PASS: claude --version runs"
+
+echo "🎉 Test passed."