Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,23 @@ skillspector scan ./my-skill/ --format markdown --output report.md
skillspector scan ./my-skill/ --format sarif --output report.sarif
```

### Recursive Collection Scanning

`--recursive` scans immediate subdirectories containing `SKILL.md` as independent
skills, with a default aggregate limit of **32 skills**. To raise or lower it:

```bash
skillspector scan ./my-skills/ --recursive --max-skills 100 --no-llm --format json
```

`--max-skills` requires a local recursive collection and a positive integer; `0`
does not mean unlimited. Discovery, byte, finding-record, time and report-size
budgets still apply, so increasing the count does not guarantee complete coverage.
When the count limit is reached, the CLI names the limit and suggests adjusting it
or scanning sub-skills separately. Omitted skills remain visible in the aggregate
report and coverage stays incomplete. Use `--fail-on-incomplete` in CI to fail on
incomplete coverage; the default exit policy is unchanged.

### Batch Scanning

Scan entire directories of skills in parallel from `contrib/batch_scan/`:
Expand Down
34 changes: 31 additions & 3 deletions src/skillspector/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,15 @@ def scan(
help="Scan immediate subdirectories that each contain a SKILL.md as independent skills.",
),
] = False,
max_skills: Annotated[
int | None,
typer.Option(
"--max-skills",
min=1,
help="Maximum skills in a local --recursive multi-skill scan (default: 32). "
"Other aggregate safety limits still apply; no unlimited value.",
),
] = None,
baseline: Annotated[
Path | None,
typer.Option(
Expand Down Expand Up @@ -679,6 +688,11 @@ def scan(
authentication session. codex_cli is registered but disabled because
its read-only sandbox permits host-file reads; use another provider.
"""
if max_skills is not None and (not recursive or mcp_registry or not Path(input_path).is_dir()):
err_console.print(
"[red]Error:[/red] --max-skills requires a local --recursive multi-skill directory"
)
raise typer.Exit(code=2)
if mcp_registry_compare is not None and not mcp_registry:
err_console.print("[red]Error:[/red] --mcp-registry-compare requires --mcp-registry")
raise typer.Exit(code=2)
Expand Down Expand Up @@ -820,6 +834,7 @@ def scan(
verbose=verbose,
fail_on_incomplete=fail_on_incomplete,
fail_on_findings=fail_on_findings,
max_skills=max_skills,
min_coverage=min_coverage,
)
except typer.Exit:
Expand All @@ -834,6 +849,12 @@ def scan(
err_console.print(f"[red]Error:[/red] {e}")
raise typer.Exit(code=2) from e
return
if max_skills is not None:
err_console.print(
"[red]Error:[/red] --max-skills requires detected sub-skills; "
"scan this input without --max-skills"
)
raise typer.Exit(code=2)
if detection.complete and not detection.has_root_skill and len(detection.skills) == 0:
discovery_console.print(
"[yellow]Warning:[/yellow] --recursive specified but no sub-skills "
Expand Down Expand Up @@ -2935,9 +2956,13 @@ def _scan_multi_skill(
fail_on_incomplete: bool = False,
min_coverage: float | None = None,
fail_on_findings: bool = False,
max_skills: int | None = None,
**legacy_kwargs: object,
) -> None:
"""Scan each detected sub-skill independently and produce a combined report."""
skill_limit = _MULTI_SKILL_MAX_SKILLS if max_skills is None else max_skills
if isinstance(skill_limit, bool) or not isinstance(skill_limit, int) or skill_limit < 1:
raise ValueError("max_skills must be a positive integer")
if yara_dir is None and isinstance(legacy_kwargs.get("yara_rules_dir"), Path):
yara_dir = str(legacy_kwargs["yara_rules_dir"])
skills = detection.skills
Expand Down Expand Up @@ -2979,10 +3004,13 @@ def _scan_multi_skill(
coverage_failed = False

for i, skill in enumerate(skills, 1):
if i > _MULTI_SKILL_MAX_SKILLS:
if i > skill_limit:
analysis_incomplete = True
aggregate_limitations.append(
f"recursive skill count budget {_MULTI_SKILL_MAX_SKILLS} reached"
aggregate_limitations.append(f"recursive skill count budget {skill_limit} reached")
progress_console.print(
f"[yellow]Recursive skill count budget {skill_limit} reached.[/yellow] "
"Use --recursive --max-skills N with a larger positive N, or scan "
"sub-skills separately. Other aggregate safety limits still apply."
)
break
if retained_public_records >= _MULTI_SKILL_MAX_PUBLIC_RECORDS:
Expand Down
118 changes: 116 additions & 2 deletions tests/unit/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import pytest
import typer
import yaml
from click import unstyle
from markdown_it import MarkdownIt
from rich.console import Console
from typer.testing import CliRunner
Expand Down Expand Up @@ -2580,8 +2581,121 @@ def test_recursive_fail_on_findings_exits_one_below_risk_threshold(
assert exit_info.value.exit_code == 1


@pytest.mark.parametrize(("limit", "scanned"), [(None, 32), (1, 1), (33, 33), (40, 33)])
@pytest.mark.parametrize("strict", [False, True])
def test_recursive_configurable_skill_count_budget(
monkeypatch, tmp_path, capsys, limit, scanned, strict
):
skills = [SkillDirectory(tmp_path / str(i), str(i), str(i)) for i in range(33)]
detection = MultiSkillDetectionResult(is_multi_skill=True, skills=skills)
calls = []

def fake_scan(**kwargs):
calls.append(kwargs)
return _bounded_recursive_result("safe", finding_count=0)

monkeypatch.setattr(cli, "_scan_skill", fake_scan)
output = tmp_path / "report.json"
context = pytest.raises(typer.Exit) if strict and scanned < 33 else nullcontext()
with context as error:
_scan_multi_skill(
detection,
FormatChoice.json,
output,
no_llm=True,
max_skills=limit,
fail_on_incomplete=strict,
)
if strict and scanned < 33:
assert error.value.exit_code == 1
payload = json.loads(output.read_text())
assert len(calls) == payload["skills_scanned"] == scanned
assert payload["skills_omitted"] == 33 - scanned
assert payload["analysis_completeness"]["is_complete"] is (scanned == 33)
assert payload["risk_recommendation"] == ("SAFE" if scanned == 33 else "CAUTION")
if scanned < 33:
effective = 32 if limit is None else limit
assert (
f"recursive skill count budget {effective} reached"
in payload["analysis_completeness"]["limitations"]
)
assert "--max-skills" in capsys.readouterr().out


@pytest.mark.parametrize("value", ["0", "-1", "abc"])
def test_cli_max_skills_rejects_invalid_count(tmp_path, monkeypatch, value):
monkeypatch.setattr(cli, "_scan_skill", lambda **_: pytest.fail("must not scan"))
result = runner.invoke(app, ["scan", str(tmp_path), "--recursive", "--max-skills", value])
assert result.exit_code == 2


@pytest.mark.parametrize("recursive", [False, True])
def test_cli_max_skills_rejects_non_collection(tmp_path, monkeypatch, recursive):
monkeypatch.setattr(cli, "_scan_skill", lambda **_: pytest.fail("must not scan"))
args = ["scan", str(tmp_path), "--max-skills", "100"]
if recursive:
args.append("--recursive")
result = runner.invoke(app, args)
assert result.exit_code == 2
assert "--max-skills" in result.stderr


def test_cli_max_skills_reaches_recursive_scan(tmp_path, monkeypatch):
for i in range(33):
skill = tmp_path / f"s{i:03}"
skill.mkdir()
(skill / "SKILL.md").write_text("---\nname: safe\n---\nHello\n")
calls = []

def fake_scan(**kwargs):
calls.append(kwargs)
return _bounded_recursive_result("safe", finding_count=0)

monkeypatch.setattr(cli, "_scan_skill", fake_scan)
result = runner.invoke(
app,
[
"scan",
str(tmp_path),
"--recursive",
"--max-skills",
"33",
"--no-llm",
"--format",
"json",
],
)
assert result.exit_code == 0, result.output
payload = json.loads(result.stdout)
assert len(calls) == payload["skills_scanned"] == 33
assert payload["skills_omitted"] == 0


@pytest.mark.parametrize("color", [False, True])
def test_cli_max_skills_is_documented_in_help(monkeypatch, color):
# Typer initializes its terminal mode at import time, so environment-only
# changes here do not reliably exercise the GitHub Actions rendering path.
monkeypatch.delenv("NO_COLOR", raising=False)
monkeypatch.setattr("typer.rich_utils.FORCE_TERMINAL", color)
monkeypatch.setattr("typer.rich_utils.COLOR_SYSTEM", "standard" if color else None)
result = runner.invoke(app, ["scan", "--help"], color=True)
assert result.exit_code == 0
assert ("\x1b[" in result.stdout) is color
help_text = unstyle(result.stdout)
assert "--max-skills" in help_text
assert "32" in help_text


@pytest.mark.parametrize("limit", [0, -1, True, 1.5])
def test_recursive_max_skills_helper_rejects_invalid_limit(tmp_path, limit):
detection = MultiSkillDetectionResult(is_multi_skill=True, skills=[])
with pytest.raises(ValueError, match="positive integer"):
_scan_multi_skill(detection, FormatChoice.json, None, no_llm=True, max_skills=limit)


@pytest.mark.parametrize("max_skills", [None, 100])
def test_recursive_json_uses_one_global_public_record_budget(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, max_skills
) -> None:
"""Per-child report caps cannot multiply in the combined JSON document."""
skills = [SkillDirectory(tmp_path / name, name, name) for name in ("one", "two", "three")]
Expand All @@ -2600,7 +2714,7 @@ def fake_invoke(*_args, **_kwargs) -> dict[str, object]:
fake_invoke,
)

_scan_multi_skill(detection, FormatChoice.json, output, no_llm=True)
_scan_multi_skill(detection, FormatChoice.json, output, no_llm=True, max_skills=max_skills)

assert len(calls) == 1
payload = json.loads(output.read_text(encoding="utf-8"))
Expand Down
Loading