diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5fc46de7..7928afaf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,10 @@ name: tests # "Option 1: keep a small Git Bash job for #596": the `tests-git-bash` job # below runs the launcher's and the installer's suites, and nothing else, on # Windows under Git Bash. +# +# The `package` job at the end runs no pytest. It builds the openreposhape +# wheel and installs it with pipx, so what it checks needs PyPI (the build +# backend, git-filter-repo; #563). The other jobs take only test tools there. on: push: @@ -218,3 +222,139 @@ jobs: # before the suites' own nesting starts. - name: the launcher's and the installer's suites, under Git Bash run: python -m pytest -q -rA tests/test_openreposhape_command.py tests/test_install_sh.py --basetemp=${{ runner.temp }}\t + + # THE openreposhape PACKAGE, BUILT AND INSTALLED AS A PERSON INSTALLS IT + # (#563). A JOB OF ITS OWN, AND NO PYTEST IN IT: the build backend + # `pyproject.toml` pins (hatchling) and git-filter-repo, the package's one + # dependency, both come from PyPI, and the suite in the `tests` job above + # never touches the network. What can be checked offline is checked there, + # on Linux like the rest of the suite since #597 + # (`tests/test_openreposhape_package.py`); this job is what only a real + # build and a real pipx can say. + # + # UBUNTU, ON THE OLDEST PYTHON THE PACKAGE SUPPORTS, 3.9. The wheel is pure + # Python, one file for every system; the release workflow (#563's next + # pull request) installs it on Linux, macOS and Windows before it uploads + # anything. + package: + runs-on: ubuntu-latest + steps: + # The same two commits the other jobs pin. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.9' + + # TWICE, AT THE COMMIT'S OWN TIME, AND THE TWO MUST BE ONE FILE: a + # release uploads bytes it can build again. `--no-deps`, because the + # wheel is this package alone; git-filter-repo arrives with the install. + - name: build the wheel twice, and compare the two + run: | + set -euo pipefail + SOURCE_DATE_EPOCH="$(git log -1 --format=%ct HEAD)" + export SOURCE_DATE_EPOCH + for out in a b; do + python -m pip wheel --disable-pip-version-check --no-deps \ + --wheel-dir "$RUNNER_TEMP/dist-$out" . + done + cmp "$RUNNER_TEMP"/dist-a/*.whl "$RUNNER_TEMP"/dist-b/*.whl + sha256sum "$RUNNER_TEMP"/dist-a/*.whl + + # EVERY FILE IN THE WHEEL BUT ITS METADATA IS THIS COMMIT'S OWN, BYTE + # FOR BYTE, read from git: the package's modules from `src/`, and + # `setup-project.py`, which the wheel carries so that `--preflight` + # runs before git is installed. + - name: the wheel holds this commit's files, byte for byte + run: | + python - "$RUNNER_TEMP"/dist-a/*.whl <<'EOF' + import subprocess, sys, zipfile + with zipfile.ZipFile(sys.argv[1]) as wheel: + for name in wheel.namelist(): + if ".dist-info/" in name: + continue + source = ("setup-project.py" + if name == "openreposhape/setup-project.py" + else "src/" + name) + committed = subprocess.run( + ["git", "show", "HEAD:" + source], + capture_output=True, check=True).stdout + if wheel.read(name) != committed: + sys.exit(f"{name} in the wheel is not {source} at HEAD") + print(f"{name} = HEAD:{source}") + EOF + + # pipx AS A PERSON RUNS IT, into directories of this job's own, and ON + # THE 3.9 ABOVE. Not the runner's own pipx with `--python`: that pipx + # runs on the image's Python and shares ONE pip with every environment + # it makes, a pip that no longer runs on 3.9 (`dataclass(slots=True)`, + # this job's first run, 2026-10-15). So pipx itself runs on 3.9, at + # 1.11.1, the newest release that installs there, and its shared pip + # is then one that 3.9 runs. Wheels only (`--only-binary :all:`), so + # installing pipx runs no package's setup script. + - name: install the wheel with pipx + run: | + set -euo pipefail + { + echo "PIPX_HOME=$RUNNER_TEMP/pipx/home" + echo "PIPX_BIN_DIR=$RUNNER_TEMP/pipx/bin" + } >> "$GITHUB_ENV" + export PIPX_HOME="$RUNNER_TEMP/pipx/home" + export PIPX_BIN_DIR="$RUNNER_TEMP/pipx/bin" + python -m pip install --disable-pip-version-check \ + --only-binary :all: pipx==1.11.1 + python -m pipx --version + python -m pipx install "$RUNNER_TEMP"/dist-a/*.whl + "$PIPX_HOME/venvs/openreposhape/bin/python" --version + ls -l "$PIPX_BIN_DIR" + + # THE FIVE COMMANDS PIPX PLACED, AGAINST THIS COMMIT. A bare repository + # on disk holds it as `main`, which an unstamped build follows, so + # nothing is cloned from github.com. No git-filter-repo is on PATH, and + # the doctor's machine row must still find the one pipx installed + # beside the package: the PATH every command gives its children. + - name: run the commands pipx placed + env: + OPENREPOSHAPE_REPO: ${{ runner.temp }}/standard.git + run: | + set -euo pipefail + git init -q --bare --initial-branch=main "$OPENREPOSHAPE_REPO" + git push -q "$OPENREPOSHAPE_REPO" HEAD:refs/heads/main + export PATH="$PIPX_BIN_DIR:$PATH" + if command -v git-filter-repo; then + echo "::error::git-filter-repo is on PATH already, so this job cannot show the commands finding their own" + exit 1 + fi + out="$RUNNER_TEMP/out" + mkdir -p "$out" + openRepoShape --version | tee "$out/version" + grep -q '^openRepoShape 0.0.0 (.* @ main)$' "$out/version" + openRepoShape --help >/dev/null + status=0 + openRepoShape --install 2>"$out/install" || status=$? + cat "$out/install" + test "$status" -eq 2 + grep -q 'pipx upgrade openreposhape' "$out/install" + openRepoShape --preflight --local-remote-dir "$RUNNER_TEMP/no-remotes" + git init -q "$RUNNER_TEMP/empty" + status=0 + openRepoShape --doctor "$RUNNER_TEMP/empty" --json \ + >"$out/doctor.json" || status=$? + test "$status" -eq 2 + python - "$out/doctor.json" \ + "$PIPX_HOME/venvs/openreposhape/bin/git-filter-repo" <<'EOF' + import json, sys + with open(sys.argv[1], encoding="utf-8") as handle: + report = json.load(handle) + print("verdict:", report["verdict"]) + [machine] = [row for row in report["rows"] if row["id"] == "machine"] + found = machine["detail"]["git-filter-repo"] + print("machine row: git-filter-repo at", found) + if found != sys.argv[2]: + sys.exit(f"the doctor found git-filter-repo at {found!r}, " + f"not at {sys.argv[2]!r}") + EOF + for tool in scaffold-project adopt-project update-shape shape-doctor; do + "$tool" --help >/dev/null + done diff --git a/install.sh b/install.sh index 7f91cd5d..9a8a78fb 100755 --- a/install.sh +++ b/install.sh @@ -41,7 +41,8 @@ # does not stay on the release tag (one older than v1.0.1 cannot record # it): the lines above say which; # 2 refused before anything was installed: a version that is not a ref, no -# release found, a fetch that failed, or a file that is not the command. +# release found, a fetch that failed, a file that is not the command, or +# a command already there as a symbolic link (pipx's, #563). # git-filter-repo never changes it: only adopting needs it, and # `adopt-project.py` refuses with its own install command where it is missing. # What it changes is the summary (#595): `adoption:` says whether it is there @@ -361,6 +362,20 @@ main() { command -v curl >/dev/null 2>&1 || die "curl is not installed, and this installer fetches with it. Install curl, or install by hand with: gh api repos/$REPO/contents/openRepoShape -H 'Accept: application/vnd.github.raw' | bash -s -- --install" + # A LINK IS NOT A PLACE TO INSTALL TO (#563), and this installer says so + # itself, before anything is asked of the network. pipx places the + # openreposhape package's `openRepoShape` as a symbolic link in this same + # directory, and the command's `--install` copies with `cp`, which writes + # through a link into the package's own environment. The command refuses + # such a target too, but only from the release that added the check: the + # release named here may be older, so its refusal cannot be relied on. + bin_dir="${OPENREPOSHAPE_BIN_DIR:-$HOME/.local/bin}" + target="$bin_dir/openRepoShape" + [[ ! -L "$target" ]] || + die "$target is a symbolic link, so nothing was installed: pipx places the openreposhape package's openRepoShape that way, and installing this one through the link would overwrite the package's command. Use one installer or the other: + keep the package's command, and update it with: pipx upgrade openreposhape + or remove the link (pipx uninstall openreposhape, or rm $target), then run this line again." + # 1. THE RELEASE. if [ -n "$version" ]; then checked_version "$version" "the version" @@ -404,8 +419,6 @@ main() { die "$launcher_url is not the openRepoShape command (its first line is not #!/usr/bin/env bash, or it has no INSTALLABLES line), so nothing was installed." say "fetch: $launcher_url" - bin_dir="${OPENREPOSHAPE_BIN_DIR:-$HOME/.local/bin}" - target="$bin_dir/openRepoShape" say "install: OPENREPOSHAPE_REF=$version OPENREPOSHAPE_RECORD_REF=1 openRepoShape --install" OPENREPOSHAPE_REF="$version" OPENREPOSHAPE_RECORD_REF=1 \ "${BASH:-bash}" "$WORKDIR/openRepoShape" --install || diff --git a/openRepoShape b/openRepoShape index 2ab98fb7..4c95873c 100755 --- a/openRepoShape +++ b/openRepoShape @@ -391,7 +391,24 @@ install_commands() { { [ "$REF_NAMED" -eq 1 ] && [ "${OPENREPOSHAPE_RECORD_REF:-}" = 1 ]; }; then recording=1 fi - # THE RECORD SITS BESIDE THE FILE, symlinks followed, as it is read. + # A LINK IS NOT A PLACE TO COPY TO (#563). pipx places its + # `openRepoShape`, the openreposhape package's, as a symbolic link into + # the package's own environment, in this same `~/.local/bin`. `cp` + # writes THROUGH a link, so an install over one would overwrite the + # package's command with this file and leave the link where it was: + # two installers, one name, each breaking the other. Refused before + # anything is collected or placed, a dangling link included, and the + # refusal names both ways out. install.sh refuses the same target + # itself, for a release whose command predates this check. + for name in "${INSTALLABLES[@]}"; do + [ ! -L "$dir/$name" ] || + die "$dir/$name is a symbolic link, so --install will not copy this command through it: pipx places the openreposhape package's openRepoShape that way, and a copy written through the link would overwrite the package's command. Nothing was placed. Use one installer or the other: + keep the package's command, and update it with: pipx upgrade openreposhape + or remove the link (pipx uninstall openreposhape, or rm $dir/$name), then run this again." + done + # THE RECORD SITS BESIDE THE FILE. The target is no link (refused just + # above), so `resolved` hands it back unchanged; the call stays because + # it is how `read_recorded_ref` names the record, and the two must agree. record="$(resolved "$dir/openRepoShape")$RECORD_SUFFIX" # BOTH CHECKED BEFORE ANYTHING IS COLLECTED OR PLACED: a ref about to be # written down and read back by every later run, and a record path this diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..7e1a364b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# THE openreposhape PACKAGE (#563): a SECOND WAY IN, NEVER A REQUIREMENT. +# Cloning this repository and running its tools with python3 and git alone +# keeps working, and every tool stays standard-library only +# (`tests/test_repo_hygiene.py` holds the package's own files to that too). +# Brett Heap approved the design on #563 at 2026-10-15T13:32:51Z ("approve +# all six for #563"). +# +# THIN: the wheel holds `src/openreposhape/` as `openreposhape/`, plus ONE +# file of this tree byte for byte, `setup-project.py`, so that `openRepoShape +# --preflight` runs before git is installed. Every other run uses a full +# clone of this repository at the commit the package was built from +# (`src/openreposhape/standard.py`), so a pin means what it always meant. +# +# `src/`, AND NOT THE ROOT, because the root already holds `openRepoShape`, +# the bash launcher: a directory `openreposhape/` beside it is the same name +# on a case-insensitive disk (macOS's and Windows' default), and a clone +# there could not hold both. + +[build-system] +# PINNED, as every tool this repository's CI installs is: a build that +# resolves the newest backend on the day it runs is a build whose bytes +# depend on that day. 1.27.0 is the newest hatchling that still runs on +# Python 3.9, the oldest Python this package supports (1.28.0 and later need +# 3.10). Hatchling, because `force-include` below puts `setup-project.py` +# into the package declaratively. Its wheels take their file times from +# `SOURCE_DATE_EPOCH`, and the `package` job builds twice on one runner at +# the commit's time and compares the two files; that is all it shows. +requires = ["hatchling==1.27.0"] +build-backend = "hatchling.build" + +[project] +name = "openreposhape" +# THE TAG IS THE ONE SOURCE OF THE VERSION: the release build writes it into +# the stamp, `src/openreposhape/_release.py`, with the tag, the commit and +# the digest, and this reads it from there. The committed stamp says 0.0.0. +dynamic = ["version"] +description = "openRepoShape's commands, run from a clone of the standard at the release's own commit" +requires-python = ">=3.9" +license = "Apache-2.0" +license-files = ["LICENSE"] +authors = [{ name = "opensoft" }] +classifiers = [ + "Environment :: Console", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Version Control :: Git", +] +# THE ONE DEPENDENCY, and none of the tools': `adopt-project.py` extracts a +# leg's history with `git filter-repo`. The version CI runs and install.sh +# installs (`.github/workflows/tests.yml`, `install.sh`'s FILTER_REPO_PIN). +# pipx exposes only this package's commands, so each of them puts the +# package environment's scripts directory first on PATH for its children +# whenever a `git-filter-repo` is there (`standard.child_environment`). +dependencies = ["git-filter-repo==2.47.0"] + +[project.urls] +Homepage = "https://github.com/opensoft/openRepoShape" +Issues = "https://github.com/opensoft/openRepoShape/issues" + +# THE FIVE COMMANDS, each a native `.exe` on Windows. `openRepoShape` is the +# launcher's modes and flags; the other four run that tool from the clone, +# arguments unchanged. The names README, AGENTS.md and `docs/cli.md` already +# use, less `./` and `.py`. +[project.scripts] +openRepoShape = "openreposhape.cli:openreposhape" +scaffold-project = "openreposhape.cli:scaffold_project" +adopt-project = "openreposhape.cli:adopt_project" +update-shape = "openreposhape.cli:update_shape" +shape-doctor = "openreposhape.cli:shape_doctor" + +[tool.hatch.version] +path = "src/openreposhape/_release.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/openreposhape"] + +# The release's own `setup-project.py`, beside the package's modules, which +# is where `standard.preflight_entry` looks for it. +[tool.hatch.build.targets.wheel.force-include] +"setup-project.py" = "openreposhape/setup-project.py" diff --git a/src/openreposhape/__init__.py b/src/openreposhape/__init__.py new file mode 100644 index 00000000..497fc755 --- /dev/null +++ b/src/openreposhape/__init__.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +"""openreposhape: openRepoShape's commands, installed with pipx (#563). + +A SECOND WAY IN, NEVER A REQUIREMENT. Cloning this repository and running its +tools with `python3` and `git` keeps working exactly as it does without this +package, and every tool stays standard-library only. The package adds five +commands, `openRepoShape`, `scaffold-project`, `adopt-project`, +`update-shape` and `shape-doctor`, and one dependency, `git-filter-repo`, +which only `adopt-project.py` needs. + +IT IS THIN. It carries no tool, template or contract: `cli` runs them from a +full clone of this repository at the commit the package was built from +(`standard`), and `_release` is the stamp that names that commit. The one +file of the standard it does carry is `setup-project.py`, byte for byte, so +that `openRepoShape --preflight` runs on a machine with no git yet. +""" diff --git a/src/openreposhape/__main__.py b/src/openreposhape/__main__.py new file mode 100644 index 00000000..4db27e33 --- /dev/null +++ b/src/openreposhape/__main__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`python -m openreposhape` is `openRepoShape` (#563): the same command, +for an interpreter whose scripts directory is not on PATH.""" + +import sys + +from .cli import openreposhape + +sys.exit(openreposhape()) diff --git a/src/openreposhape/_release.py b/src/openreposhape/_release.py new file mode 100644 index 00000000..b4b6198b --- /dev/null +++ b/src/openreposhape/_release.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The release this package was built from: its STAMP (#563). + +A release build GENERATES this file from the tag it builds, and nothing else +in the wheel differs from the tag's own tree. The four values are the +version (`v1.1.0` gives `1.1.0`), the tag, the tag's 40-hex commit, and the +`sorted-ls-tree-r-v1` digest of that commit's tree, the digest every pin this +standard writes carries (`scripts/repo_shape.py`, `tree_digest`). + +THE COMMITTED FILE IS UNSTAMPED: version 0.0.0 and no tag, commit or digest. +That is a build from a checkout rather than a release. It follows main, says +so in `openRepoShape --version`, and the release workflow refuses to publish +it. + +`pyproject.toml` reads VERSION from here (`dynamic = ["version"]`), so the +version is written down once. +""" + +VERSION = "0.0.0" +TAG = "" +COMMIT = "" +DIGEST = "" diff --git a/src/openreposhape/cli.py b/src/openreposhape/cli.py new file mode 100644 index 00000000..b92e9cd9 --- /dev/null +++ b/src/openreposhape/cli.py @@ -0,0 +1,585 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The five commands the openreposhape package installs (#563). + +`openRepoShape` RE-STATES THE BASH LAUNCHER (v1.0.1's `openRepoShape`) IN +PYTHON: its four modes, ``, `--preflight`, `--doctor` and +`--help | --version`, its argument routing, one mode per run, `--org` from +the flag or `$OPENREPOSHAPE_ORG` or a prompt and never a default, its usage +text, its checks on a ref, and `--doctor`'s run of the standard's own +`shape-doctor.py`. What it does NOT re-state is fetching: the launcher +fetches `setup.sh` and clones the standard on every run, and this runs the +package's clone of it (`standard.prepare`). Nor `--install` and the ref it +records: pipx placed this command and the stamp names its commit, so +`--install` refuses and names `pipx upgrade openreposhape`. + +`scaffold-project`, `adopt-project`, `update-shape` and `shape-doctor` each +run that file from the clone, with every argument unchanged, in the +directory the command was typed in. They are the names README, AGENTS.md and +`docs/cli.md` already use, less `./` and `.py`, so `docs/cli.md` stays the +one reference for their flags. + +EVERY CHILD GETS `standard.child_environment()`: the package environment's +scripts directory first on PATH wherever the `git-filter-repo` it depends on +is there. + +PURE ASCII, like `setup-project.py`, because Windows PowerShell 5.1 renders a +console in the machine's ANSI code page: the usage text and every refusal +here arrive there intact. +""" + +from __future__ import annotations + +import os +import subprocess +import sys + +from . import standard +from .standard import Refusal + +#: THE USAGE, the launcher's own, less what pipx and the stamp replace: the +#: lines a person meets are the same lines, in the same columns, and the +#: `--install` line says where installing went. +USAGE = """\ +openRepoShape --install refused here: pipx upgrade openreposhape updates it +openRepoShape --preflight check this MACHINE and stop; creates nothing +openRepoShape --doctor [] [--json] [--placement-plan ] + diagnose a REPOSITORY against this standard +openRepoShape [--org ] [setup-project.py options] [-- ] +openRepoShape --help | --version + +Those lines are the whole command: check the machine you are on, diagnose a +repository, scaffold a project. Run with NO arguments it prints this and +exits 1, because a bare run has nothing to do. + + is the assembly-root name, ONE CamelCase token (e.g. Atlas); omit it +and setup asks, if a terminal is attached. `--org` comes from that flag, +else $OPENREPOSHAPE_ORG, else a prompt -- it is NEVER defaulted. Every other +argument reaches setup-project.py unchanged: --visibility, --elected-by, +--family, --into, --yes, --local-remote-dir, and anything after `--`, which +setup-project.py hands on to scaffold-project.py. `python3 setup-project.py +--help` lists them all, and so does `docs/cli.md` in the repository. The +project lands in the directory you ran this from, unless --into says where. + +This command came with the openreposhape package, which installed four more +beside it. Each runs that tool of the standard with every argument +unchanged, in the directory you typed it in; `docs/cli.md` is their +reference: + + scaffold-project adopt-project update-shape shape-doctor + +THE STANDARD THEY RUN is a full clone of openRepoShape at the commit this +package was built from, made once per commit in a cache in your home +directory and kept. Before every run the clone must be at that commit, carry +no change, and digest to the value the package carries (sorted-ls-tree-r-v1, +computed by the clone's own scripts/repo_shape.py), so a project scaffolded +from it pins exactly the release you installed. `--version` names the +release, its commit, its digest and the clone. A build that is not a release +follows main; run from a checkout's src/ (an editable install), it uses that +checkout as it stands. + + $OPENREPOSHAPE_REPO owner/name, URL or path to clone (default opensoft/openRepoShape) + $OPENREPOSHAPE_REF run at this branch, tag or commit instead, for one run; + no digest is checked then (`--shape-ref` on a + scaffold run does the same) + $OPENREPOSHAPE_ORG the organisation, when --org is not given + +`--install` places nothing here. pipx (or pip) placed this command, and +keeps it current: `pipx upgrade openreposhape`. The estate verbs are not +here either. `park` and `resume` -- commit, push and RECORD an estate's open +features, then rebuild them on the next workstation -- are +opensoft/openRepoTools', and its own installer places them: + + curl -fsSL https://raw.githubusercontent.com/opensoft/openRepoTools/main/openRepoTools | bash -s -- --install + +`--doctor [] [--json] [--placement-plan ]` runs `shape-doctor.py` +over (default `.`) and prints one row per check and ONE verdict: +`COMPLIANT`, `COMPLIANT, SHAPE BEHIND`, `DRIFTED`, `MISPLACED`, `INVALID`, +`NOT A SHAPE ROOT` or `CANNOT ANSWER`. Every finding names the command that +fixes it. Exit 0 compliant, 1 findings, 2 not a shape root, 3 when it cannot +answer (`CANNOT ANSWER`, no such , a `--placement-plan` it cannot +write); this command's own refusals exit 2. `--json` is the same report as +one JSON object. The doctor writes no file but the `--placement-plan ` +you name, at the path you give (a relative lands in the directory you +ran this from, even inside the repository), and moves nothing. + +It runs from the package's clone of the standard, which is KEPT for the NEXT +lines that name files in it; the report's last line names that clone (under +`--json`, on stderr, so stdout stays one JSON object). + +`--preflight` is the MACHINE check: setup-project.py's own, run from the copy +this package carries, so it answers on a machine with no git yet. +""" + +#: Why `--install` refuses, said where it does. +INSTALL_REFUSED = ( + "--install places nothing here: this openRepoShape is the " + "openreposhape package's, and the installer that placed it keeps it " + "current:\n" + " pipx upgrade openreposhape\n" + " The one-line install (install.sh) places the bash openRepoShape " + "instead. Use one or the other.") + +#: setup-project.py's value-taking flags, named so their VALUE is never +#: mistaken for the positional . `--shape-ref` is one of them, and +#: is also the ref of this run's clone (`Run.shape_ref`). +VALUE_FLAGS = ("--id", "--name", "--visibility", "--elected-by", "--family", + "--into", "--local-remote-dir", "--shape-ref") + + +def say(text: str = "") -> None: + print(text, flush=True) + + +def utf8_streams() -> None: + """`setup-project.py`'s guard: a child's output and a path in a refusal + travel through these streams, and the console's code page may not encode + them. A stream that cannot be reconfigured is left as it is.""" + for stream in (sys.stdout, sys.stderr): + try: + stream.reconfigure(errors="replace") + except (AttributeError, ValueError, OSError): + pass + + +def terminal_attached() -> bool: + """The launcher's `[ -t 0 ]`.""" + try: + return sys.stdin is not None and sys.stdin.isatty() + except (AttributeError, ValueError, OSError): + return False + + +def run_child(argv: list, env: dict, cwd: str | None = None) -> int: + """`argv`, its stdin, stdout and stderr this command's own, and its exit + status as a shell would report it. + + A CTRL-C REACHES THE CHILD TOO, which is in the same process group and + prints its own refusal, so the first one here waits for the child to + answer it, and only a second one kills it. A child killed by a signal + reports 128 + the signal, as bash does. + """ + sys.stdout.flush() + sys.stderr.flush() + try: + child = subprocess.Popen(argv, env=env, cwd=cwd) + except OSError as exc: + raise Refusal(f"could not start {argv[0]}: {exc}") from exc + interrupted = False + while True: + try: + status = child.wait() + break + except KeyboardInterrupt: + if interrupted: + child.kill() + interrupted = True + return 128 - status if status < 0 else status + + +def guarded(body, argv) -> int: + """`body(argv)`'s exit status, with a refusal printed the launcher's way + (`\\nREFUSED: ` on stderr) and its code returned.""" + utf8_streams() + args = sys.argv[1:] if argv is None else list(argv) + try: + return body(args) + except Refusal as refusal: + sys.stdout.flush() + print(f"\nREFUSED: {refusal}", file=sys.stderr, flush=True) + return refusal.code + except KeyboardInterrupt: + print("\nREFUSED: interrupted.", file=sys.stderr, flush=True) + return 130 + + +# --- openRepoShape ------------------------------------------------------------- + +class Run: + """What the arguments asked for: the launcher's variables, one each.""" + + def __init__(self) -> None: + self.mode = "run" + self.preflight = False + self.project = "" + self.org = "" + self.doctor_path = "" + self.doctor_path_given = False + self.doctor_args: list = [] + self.doctor_json = False + self.plan_file = "" + self.passthrough: list = [] + self.shape_ref = "" + + +def set_mode(run: Run, mode: str) -> None: + """ONE MODE PER RUN: two is a refusal, never last-wins (Copilot, PR + #96). A person who typed both meant one of them.""" + if run.mode not in ("run", mode): + raise Refusal( + f"--{run.mode} and --{mode} are two different runs; pass one.\n" + " --install places nothing here (pipx upgrade openreposhape " + "updates this command);\n" + " --doctor reads a repository and writes nothing into it but " + "a\n --placement-plan you name.") + run.mode = mode + + +def value_of(args: list, at: int, flag: str) -> str: + """The value after `flag`, which must be there and not be empty: the + launcher's `${2:? needs a value}`.""" + if at + 1 >= len(args) or args[at + 1] == "": + raise Refusal(f"{flag} needs a value") + return args[at + 1] + + +#: The doctor's plan flag, in its two spellings. +PLAN_FLAG = "--placement-plan" +PLAN_EQUALS = PLAN_FLAG + "=" + + +def take_placement_plan(run: Run, args: list, at: int) -> int: + """`--placement-plan ` or `--placement-plan=` after + `--doctor`; the index after what it took. The separate word is refused + when it begins with `-`, as argparse refuses it, and a second plan is a + refusal, never last-wins (#546).""" + before = run.plan_file + arg = args[at] + if arg.startswith(PLAN_EQUALS): + run.plan_file = arg[len(PLAN_EQUALS):] + at += 1 + else: + run.plan_file = args[at + 1] if at + 1 < len(args) else "" + at += 1 + if run.plan_file.startswith("-"): + run.plan_file = "" + elif at < len(args): + at += 1 + if not run.plan_file: + raise Refusal( + f"{PLAN_FLAG} needs the file to write the plan to: {PLAN_FLAG} " + f", or {PLAN_EQUALS} for a name that begins with " + "'-'.") + if before: + raise Refusal(f"two placement plans, '{before}' and " + f"'{run.plan_file}'. --doctor writes one: pass " + f"{PLAN_FLAG} once.") + run.doctor_args.append(PLAN_EQUALS + run.plan_file) + return at + + +def take_doctor(run: Run, args: list, at: int) -> int: + """`--doctor []`: the optional path is eaten here, because it would + otherwise be taken for a ; a word beginning with a dash is a + flag, not the path. ONE path, however many `--doctor`s (#546).""" + set_mode(run, "doctor") + at += 1 + if (at < len(args) and not run.doctor_path_given + and not args[at].startswith("-")): + run.doctor_path, run.doctor_path_given = args[at], True + at += 1 + return at + + +def take_doctor_flag(run: Run, args: list, at: int) -> int: + """THE DOCTOR'S TWO FLAGS, ONLY AFTER `--doctor` (#500); anywhere else a + lone flag for setup-project.py, which refuses both.""" + arg = args[at] + if run.mode != "doctor": + run.passthrough.append(arg) + return at + 1 + if arg == "--json": + run.doctor_args.append(arg) + run.doctor_json = True + return at + 1 + return take_placement_plan(run, args, at) + + +def take_value(run: Run, args: list, at: int) -> int: + """A flag and its value: `--org` and `--project` are this command's, and + every other one in `VALUE_FLAGS` travels on, `--shape-ref` noted too.""" + arg = args[at] + value = value_of(args, at, arg) + if arg in ("--org", "--project"): + setattr(run, arg[2:], value) + else: + run.passthrough += [arg, value] + if arg == "--shape-ref": + run.shape_ref = value + return at + 2 + + +def take_word(run: Run, arg: str) -> None: + """A word that is no flag: after `--doctor` and its flags the one path it + has not had, and otherwise the one .""" + if run.mode == "doctor" and not run.doctor_path_given: + run.doctor_path, run.doctor_path_given = arg, True + elif run.project: + raise Refusal(f"two project names, '{run.project}' and '{arg}'. " + "One positional , then flags.") + else: + run.project = arg + + +def take(run: Run, args: list, at: int) -> int: + """The argument at `at`, taken; the index of the next one.""" + arg = args[at] + if arg == "--install": + set_mode(run, "install") + return at + 1 + if arg == "--doctor": + return take_doctor(run, args, at) + if arg in ("--json", PLAN_FLAG) or arg.startswith(PLAN_EQUALS): + return take_doctor_flag(run, args, at) + if arg == "--preflight": + # NAMED for the flag it sets: `--preflight` creates nothing, so the + # organisation and the are not asked for. + run.preflight = True + run.passthrough.append(arg) + return at + 1 + if arg in ("--org", "--project") or arg in VALUE_FLAGS: + return take_value(run, args, at) + if arg == "--": + run.passthrough += ["--", *args[at + 1:]] + return len(args) + if arg.startswith("-"): + run.passthrough.append(arg) + return at + 1 + take_word(run, arg) + return at + 1 + + +def parse(args: list) -> tuple: + """`(action, run)`: the launcher's argument loop, case for case. + `action` is "help" or "version" when one of those was met, which + answers at once, else "run".""" + run = Run() + at = 0 + while at < len(args): + if args[at] in ("-h", "--help"): + return "help", run + if args[at] == "--version": + return "version", run + at = take(run, args, at) + return "run", run + + +def print_version() -> None: + """WHICH BYTES A RUN WOULD USE: the version, the tag, the commit, the + digest and the clone, with no network and no clone made. The ref in + force is in the first line's `( @ )`, as the launcher puts + it.""" + stamp = standard.read_stamp() + beside = standard.beside_checkout() + if beside is not None: + say(f"openRepoShape {stamp.version} (the checkout this package sits " + f"in: {beside})") + say(f" version {stamp.version}, unstamped: a build from a checkout " + "of the standard, not a release") + say(f" checkout {beside}: this package sits in it, so every run " + "uses it as it stands and clones nothing") + return + repo = standard.repository() + standard.clone_url(repo) + wanted = standard.target(stamp) + cache = standard.cache_root() + say(f"openRepoShape {stamp.version} ({repo} @ {wanted.ref})") + if stamp.stamped: + say(f" version {stamp.version}") + say(f" tag {stamp.tag}") + say(f" commit {stamp.commit}") + say(f" digest {stamp.digest} ({standard.DIGEST_DEFINITION})") + else: + say(f" version {stamp.version}, unstamped: a build from a checkout, " + "not a release, so it follows main") + say(" tag none") + say(f" commit none: each run asks {repo} which commit main is at") + say(" digest none: no stamp to check a clone against") + if wanted.overridden: + say(f" ref {wanted.ref}, from {wanted.source}, for this run " + "instead of the commit above; no digest is checked") + if wanted.commit: + clone = cache / wanted.commit + made = "" if clone.is_dir() else \ + " (not cloned yet: the first run that needs it clones it)" + say(f" checkout {clone}{made}") + else: + say(f" checkout {cache}{os.sep}, one clone per commit, " + "kept") + + +def say_the_standard(std: standard.Standard) -> None: + """One line, before a scaffold hands over, naming the standard it uses.""" + if std.beside: + say(f"openRepoShape: the standard is {std.describe()}.") + else: + say(f"openRepoShape: the standard is {std.describe()}, cloned at " + f"{std.root}.") + + +def doctor(run: Run) -> int: + """`--doctor`: RUN THE STANDARD'S OWN `shape-doctor.py`, never a + reimplementation, IN THE PERSON'S OWN DIRECTORY, so a relative `--root` + and a relative `--placement-plan` resolve where they typed (#500). + + THE CLONE IS KEPT AND SAID SO LAST, once the doctor has run: its NEXT + lines name files in it. On stderr under `--json`, so stdout stays one + object. Beside a checkout nothing was cloned, so nothing is said. + """ + std = standard.prepare() + script = std.root / "shape-doctor.py" + if not script.is_file(): + raise Refusal( + f"{std.root} has no shape-doctor.py, so there is nothing to run. " + f"That checkout is {std.describe()}.\n" + " Set $OPENREPOSHAPE_REF to a ref that carries it, or run this " + "from a checkout of the standard.") + status = run_child([sys.executable, str(script), "--root", + run.doctor_path or ".", *run.doctor_args], + standard.child_environment()) + if not std.beside: + print(f"the standard this run used is kept at {std.root}, for any " + "NEXT command above that names it; it is safe to delete once " + "you are done with them, and the next run that needs it clones " + "it again.", file=sys.stderr if run.doctor_json else sys.stdout, + flush=True) + return status + + +def scaffold(run: Run) -> int: + """`` and `--preflight`: setup-project.py, handed what the + launcher hands setup.sh. + + `--preflight` runs the copy this package carries, so it needs no git and + no clone; it examines the machine and stops. A scaffold runs the + clone's own, which pins that clone's commit, with `--into` this + directory FIRST (a later `--into` of the person's wins) and + `$OPENREPOSHAPE_SELF_BOOTSTRAP=1`: setup.sh's hand-over from a clone it + made, where there is no fork origin behind the person to read an + organisation from. + """ + if run.preflight: + entry = standard.preflight_entry() + return run_child([sys.executable, str(entry), *forwarded(run)], + standard.child_environment()) + std = standard.prepare(typed_ref=run.shape_ref) + entry = std.root / "setup-project.py" + if not entry.is_file(): + raise Refusal( + f"{std.root} has no setup-project.py, so there is nothing to run. " + f"That checkout is {std.describe()}.\n" + " Set $OPENREPOSHAPE_REF to a ref that carries it.") + say_the_standard(std) + env = standard.child_environment() + env["OPENREPOSHAPE_SELF_BOOTSTRAP"] = "1" + return run_child([sys.executable, str(entry), "--into", os.getcwd(), + *forwarded(run)], env) + + +def forwarded(run: Run) -> list: + """`--org`, `--project` and every other argument, as the launcher's + `ARGS`. An organisation typed alongside `--preflight` still travels: the + preflight ignores it, and a flag this command ate would be a flag the + person has to type twice to find out about.""" + args = [] + if run.org: + args += ["--org", run.org] + if run.project: + args += ["--project", run.project] + return args + run.passthrough + + +def doctor_alone(run: Run) -> None: + """ONE PATH AND THE DOCTOR'S TWO FLAGS, NOTHING ELSE: every other + argument is about a run that CREATES something.""" + if run.project or run.org or run.passthrough: + raise Refusal( + f"--doctor takes a path, --json and {PLAN_FLAG} , and " + "nothing else; it reads a repository and writes nothing into it " + "but that .\n" + f" openRepoShape --doctor [] [--json] [{PLAN_FLAG} " + "]\n" + " To scaffold, drop --doctor. To check this machine, run: " + "openRepoShape --preflight") + + +def settle_the_names(run: Run) -> None: + """NEVER a default organisation: a wrong one creates three repositories + in somebody else's namespace. The flag, else `$OPENREPOSHAPE_ORG`, else + the terminal; and a setup-project.py can ask for only where a + terminal is attached.""" + run.org = run.org or os.environ.get("OPENREPOSHAPE_ORG", "") + if not run.org and terminal_attached(): + try: + run.org = input("organisation to scaffold into: ").strip() + except EOFError: + run.org = "" + if not run.org: + raise Refusal("no organisation to scaffold into. Re-run with " + "--org , or set $OPENREPOSHAPE_ORG.") + if not run.project and not terminal_attached(): + sys.stderr.write(USAGE) + raise Refusal("no given, and no terminal to ask on.") + + +def main_openreposhape(args: list) -> int: + action, run = parse(args) + if action == "help": + sys.stdout.write(USAGE) + return 0 + if action == "version": + print_version() + return 0 + if run.mode == "doctor": + doctor_alone(run) + return doctor(run) + if run.mode == "install": + raise Refusal(INSTALL_REFUSED) + # A BARE RUN HAS NOTHING TO DO: the usage, and exit 1. Any argument at + # all keeps the scaffold path and the refusal it earned. + if not (run.preflight or run.project or run.org or run.passthrough): + sys.stdout.write(USAGE) + return 1 + if not run.preflight: + settle_the_names(run) + return scaffold(run) + + +def openreposhape(argv=None) -> int: + """The `openRepoShape` console script.""" + return guarded(main_openreposhape, argv) + + +# --- the four tools, passed through --------------------------------------------- + +def run_tool(filename: str, args: list) -> int: + """`/ `, every argument unchanged, in the + directory the command was typed in, and its exit status.""" + std = standard.prepare() + script = std.root / filename + if not script.is_file(): + raise Refusal( + f"{std.root} has no {filename}, so there is nothing to run. That " + f"checkout is {std.describe()}.\n" + " Set $OPENREPOSHAPE_REF to a ref that carries it.") + return run_child([sys.executable, str(script), *args], + standard.child_environment()) + + +def scaffold_project(argv=None) -> int: + """The `scaffold-project` console script: `scaffold-project.py`.""" + return guarded(lambda args: run_tool("scaffold-project.py", args), argv) + + +def adopt_project(argv=None) -> int: + """The `adopt-project` console script: `adopt-project.py`.""" + return guarded(lambda args: run_tool("adopt-project.py", args), argv) + + +def update_shape(argv=None) -> int: + """The `update-shape` console script: `update-shape.py`.""" + return guarded(lambda args: run_tool("update-shape.py", args), argv) + + +def shape_doctor(argv=None) -> int: + """The `shape-doctor` console script: `shape-doctor.py`.""" + return guarded(lambda args: run_tool("shape-doctor.py", args), argv) diff --git a/src/openreposhape/standard.py b/src/openreposhape/standard.py new file mode 100644 index 00000000..af803327 --- /dev/null +++ b/src/openreposhape/standard.py @@ -0,0 +1,654 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The standard a packaged command runs: a clone of openRepoShape (#563). + +A WHEEL HAS NO CHECKOUT, AND THE TOOLS NEED ONE. `scaffold-project.py` pins +the commit it runs from (`git rev-parse HEAD` in its own checkout), +`update-shape.py` and the doctor's currency row read the standard's history, +and every tool finds its templates beside itself. So the package carries no +tool: every command but `--preflight` runs the tools from a FULL clone of +this repository at the commit the package was built from, the commit its +stamp (`_release`) names. + +ONE CLONE PER COMMIT, IN A PER-USER CACHE, KEPT. The first run that needs the +standard clones it into `/` and every later run at that commit +uses it as it is. It is kept because the doctor's NEXT lines name files in it +(#500), and it is a full clone because the doctor's currency row compares +commits (`--depth 1` holds one). + +CHECKED BEFORE EVERY RUN, WHEN NOTHING OVERRIDES THE STAMP. The clone must be +at the stamped commit, carry no change, and digest to the stamped value. The +digest is computed by the clone's OWN `scripts/repo_shape.py` +(`tree_digest`, `sorted-ls-tree-r-v1`), never by a copy in this package, so +it is the digest the scaffold then pins. + +THE REF IS THE STAMPED 40-HEX COMMIT, NEVER THE TAG, because a tag can be +moved. `$OPENREPOSHAPE_REF`, set for one run, names another branch, tag or +commit, as it does for the bash launcher, and then no digest is checked: +the stamp's digest is the stamped commit's. `$OPENREPOSHAPE_REPO` names a +fork or a mirror to clone from. A build with no stamp follows main. + +BESIDE A CHECKOUT, THAT CHECKOUT ANSWERS. Run from the `src/` of a checkout +of this standard (an editable install, or the test suite), the package uses +the checkout it sits in, as it stands, and clones nothing: the launcher's +`--doctor` rule for a copy of itself that sits in a checkout. + +STANDARD LIBRARY ONLY, like every tool here: `tests/test_repo_hygiene.py` +holds this file to it. +""" + +from __future__ import annotations + +import os +import re +import shutil +import stat +import subprocess +import sys +import sysconfig +import tempfile +import time +from pathlib import Path + +from . import _release + +#: Where the standard comes from when `$OPENREPOSHAPE_REPO` is unset or empty. +DEFAULT_REPO = "opensoft/openRepoShape" + +#: What the stamped digest means: `scripts/repo_shape.py`'s +#: `TREE_DIGEST_DEFINITION`, the definition every pin this standard writes +#: records. A clone whose own module says otherwise is not compared. +DIGEST_DEFINITION = "sorted-ls-tree-r-v1" + +#: The one dependency, and the only program whose presence moves PATH. +FILTER_REPO = "git-filter-repo" + +#: A stamped commit and digest are lowercase hex of exactly these lengths. +COMMIT_RE = re.compile(r"[0-9a-f]{40}") +ANY_CASE_COMMIT_RE = re.compile(r"[0-9a-fA-F]{40}") +DIGEST_RE = re.compile(r"[0-9a-f]{64}") + +#: The launcher's ref checks (`ref_fault`), in its order: a control +#: character, then the shape. +CONTROL_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]") +REF_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*") + +#: A Windows path with a drive letter, which is a path to clone and never an +#: owner/name on github.com. +DRIVE_RE = re.compile(r"[A-Za-z]:[\\/]") + +#: The entry point the preflight and a scaffold run, and the two names a +#: ref can come by besides the stamp. +SETUP_ENTRY = "setup-project.py" +REF_VARIABLE = "$OPENREPOSHAPE_REF" +REF_FLAG = "--shape-ref" + +#: What makes a directory a checkout of this standard for the BESIDE rule: +#: the three tools a run reaches for and the naming policy `setup-project.py` +#: itself asks for (`is_shape_checkout`). +CHECKOUT_MARKERS = (SETUP_ENTRY, "scaffold-project.py", + "shape-doctor.py", "contracts/repository-naming.yaml") + +#: The way out of every refusal about a cached clone. The cache is this +#: command's, and a clone it made again is the same clone. +CLONE_AFRESH = "Delete it and run this again: the next run clones it afresh." + +#: Run by the CLONE'S OWN interpreter path with `-I`: its `scripts/` first on +#: `sys.path`, and its `repo_shape` answers, never a copy carried here. +DIGEST_PROGRAM = """\ +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import repo_shape +print(repo_shape.TREE_DIGEST_DEFINITION) +print(repo_shape.tree_digest(Path(sys.argv[2]), sys.argv[3])) +""" + + +class Refusal(Exception): + """A refusal and its exit code, printed as `REFUSED: ` by the + command that raised it, as the launcher's `die` prints one.""" + + def __init__(self, message: str, code: int = 2) -> None: + super().__init__(message) + self.code = code + + +def note(text: str) -> None: + """A line about what this command is doing, on stderr, so that a + `--json` report on stdout stays one object.""" + print(text, file=sys.stderr, flush=True) + + +# --- the stamp --------------------------------------------------------------- + +class Stamp: + """`_release`, read and checked: the version, the tag, the commit and the + digest this package was built from, or a version and three empty strings + for a build with no stamp.""" + + def __init__(self, version: str, tag: str, commit: str, + digest: str) -> None: + self.version = version + self.tag = tag + self.commit = commit + self.digest = digest + + @property + def stamped(self) -> bool: + return bool(self.commit) + + +def read_stamp(module=_release) -> Stamp: + """The stamp, ALL FOUR VALUES OR NONE OF THEM. + + A stamp that names part of a release is a broken build: a commit with no + digest would run unchecked, and a digest with no commit names nothing. + It is refused, never read as the half it holds. + """ + values = [getattr(module, name, "") for name in + ("VERSION", "TAG", "COMMIT", "DIGEST")] + if not all(isinstance(value, str) for value in values): + values = ["", "?", "?", "?"] + version, tag, commit, digest = values + if not (tag or commit or digest): + return Stamp(version or "0.0.0", "", "", "") + if not (version and version != "0.0.0" and tag + and COMMIT_RE.fullmatch(commit) and DIGEST_RE.fullmatch(digest)): + raise Refusal( + "this package's stamp (openreposhape/_release.py) names part of " + "a release and not all of it: a release carries its version, its " + "tag, a 40-hex commit and a 64-hex digest, together. This build " + "is broken. Install a release: pipx install --force openreposhape") + return Stamp(version, tag, commit, digest) + + +# --- where the standard comes from --------------------------------------------- + +def repository() -> str: + """`$OPENREPOSHAPE_REPO`, or opensoft/openRepoShape where it is unset or + empty: the shell's `${VAR:-default}`, as setup.sh and the launcher read + it.""" + return os.environ.get("OPENREPOSHAPE_REPO") or DEFAULT_REPO + + +def clone_url(repo: str) -> str: + """What `git clone` is handed for `repo`: an owner/name becomes its + github.com URL, and a URL, an scp-style address or a path on disk goes + through untouched (the launcher's `shape_clone_url`, and a Windows + drive path besides). + + REFUSED, NOT ECHOED, when it starts with `-` or carries a control + character: the launcher's guard, in its words. The `--` before it on + every git command line is defence in depth, not a substitute. + """ + after_at = repo.split("@", 1)[1] if "@" in repo else "" + if ("://" in repo or ":" in after_at or repo.startswith(("/", ".")) + or os.path.isabs(repo) or DRIVE_RE.match(repo)): + url = repo + else: + url = f"https://github.com/{repo}.git" + if url.startswith("-") or CONTROL_RE.search(url): + raise Refusal( + "OPENREPOSHAPE_REPO is not a value this tool will put on a `git " + "clone` command line: it starts with '-' or carries a control " + "character. Set it to a URL or a path.") + return url + + +def ref_fault(ref: str, what: str) -> str: + """What is wrong with `ref`, or "" when nothing is: the launcher's + `ref_fault`, in its order and its words. LENGTH first, so an over-long + value is refused without being echoed; a CONTROL CHARACTER next, because + a newline in it could forge a line of output; and only then the SHAPE, + which is safe to echo back because it has passed both.""" + if not ref: + return (f"{what} is empty, which is not a branch, tag or commit this " + "tool will pass to `git checkout`.") + if len(ref) > 255: + return (f"{what} is {len(ref)} characters long; a branch, tag or " + "commit this tool will pass to `git checkout` is at most 255 " + "characters.") + if CONTROL_RE.search(ref): + return (f"{what} carries a control character, which is not a " + "branch, tag or commit this tool will pass to `git " + "checkout`.") + if not REF_RE.fullmatch(ref) or ".." in ref or ref.endswith(".lock"): + return (f"{what} is '{ref}', which is not a branch, tag or commit " + "this tool will pass to `git checkout`: a ref starts with a " + "letter or digit, uses only letters, digits, '.', '_', '/', " + "'-', has no '..' and does not end in '.lock'.") + return "" + + +def checked_ref(ref: str, what: str = REF_VARIABLE) -> str: + """`ref`, or a refusal saying what is wrong with it.""" + fault = ref_fault(ref, what) + if fault: + raise Refusal(fault) + return ref + + +class Target: + """The commit a run asks for: `ref` as a person would read it, the + 40-hex `commit` where that is already known, the `digest` to check the + clone against (the stamp's, and only with no override), and `source`, + which of the four ways it was named.""" + + def __init__(self, ref: str, commit: str, digest: str, + source: str) -> None: + self.ref = ref + self.commit = commit + self.digest = digest + self.source = source + + @property + def overridden(self) -> bool: + return self.source in (REF_FLAG, REF_VARIABLE) + + +def target(stamp: Stamp, typed_ref: str = "") -> Target: + """WHICH COMMIT, first match wins: a `--shape-ref` typed on a scaffold + run (the launcher hands setup.sh its own ref FIRST, so a typed one is the + last and wins), `$OPENREPOSHAPE_REF` set for this run, the stamped + commit, and with none of those main. Each named ref is checked here, + before anything reaches git.""" + for ref, source in ((typed_ref, REF_FLAG), + (os.environ.get("OPENREPOSHAPE_REF", ""), + REF_VARIABLE)): + if ref: + checked_ref(ref, source) + commit = ref.lower() if ANY_CASE_COMMIT_RE.fullmatch(ref) else "" + return Target(ref, commit, "", source) + if stamp.stamped: + return Target(stamp.commit, stamp.commit, stamp.digest, "stamp") + return Target("main", "", "", "main") + + +# --- the cache ----------------------------------------------------------------- + +def cache_root() -> Path: + """Where the clones live, one directory per commit: the platform's own + per-user cache. `%LOCALAPPDATA%` on Windows, `~/Library/Caches` on macOS, + and `$XDG_CACHE_HOME` (else `~/.cache`) everywhere else; a relative + `$XDG_CACHE_HOME` is ignored, as the XDG specification says.""" + try: + if os.name == "nt": + base = os.environ.get("LOCALAPPDATA") or str( + Path.home() / "AppData" / "Local") + return Path(base) / "openreposhape" / "checkouts" + if sys.platform == "darwin": + return (Path.home() / "Library" / "Caches" / "openreposhape" + / "checkouts") + base = os.environ.get("XDG_CACHE_HOME", "") + if not os.path.isabs(base): + base = str(Path.home() / ".cache") + return Path(base) / "openreposhape" / "checkouts" + except RuntimeError as exc: + raise Refusal(f"could not find your home directory, where this " + f"command keeps its clone of the standard: {exc}. Set " + f"HOME.") from exc + + +def remove_tree(path: Path) -> None: + """`shutil.rmtree` that also removes a clone on Windows, where git's + object files are read-only and an unlink of one is refused until the bit + is cleared. `onerror`, not `onexc`, which is 3.12 and newer.""" + def clear_readonly(func, target_path, _exc): + os.chmod(target_path, stat.S_IWRITE) + func(target_path) + + shutil.rmtree(str(path), onerror=clear_readonly) + + +# --- the environment every child gets ------------------------------------------ + +def filter_repo_in(directory: str) -> bool: + """Whether `directory` holds a `git-filter-repo` a command line would + start: the bare name, or on Windows any name `PATHEXT` adds an extension + to, which is how pip writes it (`git-filter-repo.exe`).""" + names = [FILTER_REPO] + if os.name == "nt": + extensions = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD") + names += [FILTER_REPO + ext for ext in extensions.split(os.pathsep) + if ext] + for name in names: + candidate = os.path.join(directory, name) + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return True + return False + + +def child_environment(environ=None) -> dict: + """This process's environment for a child, with the package's own + scripts directory FIRST ON PATH WHENEVER A `git-filter-repo` IS THERE. + + pipx exposes the package's own commands and not its dependencies', so the + `git-filter-repo` the package depends on sits in the package + environment's scripts directory, which is on nobody's PATH. adopt finds + it with `shutil.which` and runs `git filter-repo`, which git finds on + PATH; the doctor's machine row looks for it the same way. Putting that + directory first for the children this command starts is the whole of the + fix, and no tool changes for it: where the directory holds none, PATH is + left exactly as it was, and adopt still refuses with + `filter-repo-missing` where the tool is absent. + """ + env = dict(os.environ if environ is None else environ) + scripts = sysconfig.get_path("scripts") + if scripts and filter_repo_in(scripts): + current = env.get("PATH", "") + env["PATH"] = scripts + (os.pathsep + current if current else "") + return env + + +def git(args: list, env: dict) -> subprocess.CompletedProcess: + """`git `, its output captured and decoded as UTF-8.""" + return subprocess.run(["git", *args], env=env, capture_output=True, + encoding="utf-8", errors="replace", check=False) + + +def last_line(text: str) -> str: + lines = [line for line in (text or "").splitlines() if line.strip()] + return lines[-1].strip() if lines else "(nothing on stderr)" + + +# --- the checkout a run uses --------------------------------------------------- + +class Standard: + """The standard a run uses: its `root`, the `commit` it is at, which + `repo` it came from and which `target` named it; or, `beside`, the + checkout this package sits in, used as it stands.""" + + def __init__(self, root: Path, commit: str = "", repo: str = "", + wanted: Target | None = None, beside: bool = False) -> None: + self.root = root + self.commit = commit + self.repo = repo + self.target = wanted + self.beside = beside + + def describe(self) -> str: + """Which standard this is, in a refusal or a progress line.""" + if self.beside: + return f"the checkout this package sits in, {self.root}" + named = self.target.ref if self.target else self.commit + if named == self.commit: + return f"{self.repo} at {self.commit}" + return f"{self.repo} at {named} ({self.commit})" + + +def beside_checkout() -> Path | None: + """The checkout of the standard this package sits in, or None. + + `src/openreposhape/` in a checkout: the package's parent is `src`, and + the directory above that holds the files a run needs. An installed + package sits in a `site-packages`, which holds none of them. + """ + here = Path(__file__).resolve().parent + root = here.parent.parent + if here.parent.name == "src" and all( + (root / marker).is_file() for marker in CHECKOUT_MARKERS): + return root + return None + + +def preflight_entry() -> Path: + """The `setup-project.py` `--preflight` runs: the copy the wheel carries + beside this file, byte for byte the release's own, or in a checkout the + one at its root. Either way no git is needed to reach it.""" + packaged = Path(__file__).resolve().parent / SETUP_ENTRY + if packaged.is_file(): + return packaged + beside = beside_checkout() + if beside is not None: + return beside / SETUP_ENTRY + raise Refusal("this package carries no setup-project.py, so there is no " + "preflight to run. The install is incomplete: pipx install " + "--force openreposhape") + + +def remote_commit(url: str, repo: str, ref: str, env: dict) -> str: + """The commit `ref` names on the remote, read with `git ls-remote`, or + "" when no ref has exactly that name (an abbreviated commit, say, which + only a clone can resolve). + + A TAG FIRST, then a branch, then the name as given (`HEAD`, or a full + `refs/...`): `git checkout ` in a fresh clone resolves it in that + order, and the two must not disagree. An annotated tag is read through + to its commit (`^{}`), named short or in full (`refs/tags/`), and + that line is asked for BY NAME: `git ls-remote` given a name prints a + tag's peeled line only when a pattern names it too, and the tag's own + line names a tag object, not a commit. (A ref here is letters, digits, + `.`, `_`, `/` and `-`, so neither pattern holds a wildcard.) + """ + proc = git(["ls-remote", "--", url, ref, f"{ref}^{{}}"], env) + if proc.returncode != 0: + raise Refusal( + f"could not ask {repo} which commit {ref} is (`git ls-remote` " + f"exited {proc.returncode}: {last_line(proc.stderr)}). This " + "command runs the standard from a clone of it: make the network " + "work, or set $OPENREPOSHAPE_REPO to a mirror you can reach.") + found = {} + for line in proc.stdout.splitlines(): + sha, _, name = line.partition("\t") + found[name.strip()] = sha.strip().lower() + for name in (f"refs/tags/{ref}^{{}}", f"refs/tags/{ref}", + f"refs/heads/{ref}", f"{ref}^{{}}", ref): + if COMMIT_RE.fullmatch(found.get(name, "")): + return found[name] + return "" + + +def move_into_place(incoming: Path, final: Path) -> None: + """`os.rename`, and on Windows a few more tries a moment apart. A + scanner that opens a file git has just written holds it for a moment, + and Windows refuses to rename a directory while it does; an existing + `final` is never retried, because another run placed it first.""" + tries = 10 if os.name == "nt" else 1 + for attempt in range(tries): + try: + os.rename(str(incoming), str(final)) + return + except PermissionError: + if attempt == tries - 1 or final.exists(): + raise + time.sleep(0.2) + + +#: Where a clone is made before it is moved into place, and how long one +#: may sit untouched before a later run takes it for the leftover of a run +#: that was killed mid-clone. +INCOMING_PREFIX = ".incoming-" +INCOMING_STALE_SECONDS = 3600 + + +def sweep_incoming(cache: Path) -> None: + """Remove what a KILLED run left: an `.incoming-*` directory nothing has + touched for an hour. A run that fails, or is interrupted with Ctrl-C, + removes its own (`clone_into`'s `finally`); one ended by SIGTERM, + SIGHUP or SIGKILL cannot, and its half-made clone would otherwise sit in + the cache for good. An hour, so a clone another run is making right now + is left alone. A link is never followed, and a directory that cannot be + removed is left for the next run: this is tidying, never a refusal.""" + cutoff = time.time() - INCOMING_STALE_SECONDS + for leftover in cache.glob(INCOMING_PREFIX + "*"): + try: + if (not leftover.is_symlink() and leftover.is_dir() + and leftover.stat().st_mtime < cutoff): + remove_tree(leftover) + except OSError: + continue + + +def clone_into(cache: Path, url: str, repo: str, wanted: Target, + commit: str, env: dict) -> Path: + """A full clone of `url` at `commit` (or at `wanted.ref`, resolved by the + clone where `commit` is not known), placed at `/`. + + CLONED BESIDE ITS PLACE AND MOVED IN WHOLE. A clone that failed half way, + or a run interrupted, leaves nothing at `/` for the next + run to mistake for a clone; and of two runs cloning at once, the one that + moves second uses the first one's and discards its own. What a killed + run left beside it is swept first (`sweep_incoming`). + """ + try: + cache.mkdir(parents=True, exist_ok=True) + sweep_incoming(cache) + incoming = Path(tempfile.mkdtemp(prefix=INCOMING_PREFIX, + dir=str(cache))) + except OSError as exc: + raise Refusal(f"could not make a directory under {cache}, where " + f"this command keeps its clones of the standard: " + f"{exc}") from exc + placed = False + try: + named = f"commit {commit}" if commit else wanted.ref + note(f"openRepoShape: cloning {repo} into {cache}, once for {named}; " + "every later run at that commit uses this clone.") + cloned = git(["clone", "--quiet", "--no-checkout", "--", url, + str(incoming)], env) + if cloned.returncode != 0: + raise Refusal( + f"could not clone {repo} ({url}): `git clone` exited " + f"{cloned.returncode}: {last_line(cloned.stderr)}. This " + "command runs the standard from a clone of it: make the " + "clone work, or set $OPENREPOSHAPE_REPO to a mirror.") + # ` --`, never `-- `: before the `--` git reads it as a + # commit, after it as a pathspec. A commit already known is checked + # out detached; a name only the clone can resolve is checked out as + # git resolves it. + checkout = (["checkout", "--quiet", "--detach", commit] if commit + else ["checkout", "--quiet", wanted.ref, "--"]) + checked_out = git(["-C", str(incoming), *checkout], env) + if checked_out.returncode != 0: + raise Refusal( + f"cloned {repo} but could not check out " + f"'{commit or wanted.ref}': {last_line(checked_out.stderr)}. " + "Set $OPENREPOSHAPE_REF to a ref that repository carries, or " + "$OPENREPOSHAPE_REPO to one that carries this one.") + if not commit: + head = git(["-C", str(incoming), "rev-parse", "HEAD"], env) + commit = head.stdout.strip().lower() + if head.returncode != 0 or not COMMIT_RE.fullmatch(commit): + raise Refusal(f"cloned {repo} at '{wanted.ref}', but git " + f"could not say which commit that is: " + f"{last_line(head.stderr)}") + final = cache / commit + try: + move_into_place(incoming, final) + placed = True + except OSError as exc: + if not final.is_dir(): + raise Refusal(f"could not move the new clone to {final}: " + f"{exc}") from exc + return final + finally: + if not placed and incoming.exists(): + remove_tree(incoming) + + +def tree_digest(root: Path, commit: str, env: dict) -> tuple: + """`(definition, digest)` of `commit`'s tree, as the clone's own + `scripts/repo_shape.py` computes them. `-I`, so nothing in this process's + environment or user site can stand in for that module, and `-B`, so the + clone gains no bytecode.""" + module = root / "scripts" / "repo_shape.py" + if not module.is_file(): + raise Refusal( + f"{root} has no scripts/repo_shape.py, so its tree digest cannot " + "be computed the way every pin this standard writes computes " + f"it, and nothing was run from it. {CLONE_AFRESH}") + proc = subprocess.run( + [sys.executable, "-I", "-B", "-c", DIGEST_PROGRAM, + str(root / "scripts"), str(root), commit], + env=env, capture_output=True, encoding="utf-8", errors="replace", + check=False) + said = proc.stdout.split() + if proc.returncode != 0 or len(said) != 2: + raise Refusal( + f"could not compute the tree digest of {root} with its own " + f"scripts/repo_shape.py (exit {proc.returncode}): " + f"{last_line(proc.stderr)}") + return said[0], said[1] + + +def verify(standard: Standard, digest: str, env: dict) -> None: + """REFUSE TO RUN FROM A CLONE THAT IS NOT THE COMMIT AS IT WAS MADE: at + another commit, carrying a change (an untracked file included), or, where + a digest is given, digesting to anything else. Nothing is repaired or + removed here, because the doctor's NEXT lines may have sent somebody into + this directory, and what they changed is theirs to look at first.""" + root, commit = standard.root, standard.commit + head = git(["-C", str(root), "rev-parse", "--verify", "HEAD"], env) + at = head.stdout.strip().lower() + if head.returncode != 0 or at != commit: + raise Refusal( + f"{root} should be this command's clone of the standard at " + f"{commit}, and it is at {at or 'no commit git can read'}, so " + f"nothing was run from it. {CLONE_AFRESH}") + status = git(["-C", str(root), "status", "--porcelain", + "--untracked-files=normal"], env) + if status.returncode != 0: + raise Refusal(f"could not ask git whether {root} has changes: " + f"{last_line(status.stderr)}. {CLONE_AFRESH}") + changed = [line[3:] for line in status.stdout.splitlines() if line.strip()] + if changed: + shown = ", ".join(changed[:3]) + if len(changed) > 3: + shown += f", and {len(changed) - 3} more" + raise Refusal( + f"{root}, this command's clone of the standard at {commit}, has " + f"changes ({shown}), so nothing was run from it: a run uses the " + f"commit exactly as it was made. {CLONE_AFRESH}") + if not digest: + return + definition, computed = tree_digest(root, commit, env) + if definition != DIGEST_DEFINITION: + raise Refusal( + f"{root}'s own scripts/repo_shape.py computes the " + f"{definition} digest, and this package was stamped with a " + f"{DIGEST_DEFINITION} one, so the two cannot be compared and " + f"nothing was run from it. {CLONE_AFRESH}") + if computed != digest: + raise Refusal( + f"{root} is at {commit}, and its tree digests to {computed} " + f"({DIGEST_DEFINITION}), not to {digest}, the digest this " + "package was stamped with, so nothing was run from it: that " + f"clone is not the release this package names. {CLONE_AFRESH} " + f"If the new clone says the same, {standard.repo} does not hold " + "that release.") + + +def prepare(typed_ref: str = "") -> Standard: + """The standard this run uses, cloned if it has to be and checked. + + In order: BESIDE a checkout, that checkout, untouched. Otherwise the ref + is checked before anything reaches git; git must be there, because the + standard is a clone; the commit is the stamp's, the 40-hex ref's, or what + the remote says the ref is; a clone of it already in the cache is used, + and one that is not is made; and the clone is verified, against the + stamped digest where nothing overrode the stamp. + """ + beside = beside_checkout() + if beside is not None: + return Standard(beside, beside=True) + stamp = read_stamp() + wanted = target(stamp, typed_ref) + repo = repository() + url = clone_url(repo) + env = child_environment() + if shutil.which("git", path=env.get("PATH")) is None: + raise Refusal( + "git is not installed, and this command runs the standard from " + "a clone of it, so there is nothing to clone one with. Install " + "it: https://git-scm.com/downloads") + commit = wanted.commit or remote_commit( + url, repo, "HEAD" if wanted.source == "main" else wanted.ref, env) + cache = cache_root() + if commit and (cache / commit).is_dir(): + root = cache / commit + else: + root = clone_into(cache, url, repo, wanted, commit, env) + commit = root.name + standard = Standard(root, commit, repo, wanted) + verify(standard, wanted.digest, env) + return standard diff --git a/tests/test_install_sh.py b/tests/test_install_sh.py index dba92000..2c69f66c 100644 --- a/tests/test_install_sh.py +++ b/tests/test_install_sh.py @@ -647,6 +647,45 @@ def test_openreposhape_bin_dir_is_where_the_command_and_its_record_go(net): assert not default.exists() +@pytest.mark.parametrize("dangling", [False, True], ids=["live", "dangling"]) +def test_a_command_that_is_a_symlink_is_refused_before_any_request( + net, dangling): + """pipx places the openreposhape package's `openRepoShape` as a LINK + into the package's own environment, in this same directory (#563). + + The command's `--install` would copy through it, so install.sh refuses + the target ITSELF, before it asks the network anything: the release it + installs may carry a command older than that command's own check. The + link stays as it was, the file it names keeps its bytes, and nothing is + left in `$TMPDIR`. + """ + packaged = net.tmp_path / "pipx" / "venvs" / "openreposhape" / "bin" / \ + "openRepoShape" + original = b"#!/usr/bin/env python3\n# the package's own command\n" + if not dangling: + packaged.parent.mkdir(parents=True) + packaged.write_bytes(original) + net.bin.mkdir(parents=True) + link = net.bin / "openRepoShape" + link.symlink_to(packaged) + result = net.run() + assert result.returncode == 2, result.stdout + result.stderr + said = " ".join(result.stderr.split()) + assert f"{bash_path(link)} is a symbolic link, so nothing was installed" \ + in said, said + assert "pipx upgrade openreposhape" in said + assert "pipx uninstall openreposhape" in said + assert net.calls() == [], net.calls() + assert link.is_symlink() + # Windows reads a link back with `\\?\` ahead of its target. + assert str(link.readlink()).removeprefix("\\\\?\\") == str(packaged) + if not dangling: + assert packaged.read_bytes() == original + assert sorted(path.name for path in net.bin.iterdir()) == \ + ["openRepoShape"] + assert list(net.temporary.iterdir()) == [] + + #: A curl that RUNS, in either script: tab-indented, alone or as the value #: of an assignment. Usage text and refusals carry curl lines too, indented #: with spaces, and are printed, never run. diff --git a/tests/test_openreposhape_command.py b/tests/test_openreposhape_command.py index a91e3409..4f6f2ea6 100644 --- a/tests/test_openreposhape_command.py +++ b/tests/test_openreposhape_command.py @@ -385,6 +385,48 @@ def test_install_refuses_to_also_scaffold(tmp_path): assert "--install takes no other arguments" in result.stderr +@pytest.mark.parametrize("dangling", [False, True], ids=["live", "dangling"]) +def test_install_refuses_a_target_that_is_a_symlink(tmp_path, dangling): + """pipx places the openreposhape package's `openRepoShape` as a LINK + into the package's own environment, in this same directory (#563). + + `cp` writes through a link, so an install over it would overwrite the + package's command and leave the link standing. It is refused before + anything is collected or placed, a dangling link too: the link stays as + it was, the file it names keeps its bytes, nothing is recorded, and the + refusal names both ways out. + """ + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + packaged = tmp_path / "pipx" / "venvs" / "openreposhape" / "bin" / \ + "openRepoShape" + original = b"#!/usr/bin/env python3\n# the package's own command\n" + if not dangling: + packaged.parent.mkdir(parents=True) + packaged.write_bytes(original) + link = bin_dir / "openRepoShape" + link.symlink_to(packaged) + result = run_cmd("--install", home=tmp_path, + env={"OPENREPOSHAPE_BIN_DIR": bash_path(bin_dir), + "OPENREPOSHAPE_REF": "v9.9.9", + "OPENREPOSHAPE_RECORD_REF": "1"}) + assert result.returncode == 2, result.stdout + result.stderr + said = " ".join(result.stderr.split()) + assert f"{bash_path(link)} is a symbolic link, so --install will not " \ + "copy this command through it" in said, said + assert "pipx upgrade openreposhape" in said + assert "pipx uninstall openreposhape" in said + assert link.is_symlink() + # Windows reads a link back with `\\?\` ahead of its target. + assert os.readlink(link).removeprefix("\\\\?\\") == str(packaged) + if not dangling: + assert packaged.read_bytes() == original + assert not packaged.with_name("openRepoShape.ref").exists() + assert sorted(path.name for path in bin_dir.iterdir()) == \ + ["openRepoShape"] + assert "placed" not in result.stdout + + # --- the real thing, offline ------------------------------------------------ def test_it_scaffolds_through_setup_sh(tmp_path): diff --git a/tests/test_openreposhape_package.py b/tests/test_openreposhape_package.py new file mode 100644 index 00000000..5273f1bb --- /dev/null +++ b/tests/test_openreposhape_package.py @@ -0,0 +1,949 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The openreposhape package (#563): its five commands, its stamp, its clone +of the standard, and the PATH its children get. + +THE LAUNCHER'S CASES FIRST, UNCHANGED. `tests/test_openreposhape_command.py` +states the bash `openRepoShape`'s rules; the cases that run it through its +`run_cmd` are imported below, so pytest collects them here as well. A test +function reads `run_cmd` and `USAGE_LINES` from its OWN module when it runs, +so `the_python_command`, autouse here and only here, points both at the +package's `openRepoShape` for each test this module collects: one case, one +rule, both commands, and not a line of the launcher's module changed for +it. CI runs this module on Linux, as it runs the whole suite since #597, +and no CI job runs it on Windows. The four launcher cases that run its FILE +(from stdin, or by its path) are re-stated below for this command. + +`PythonCommand`, called, runs the package from this checkout's `src/`, +where it sits BESIDE a checkout of the standard and uses this one, as the +launcher uses the `setup.sh` beside it; `detached`, it runs an installed +copy of the package (its modules and `setup-project.py`, laid out as the +wheel lays them out) from a directory that is no checkout. + +THEN THE PACKAGE'S OWN. A SYNTHETIC STANDARD stands in for this repository: +a bare repository whose tools are stubs that print what they were handed, +at three refs (the `PINNED` tag, the `OTHER` branch and `main`), with this +checkout's own `scripts/repo_shape.py`, because the package computes a +clone's digest with the clone's own module. A test stamps its installed copy +with that standard's commit and digest, or leaves it unstamped. + +OFFLINE, LIKE THE REST OF THIS SUITE. Every clone is of a bare repository on +disk, and every cache is under the test's own directory: `$XDG_CACHE_HOME`, +`%LOCALAPPDATA%` and, for macOS's `~/Library/Caches`, `$HOME`. Nothing here +builds the wheel or installs it: that needs a build backend and +git-filter-repo from PyPI, so the `package` job in +`.github/workflows/tests.yml` does it, outside pytest. +""" + +from __future__ import annotations + +import ast +import importlib +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +import test_openreposhape_command as launcher_cases +from conftest import REPO, git +from repo_shape import tree_digest +# THE LAUNCHER'S CASES, collected again in this module, where +# `the_python_command` runs them against the package's command. +from test_openreposhape_command import ( + test_a_bare_run_prints_the_usage_and_exits_one, + test_a_family_value_is_never_taken_as_the_project, + test_any_scaffold_argument_keeps_the_old_behaviour, + test_help_prints_every_usage_line, + test_help_still_exits_zero, + test_it_scaffolds_through_setup_sh, + test_no_org_and_no_terminal_refuses_before_setup_sh_runs, + test_no_project_and_no_terminal_refuses_with_the_usage, + test_preflight_forwards_an_org_it_was_given, + test_preflight_passes_through_without_an_org, + test_scaffold_flags_after_a_double_dash_still_reach_the_scaffold, + test_the_org_can_come_from_the_environment, + test_two_modes_is_a_refusal_not_last_wins, +) +from test_openreposhape_command import USAGE_LINES as LAUNCHER_USAGE +from test_openreposhape_command import command_env + +#: The launcher's cases this module runs again, named once more: they are +#: imported for pytest to collect, and nothing here calls them. +LAUNCHER_CASES = ( + test_a_bare_run_prints_the_usage_and_exits_one, + test_a_family_value_is_never_taken_as_the_project, + test_any_scaffold_argument_keeps_the_old_behaviour, + test_help_prints_every_usage_line, + test_help_still_exits_zero, + test_it_scaffolds_through_setup_sh, + test_no_org_and_no_terminal_refuses_before_setup_sh_runs, + test_no_project_and_no_terminal_refuses_with_the_usage, + test_preflight_forwards_an_org_it_was_given, + test_preflight_passes_through_without_an_org, + test_scaffold_flags_after_a_double_dash_still_reach_the_scaffold, + test_the_org_can_come_from_the_environment, + test_two_modes_is_a_refusal_not_last_wins, +) + +SOURCE = REPO / "src" +PACKAGE = SOURCE / "openreposhape" + +#: The package's first usage line, where the launcher's says what +#: `--install` places; the other four are the launcher's own. +USAGE_FIRST = ("openRepoShape --install refused here: pipx " + "upgrade openreposhape updates it") + +#: What a console script does, for the entry named: the package's directory +#: first on `sys.path`, then the function. `-B`, so no bytecode lands in +#: this checkout's `src/`. +BOOT = ("import sys; sys.path.insert(0, sys.argv.pop(1)); " + "import openreposhape.cli as cli; sys.exit(cli.{entry}())") + +#: The synthetic standard's refs, as `test_openreposhape_command.py` names +#: its own. +PINNED = "v9.9.9" +OTHER = "other" + +#: The tools a stub stands in for, each named as the package runs it. +TOOLS = ("setup-project.py", "scaffold-project.py", "adopt-project.py", + "update-shape.py", "shape-doctor.py") + +#: Every stub: one line, `STUB {json}`, naming the file, the ref it was +#: committed at, its arguments, where it ran, what it finds of +#: git-filter-repo, the first PATH entry and the hand-over variable; then +#: `$STUB_SIGNAL` sent to itself where it is set, or `$STUB_EXIT`, so a test +#: sees an exit status travel back. +STUB = '''\ +import json, os, shutil, sys +print("STUB " + json.dumps({ + "tool": os.path.basename(__file__), "at": AT, "argv": sys.argv[1:], + "cwd": os.getcwd(), "filter_repo": shutil.which("git-filter-repo"), + "path0": os.environ.get("PATH", "").split(os.pathsep)[0], + "self_bootstrap": os.environ.get("OPENREPOSHAPE_SELF_BOOTSTRAP")}), + flush=True) +if os.environ.get("STUB_SIGNAL"): + os.kill(os.getpid(), int(os.environ["STUB_SIGNAL"])) +sys.exit(int(os.environ.get("STUB_EXIT", "0"))) +''' + + +# --- the package, run --------------------------------------------------------- + +def installed_copy(directory: Path, stamp: dict | None = None) -> Path: + """The package as the wheel lays it out, under `/site`: its + modules, `setup-project.py` beside them, and `stamp` (`version`, `tag`, + `commit`, `digest`) written into `_release.py` where one is given. The + directory above `site` is no checkout, so the copy is not BESIDE one.""" + package = directory / "site" / "openreposhape" + package.mkdir(parents=True) + for module in sorted(PACKAGE.glob("*.py")): + shutil.copyfile(module, package / module.name) + shutil.copyfile(REPO / "setup-project.py", package / "setup-project.py") + if stamp is not None: + (package / "_release.py").write_bytes("".join( + f'{name.upper()} = "{stamp[name]}"\n' + for name in ("version", "tag", "commit", "digest")).encode()) + return package.parent + + +def package_env(tmp_path: Path, home: Path | None = None, + env: dict | None = None, detached: bool = False) -> dict: + """`command_env`, with every per-user cache the package could reach + under `tmp_path`. `HOME` too for a DETACHED run, where it is where + macOS keeps the cache; a run beside the checkout keeps the person's.""" + caches = {"XDG_CACHE_HOME": str(tmp_path / "cache" / "xdg"), + "LOCALAPPDATA": str(tmp_path / "cache" / "local")} + if detached and home is None: + home = tmp_path / "home" + return command_env(home=home, setup_sh=None, env={**caches, **(env or {})}) + + +def cache_dir(tmp_path: Path) -> Path: + """Where `package_env` puts a detached run's clones, by the platform's + own rule, said here a second time so the package's is checked.""" + if os.name == "nt": + return tmp_path / "cache" / "local" / "openreposhape" / "checkouts" + if sys.platform == "darwin": + return (tmp_path / "home" / "Library" / "Caches" / "openreposhape" + / "checkouts") + return tmp_path / "cache" / "xdg" / "openreposhape" / "checkouts" + + +def run_package(site: Path, entry: str, args, env: dict, + cwd: Path | None = None, + python: Path | str = sys.executable + ) -> subprocess.CompletedProcess: + """The console script `entry` of the package at `site`, run as a person + runs one: stdin a pipe, and a session of its own (no terminal).""" + return subprocess.run( + [str(python), "-B", "-c", BOOT.format(entry=entry), str(site), + *args], capture_output=True, encoding="utf-8", errors="replace", + check=False, input="", cwd=str(cwd) if cwd else None, env=env, + start_new_session=True) + + +class PythonCommand: + """The package's `openRepoShape`, run as the launcher's `run_cmd` runs + the bash one: called with the same keywords, of which `setup_sh` and + `command` mean nothing here, because this command fetches no `setup.sh` + and has no file of its own to run. + + Either way `$OPENREPOSHAPE_REPO` names a repository that does not exist, + unless the test names one, so a run that reached for a clone it should + not need fails on disk rather than asking github.com where main is; and + either way it runs in the test's own directory, so a scaffold that lost + its `--into` lands there and never in this checkout.""" + + usage_lines = (USAGE_FIRST, *LAUNCHER_USAGE[1:]) + + def __init__(self, tmp_path: Path) -> None: + self.tmp_path = tmp_path + + def nowhere(self, env: dict | None) -> dict: + """`env` over a `$OPENREPOSHAPE_REPO` that does not exist.""" + return {"OPENREPOSHAPE_REPO": str(self.tmp_path / "nowhere.git"), + **(env or {})} + + def __call__(self, *args: str, home: Path | None = None, + env: dict | None = None, setup_sh: Path | None = None, + command: Path | None = None) -> subprocess.CompletedProcess: + return run_package(SOURCE, "openreposhape", args, + package_env(self.tmp_path, home, + self.nowhere(env)), + cwd=self.tmp_path) + + def detached(self, *args: str, home: Path | None = None, + env: dict | None = None, + cwd: Path | None = None) -> subprocess.CompletedProcess: + site = self.tmp_path / "package" / "site" + if not site.is_dir(): + installed_copy(self.tmp_path / "package") + return run_package(site, "openreposhape", args, + package_env(self.tmp_path, home, + self.nowhere(env), detached=True), + cwd=cwd or self.tmp_path) + + +@pytest.fixture +def cmd(tmp_path) -> PythonCommand: + return PythonCommand(tmp_path) + + +@pytest.fixture(autouse=True) +def the_python_command(monkeypatch, cmd) -> None: + """For each test THIS module collects, the launcher's module runs the + package's command: its `run_cmd` is `cmd`, and the usage lines it + expects are this command's. Put back after every test, so the + launcher's own module, collected on its own, runs the bash command.""" + monkeypatch.setattr(launcher_cases, "run_cmd", cmd) + monkeypatch.setattr(launcher_cases, "USAGE_LINES", cmd.usage_lines) + + +def stub_report(stdout: str) -> dict: + """What the one stub that ran said about itself.""" + reports = [json.loads(line[len("STUB "):]) for line in stdout.splitlines() + if line.startswith("STUB ")] + assert len(reports) == 1, stdout + return reports[0] + + +def said(result: subprocess.CompletedProcess) -> str: + """stderr with its line breaks folded, for a refusal's sentences.""" + return " ".join(result.stderr.split()) + + +# --- the synthetic standard ----------------------------------------------------- + +class Standard: + """A bare repository on disk, its commit at each ref, and the + `sorted-ls-tree-r-v1` digest of each commit's tree.""" + + def __init__(self, bare: Path, commits: dict, digests: dict) -> None: + self.bare = bare + self.commits = commits + self.digests = digests + + def stamp(self, ref: str = PINNED, **changed) -> dict: + """The stamp a release at `ref` would carry, with `changed` over it.""" + return {"version": "9.9.9", "tag": PINNED, + "commit": self.commits[ref], "digest": self.digests[ref], + **changed} + + +def make_standard(directory: Path) -> Standard: + """`PINNED` (a tag), then `OTHER` (a branch off it), then `main`, each a + commit of every stub saying which ref it is.""" + seed = directory / "seed" + seed.mkdir(parents=True) + git("init", "-q", "-b", "main", cwd=seed) + (seed / "scripts").mkdir() + shutil.copyfile(REPO / "scripts" / "repo_shape.py", + seed / "scripts" / "repo_shape.py") + commits, digests = {}, {} + + def commit(name: str) -> None: + for tool in TOOLS: + (seed / tool).write_text(f"AT = {name!r}\n" + STUB, + encoding="utf-8") + git("add", "-A", cwd=seed) + git("commit", "-q", "-m", name, cwd=seed) + commits[name] = git("rev-parse", "HEAD", cwd=seed).stdout.strip() + digests[name] = tree_digest(seed, commits[name]) + + commit(PINNED) + git("tag", PINNED, cwd=seed) + git("switch", "-q", "-c", OTHER, cwd=seed) + commit(OTHER) + git("switch", "-q", "main", cwd=seed) + commit("main") + bare = directory / "standard.git" + git("clone", "-q", "--bare", str(seed), str(bare), cwd=directory) + return Standard(bare, commits, digests) + + +@pytest.fixture(scope="module") +def shape_standard(tmp_path_factory) -> Standard: + """One synthetic standard for the module: every test only clones it.""" + return make_standard(tmp_path_factory.mktemp("standard")) + + +def stamped_run(tmp_path: Path, standard: Standard, args, *, + stamp: dict | None = None, entry: str = "openreposhape", + env: dict | None = None, cwd: Path | None = None, + site: Path | None = None) -> subprocess.CompletedProcess: + """`entry` of a copy stamped at `PINNED` (or with `stamp`), cloning from + the synthetic standard, in `cwd` (the test's directory by default).""" + if site is None: + site = installed_copy(tmp_path / "package", + standard.stamp() if stamp is None else stamp) + return run_package(site, entry, args, package_env( + tmp_path, env={"OPENREPOSHAPE_REPO": str(standard.bare), + **(env or {})}, detached=True), cwd=cwd or tmp_path) + + +# --- the stamp and the build ------------------------------------------------------ + +def stamp_values(source: bytes) -> dict: + """The names `_release.py` assigns, and their values.""" + return {node.targets[0].id: node.value.value + for node in ast.parse(source).body + if isinstance(node, ast.Assign)} + + +def test_the_committed_stamp_is_unstamped(): + """0.0.0, and no tag, commit or digest: a build from a checkout, which + the release workflow refuses to publish. A release build writes the four + values; nothing commits them.""" + assert stamp_values((PACKAGE / "_release.py").read_bytes()) == { + "VERSION": "0.0.0", "TAG": "", "COMMIT": "", "DIGEST": ""} + + +def test_pyproject_takes_its_version_from_the_stamp_and_names_five_commands( + monkeypatch): + """The version is written once, in the stamp; the one dependency is the + git-filter-repo CI runs and install.sh installs; the wheel is the + package plus `setup-project.py` beside its modules; and each of the five + commands names a function the package has.""" + tomllib = pytest.importorskip("tomllib") + data = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8")) + project = data["project"] + assert project["name"] == "openreposhape" + assert project["dynamic"] == ["version"] + assert "version" not in project + assert project["requires-python"] == ">=3.9" + assert project["dependencies"] == ["git-filter-repo==2.47.0"] + assert 'FILTER_REPO_PIN="git-filter-repo==2.47.0"' in \ + (REPO / "install.sh").read_text(encoding="utf-8") + assert data["build-system"] == {"requires": ["hatchling==1.27.0"], + "build-backend": "hatchling.build"} + hatch = data["tool"]["hatch"] + assert hatch["version"] == {"path": "src/openreposhape/_release.py"} + # hatchling's own default pattern for a version file, read against the + # stamp: the version it finds is the stamp's. + found = re.search(r"(?i)^(__version__|VERSION) *= *(['\"])v?" + r"(?P.+?)\2", + (PACKAGE / "_release.py").read_text(encoding="utf-8"), + re.M) + assert found is not None + assert found.group("version") == "0.0.0" + wheel = hatch["build"]["targets"]["wheel"] + assert wheel["packages"] == ["src/openreposhape"] + assert wheel["force-include"] == { + "setup-project.py": "openreposhape/setup-project.py"} + scripts = project["scripts"] + assert sorted(scripts) == ["adopt-project", "openRepoShape", + "scaffold-project", "shape-doctor", + "update-shape"] + cli = package_module("cli", monkeypatch) + for name, target in scripts.items(): + module, _, function = target.partition(":") + assert module == "openreposhape.cli", name + assert callable(getattr(cli, function)), name + + +def test_a_stamp_that_names_part_of_a_release_is_refused(tmp_path): + """A commit with no digest would run unchecked: the whole stamp or + none of it.""" + site = installed_copy(tmp_path, {"version": "9.9.9", "tag": PINNED, + "commit": "a" * 40, "digest": ""}) + result = run_package(site, "openreposhape", ["--version"], + package_env(tmp_path, detached=True)) + assert result.returncode == 2, result.stdout + result.stderr + assert "names part of a release and not all of it" in said(result) + assert result.stdout == "" + + +# --- --version ---------------------------------------------------------------------- + +def test_version_names_the_release_and_clones_nothing(tmp_path, + shape_standard): + """The version, the tag, the commit, the digest and the clone, from the + stamp alone: no network and no clone, before the first run or after.""" + stamp = shape_standard.stamp() + result = stamped_run(tmp_path, shape_standard, ["--version"]) + assert result.returncode == 0, result.stdout + result.stderr + clone = cache_dir(tmp_path) / stamp["commit"] + assert result.stdout.splitlines() == [ + f"openRepoShape 9.9.9 ({shape_standard.bare} @ {stamp['commit']})", + " version 9.9.9", + f" tag {PINNED}", + f" commit {stamp['commit']}", + f" digest {stamp['digest']} (sorted-ls-tree-r-v1)", + f" checkout {clone} (not cloned yet: the first run that needs it " + "clones it)"] + assert not cache_dir(tmp_path).exists() + + +def test_version_says_an_unstamped_build_follows_main(cmd): + """`$OPENREPOSHAPE_REPO` set EMPTY is unset, the shell's + `${VAR:-default}`: the default repository, named.""" + result = cmd.detached("--version", env={"OPENREPOSHAPE_REPO": ""}) + assert result.returncode == 0, result.stderr + lines = result.stdout.splitlines() + assert lines[0] == "openRepoShape 0.0.0 (opensoft/openRepoShape @ main)" + assert " version 0.0.0, unstamped: a build from a checkout, not a " \ + "release, so it follows main" in lines + + +def test_version_beside_a_checkout_names_the_checkout(cmd): + result = cmd("--version", env={"OPENREPOSHAPE_REF": OTHER}) + assert result.returncode == 0, result.stderr + assert f" checkout {REPO}: this package sits in it, so every run uses " \ + "it as it stands and clones nothing" in result.stdout.splitlines() + + +# --- the launcher's cases that run its FILE, for this command ----------------------- + +def test_the_install_line_keeps_the_launchers_description_column(cmd): + """The launcher's column rule (#546) for this command's usage: the + `--install` line describes itself in the column `--preflight`'s does, + and the doctor's description sits under its form, in that column.""" + lines = cmd("--help").stdout.splitlines() + column = LAUNCHER_USAGE[1].index("check this MACHINE") + assert lines[0] == USAGE_FIRST + assert USAGE_FIRST.index("refused here") == column + doctor = lines.index(LAUNCHER_USAGE[2]) + assert lines[doctor + 1] == \ + " " * column + "diagnose a REPOSITORY against this standard" + + +def test_a_bare_run_asks_nothing_of_git_gh_or_curl(cmd, tmp_path): + """The usage, exit 1, and NO CALL: a `git`, a `gh` and a `curl` that + log every call stand first on PATH (a POSIX script and a `.cmd` each), + and the log must not exist afterwards. DETACHED, where the standard is + a clone and so a git call away, with the repository nowhere: a bare run + that reached for its standard would log a `git ls-remote` here.""" + log = tmp_path / "calls.log" + fakes = tmp_path / "fakes" + fakes.mkdir() + for name in ("git", "gh", "curl"): + posix = fakes / name + posix.write_bytes(f"#!/bin/sh\necho {name} \"$*\" >> '{log}'\n" + "exit 1\n".encode("utf-8")) + posix.chmod(0o755) + (fakes / f"{name}.cmd").write_bytes( + f'@echo {name} %* >> "{log}"\r\n@exit /b 1\r\n'.encode("utf-8")) + result = cmd.detached( + env={"PATH": f"{fakes}{os.pathsep}{os.environ['PATH']}"}) + assert result.returncode == 1, result.stdout + result.stderr + assert USAGE_FIRST in result.stdout.splitlines() + assert not log.exists(), log.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("ref", ["-x", "--upload-pack=touch /tmp/pwned", + "a" * 256, "refs/heads/..", "x.lock", + "v1\nforged"]) +def test_a_ref_is_checked_before_anything_is_cloned(cmd, tmp_path, ref): + """`$OPENREPOSHAPE_REF` reaches `git checkout`, so it is checked first, + in the launcher's three steps and its words. The repository named here + does not exist, so a ref that got past the check would fail on the + clone instead; the over-long and the two-line value are not echoed.""" + result = cmd.detached("--doctor", ".", env={ + "OPENREPOSHAPE_REPO": str(tmp_path / "never-cloned.git"), + "OPENREPOSHAPE_REF": ref}) + assert result.returncode == 2, result.stdout + result.stderr + assert "$OPENREPOSHAPE_REF" in result.stderr + assert "git checkout" in result.stderr + assert "clone" not in said(result) + assert "a" * 256 not in result.stderr + assert "forged" not in result.stderr + + +def test_a_standard_with_no_doctor_is_refused_by_name(cmd, tmp_path): + """A ref of the standard older than `shape-doctor.py` is refused by + name, and the way out is a ref that carries it.""" + seed = tmp_path / "seed" + seed.mkdir() + git("init", "-q", "-b", "main", cwd=seed) + (seed / "README.md").write_bytes(b"# an older standard\n") + git("add", "-A", cwd=seed) + git("commit", "-q", "-m", "before the doctor", cwd=seed) + bare = tmp_path / "old.git" + git("clone", "-q", "--bare", str(seed), str(bare), cwd=tmp_path) + result = cmd.detached("--doctor", ".", + env={"OPENREPOSHAPE_REPO": str(bare)}) + assert result.returncode == 2, result.stdout + result.stderr + assert "has no shape-doctor.py, so there is nothing to run" \ + in said(result) + assert "Set $OPENREPOSHAPE_REF to a ref that carries it" in said(result) + + +def test_version_names_the_ref_set_for_the_run(cmd): + result = cmd.detached("--version", env={"OPENREPOSHAPE_REF": "v1.2.3", + "OPENREPOSHAPE_REPO": ""}) + assert result.returncode == 0, result.stderr + assert result.stdout.splitlines()[0] == \ + "openRepoShape 0.0.0 (opensoft/openRepoShape @ v1.2.3)" + + +# --- the clone: once per commit, and checked before every run ------------------------ + +def test_a_stamped_run_clones_its_commit_once_and_runs_from_it( + tmp_path, shape_standard): + """The first run clones the stamped commit into the cache and runs the + clone's own tool; the second uses that clone as it is, and asks the + repository nothing: it answers with the repository gone.""" + commit = shape_standard.stamp()["commit"] + clone = cache_dir(tmp_path) / commit + first = stamped_run(tmp_path, shape_standard, ["--doctor", "."]) + assert first.returncode == 0, first.stdout + first.stderr + assert stub_report(first.stdout)["at"] == PINNED + assert f"openRepoShape: cloning {shape_standard.bare} into " \ + f"{cache_dir(tmp_path)}, once for commit {commit}" in said(first) + second = run_package( + tmp_path / "package" / "site", "openreposhape", ["--doctor", "."], + package_env(tmp_path, detached=True, + env={"OPENREPOSHAPE_REPO": str(tmp_path / "gone.git")}), + cwd=tmp_path) + assert second.returncode == 0, second.stdout + second.stderr + assert stub_report(second.stdout)["at"] == PINNED + assert "cloning" not in second.stderr + assert sorted(cache_dir(tmp_path).iterdir()) == [clone], ( + "a clone half made, or a second one, was left in the cache") + assert git("rev-parse", "HEAD", cwd=clone).stdout.strip() == commit + + +def test_the_clone_is_full(tmp_path, shape_standard): + """A FULL clone, never a shallow one: the doctor's currency row compares + commits, and `update-shape.py` reads the standard's history. Cloned + through a `file://` URL, because git ignores `--depth` in a clone from a + plain path, so only a URL would show a shallow one: every commit of the + standard is there, the stamped commit's parent and the other branch's + included.""" + result = stamped_run( + tmp_path, shape_standard, ["--doctor", "."], + env={"OPENREPOSHAPE_REPO": shape_standard.bare.as_uri()}) + assert result.returncode == 0, result.stdout + result.stderr + clone = cache_dir(tmp_path) / shape_standard.stamp()["commit"] + assert git("rev-parse", "--is-shallow-repository", + cwd=clone).stdout.strip() == "false" + for ref, commit in shape_standard.commits.items(): + assert git("cat-file", "-e", commit, cwd=clone, + check=False).returncode == 0, f"{ref} ({commit}) is missing" + + +def test_a_clone_a_killed_run_left_is_swept_once_an_hour_has_passed( + tmp_path, shape_standard): + """A run killed mid-clone leaves its `.incoming-*` directory beside the + clones. The next run that clones removes one nothing has touched for an + hour, and leaves a newer one alone: another run may be cloning into it + right now.""" + cache = cache_dir(tmp_path) + old, new = cache / ".incoming-old", cache / ".incoming-new" + for leftover in (old, new): + (leftover / ".git").mkdir(parents=True) + (leftover / ".git" / "HEAD").write_text("ref: refs/heads/main\n", + encoding="utf-8") + two_hours_ago = time.time() - 2 * 3600 + os.utime(old, (two_hours_ago, two_hours_ago)) + result = stamped_run(tmp_path, shape_standard, ["--doctor", "."]) + assert result.returncode == 0, result.stdout + result.stderr + assert not old.exists(), "the killed run's leftover was not swept" + assert (new / ".git" / "HEAD").is_file(), "a fresh one was swept" + assert sorted(path.name for path in cache.iterdir()) == sorted( + [new.name, shape_standard.stamp()["commit"]]) + + +def test_a_clone_whose_digest_is_not_the_stamped_one_is_refused( + tmp_path, shape_standard): + """The digest is the clone's own `repo_shape.tree_digest`, compared with + the stamp's; a clone that digests to anything else runs nothing.""" + wrong = "0" * 64 + result = stamped_run(tmp_path, shape_standard, ["--doctor", "."], + stamp=shape_standard.stamp(digest=wrong)) + assert result.returncode == 2, result.stdout + result.stderr + assert f"its tree digests to {shape_standard.digests[PINNED]} " \ + f"(sorted-ls-tree-r-v1), not to {wrong}, the digest this package " \ + "was stamped with, so nothing was run from it" in said(result) + assert "STUB" not in result.stdout + + +@pytest.mark.parametrize("change", ["modified", "untracked"]) +def test_a_clone_with_changes_is_refused_and_left_as_it_is( + tmp_path, shape_standard, change): + """A run uses the commit exactly as it was made. A clone somebody + changed, an untracked file included, runs nothing and is not repaired: + what changed is theirs to look at first.""" + assert stamped_run(tmp_path, shape_standard, + ["--doctor", "."]).returncode == 0 + clone = cache_dir(tmp_path) / shape_standard.stamp()["commit"] + name = "shape-doctor.py" if change == "modified" else "extra.txt" + with open(clone / name, "ab") as handle: + handle.write(b"# a change\n") + result = stamped_run(tmp_path, shape_standard, ["--doctor", "."], + site=tmp_path / "package" / "site") + assert result.returncode == 2, result.stdout + result.stderr + assert f"has changes ({name}), so nothing was run from it" in said(result) + assert "Delete it and run this again" in said(result) + assert "STUB" not in result.stdout + assert (clone / name).read_bytes().endswith(b"# a change\n") + + +def test_a_clone_at_another_commit_is_refused(tmp_path, shape_standard): + assert stamped_run(tmp_path, shape_standard, + ["--doctor", "."]).returncode == 0 + clone = cache_dir(tmp_path) / shape_standard.stamp()["commit"] + git("checkout", "-q", "--detach", shape_standard.commits["main"], + cwd=clone) + result = stamped_run(tmp_path, shape_standard, ["--doctor", "."], + site=tmp_path / "package" / "site") + assert result.returncode == 2, result.stdout + result.stderr + assert f"and it is at {shape_standard.commits['main']}, so nothing was " \ + "run from it" in said(result) + assert "STUB" not in result.stdout + + +def test_a_commit_the_repository_lacks_leaves_nothing_in_the_cache( + tmp_path, shape_standard): + """The clone is made beside its place and moved in whole, so a clone + that could not check out the commit leaves nothing to mistake for one.""" + result = stamped_run(tmp_path, shape_standard, ["--doctor", "."], + stamp=shape_standard.stamp(commit="b" * 40)) + assert result.returncode == 2, result.stdout + result.stderr + assert f"could not check out '{'b' * 40}'" in said(result) + assert list(cache_dir(tmp_path).iterdir()) == [] + + +# --- which commit ------------------------------------------------------------------- + +@pytest.mark.parametrize("ref", [OTHER, "main", PINNED]) +def test_the_variable_overrides_the_stamp_for_one_run(tmp_path, + shape_standard, ref): + """`$OPENREPOSHAPE_REF` names another branch, tag or commit for the run + it is set for, and no digest is checked then: the stamped digest is the + stamped commit's. (The stamp's digest here is wrong, which a stamped run + would refuse.)""" + stamp = shape_standard.stamp(digest="0" * 64) + result = stamped_run(tmp_path, shape_standard, ["--doctor", "."], + stamp=stamp, env={"OPENREPOSHAPE_REF": ref}) + assert result.returncode == 0, result.stdout + result.stderr + assert stub_report(result.stdout)["at"] == ref + version = stamped_run(tmp_path, shape_standard, ["--version"], + site=tmp_path / "package" / "site", + env={"OPENREPOSHAPE_REF": ref}) + assert f"({shape_standard.bare} @ {ref})" in version.stdout + assert f" ref {ref}, from $OPENREPOSHAPE_REF, for this run " \ + "instead of the commit above; no digest is checked" in version.stdout + + +def test_a_ref_that_is_a_tag_and_a_branch_is_the_tag(tmp_path): + """A name that is a tag AND a branch is the tag, which is what `git + checkout` in a fresh clone makes of it; an annotated tag is read through + to its commit (`^{}`), never taken for one, by its short name and by + its full `refs/tags/` name alike.""" + standard = make_standard(tmp_path / "both") + commits = standard.commits + git("branch", PINNED, commits[OTHER], cwd=standard.bare) + git("tag", "-a", "-m", "annotated", "annotated", commits[OTHER], + cwd=standard.bare) + git("branch", "annotated", commits["main"], cwd=standard.bare) + site = installed_copy(tmp_path / "package", standard.stamp()) + for ref, at in ((PINNED, PINNED), ("annotated", OTHER), + ("refs/tags/annotated", OTHER)): + result = stamped_run(tmp_path, standard, ["--doctor", "."], + site=site, env={"OPENREPOSHAPE_REF": ref}) + assert result.returncode == 0, result.stdout + result.stderr + assert stub_report(result.stdout)["at"] == at, ref + + +def test_an_unstamped_build_follows_main(tmp_path): + """Each run asks the repository where main is: a commit landing there + is the next run's standard, in a clone of its own.""" + standard = make_standard(tmp_path / "moving") + site = installed_copy(tmp_path / "package") + env = package_env(tmp_path, detached=True, + env={"OPENREPOSHAPE_REPO": str(standard.bare)}) + first = run_package(site, "openreposhape", ["--doctor", "."], env, + cwd=tmp_path) + assert first.returncode == 0, first.stdout + first.stderr + assert stub_report(first.stdout)["at"] == "main" + seed = tmp_path / "moving" / "seed" + (seed / "shape-doctor.py").write_text("AT = 'main, later'\n" + STUB, + encoding="utf-8") + git("commit", "-q", "-am", "later", cwd=seed) + git("push", "-q", str(standard.bare), "main", cwd=seed) + later = git("rev-parse", "HEAD", cwd=seed).stdout.strip() + second = run_package(site, "openreposhape", ["--doctor", "."], env, + cwd=tmp_path) + assert second.returncode == 0, second.stdout + second.stderr + assert stub_report(second.stdout)["at"] == "main, later" + assert sorted(path.name for path in cache_dir(tmp_path).iterdir()) == \ + sorted([standard.commits["main"], later]) + + +# --- the five commands ------------------------------------------------------------ + +@pytest.mark.parametrize("entry, tool", [ + ("scaffold_project", "scaffold-project.py"), + ("adopt_project", "adopt-project.py"), + ("update_shape", "update-shape.py"), + ("shape_doctor", "shape-doctor.py"), +]) +def test_each_tool_command_runs_its_file_with_every_argument_unchanged( + tmp_path, shape_standard, entry, tool): + """The clone's own file, every argument as typed (an empty one, a `--` + and a flag after it included), in the directory the command was typed + in, and the file's exit status back.""" + args = ["plan", "--source", "a path/with a space", "--", "-x", "", + "--json"] + typed_in = tmp_path / "typed here" + typed_in.mkdir() + result = stamped_run(tmp_path, shape_standard, args, entry=entry, + env={"STUB_EXIT": "7"}, cwd=typed_in) + assert result.returncode == 7, result.stdout + result.stderr + report = stub_report(result.stdout) + assert (report["tool"], report["at"]) == (tool, PINNED) + assert report["argv"] == args + assert Path(report["cwd"]).resolve() == typed_in.resolve() + + +@pytest.mark.skipif(os.name == "nt", reason="a POSIX signal's exit status: " + "Windows ends a process with the status it is handed") +def test_a_tool_killed_by_a_signal_exits_128_plus_the_signal( + tmp_path, shape_standard): + """As a shell reports it: a tool ended by SIGTERM exits 143, never the + -15 `subprocess` reports, which an exit status would wrap to 241.""" + result = stamped_run(tmp_path, shape_standard, ["--json"], + entry="shape_doctor", + env={"STUB_SIGNAL": str(int(signal.SIGTERM))}) + assert result.returncode == 128 + signal.SIGTERM, ( + result.stdout + result.stderr) + assert stub_report(result.stdout)["tool"] == "shape-doctor.py" + + +def test_a_scaffold_hands_over_to_the_clones_setup_project(tmp_path, + shape_standard): + """setup.sh's hand-over from a clone it made: `--into` where the + command was typed, FIRST, so a later one of the person's wins; then + `--org` and `--project`; then every other argument; and + `$OPENREPOSHAPE_SELF_BOOTSTRAP=1`, because no fork origin stands behind + the person.""" + typed_in = tmp_path / "here" + typed_in.mkdir() + result = stamped_run(tmp_path, shape_standard, + ["Atlas", "--org", "TestOrg", "--visibility", + "private", "--", "--reference", "x.md"], + cwd=typed_in) + assert result.returncode == 0, result.stdout + result.stderr + report = stub_report(result.stdout) + assert (report["tool"], report["at"]) == ("setup-project.py", PINNED) + assert report["argv"][0] == "--into" + assert Path(report["argv"][1]).resolve() == typed_in.resolve() + assert report["argv"][2:] == ["--org", "TestOrg", "--project", "Atlas", + "--visibility", "private", "--", + "--reference", "x.md"] + assert report["self_bootstrap"] == "1" + commit = shape_standard.stamp()["commit"] + assert f"openRepoShape: the standard is {shape_standard.bare} at " \ + f"{commit}, cloned at {cache_dir(tmp_path) / commit}." \ + in result.stdout.splitlines() + + +def test_a_typed_shape_ref_is_the_ref_of_the_scaffolds_clone( + tmp_path, shape_standard): + """`--shape-ref` reached setup.sh after the launcher's own ref, so it + named the ref of the scaffold's clone. Here it does the same, over the + stamp, with no digest checked, and still travels on to setup-project.py, + which checks it as it checks every flag.""" + result = stamped_run(tmp_path, shape_standard, + ["Atlas", "--org", "TestOrg", "--shape-ref", OTHER], + stamp=shape_standard.stamp(digest="0" * 64)) + assert result.returncode == 0, result.stdout + result.stderr + report = stub_report(result.stdout) + assert report["at"] == OTHER + assert report["argv"][-2:] == ["--shape-ref", OTHER] + + +def test_the_doctor_says_last_which_clone_it_kept(tmp_path, shape_standard): + """Its NEXT lines name files in the clone, so the clone is kept and the + report's last line says where; under `--json` that line goes to stderr + and stdout stays the doctor's. The doctor's own flags arrive as the + launcher hands them on.""" + clone = cache_dir(tmp_path) / shape_standard.stamp()["commit"] + kept = (f"the standard this run used is kept at {clone}, for any NEXT " + "command above that names it") + plain = stamped_run(tmp_path, shape_standard, ["--doctor", "../x"]) + assert plain.returncode == 0, plain.stdout + plain.stderr + assert plain.stdout.splitlines()[-1].startswith(kept), plain.stdout + as_json = stamped_run(tmp_path, shape_standard, + ["--doctor", "--json", "../x", + "--placement-plan", "plan.yaml"], + site=tmp_path / "package" / "site") + assert as_json.returncode == 0, as_json.stdout + as_json.stderr + assert stub_report(as_json.stdout)["argv"] == [ + "--root", "../x", "--json", "--placement-plan=plan.yaml"] + assert "kept at" not in as_json.stdout + assert kept in as_json.stderr + + +def test_install_is_refused_and_names_pipx_upgrade(cmd, tmp_path): + """pipx (or pip) placed this command and keeps it current, so + `--install` places nothing, from a checkout or installed, and says how + to update it.""" + for run in (cmd, cmd.detached): + result = run("--install", home=tmp_path, + env={"OPENREPOSHAPE_BIN_DIR": str(tmp_path / "bin")}) + assert result.returncode == 2, result.stdout + result.stderr + assert "--install places nothing here" in said(result) + assert "pipx upgrade openreposhape" in said(result) + assert not (tmp_path / "bin").exists() + assert not (tmp_path / ".local").exists() + + +# --- with no git ---------------------------------------------------------------------- + +def no_git_path(tmp_path: Path) -> dict: + """A PATH with nothing on it, and so no git.""" + empty = tmp_path / "nothing-on-path" + empty.mkdir() + return {"PATH": str(empty)} + + +def test_preflight_runs_before_git_is_installed(tmp_path): + """`--preflight` runs the `setup-project.py` the package carries, so a + machine with no git still gets the preflight, which says git is + missing, rather than this command's refusal for want of a clone.""" + site = installed_copy(tmp_path) + result = run_package(site, "openreposhape", + ["--preflight", "--local-remote-dir", + str(tmp_path / "remotes")], + package_env(tmp_path, detached=True, + env=no_git_path(tmp_path))) + assert result.returncode == 1, result.stdout + result.stderr + assert "(1) preflight" in result.stdout + assert "git is not installed" in result.stderr + assert "REFUSED" not in result.stderr + assert not cache_dir(tmp_path).exists() + + +def test_with_no_git_a_run_that_needs_the_standard_is_refused(tmp_path): + site = installed_copy(tmp_path) + result = run_package(site, "openreposhape", ["--doctor", "."], + package_env(tmp_path, detached=True, + env=no_git_path(tmp_path)), + cwd=tmp_path) + assert result.returncode == 2, result.stdout + result.stderr + assert "git is not installed, and this command runs the standard from " \ + "a clone of it" in said(result) + + +# --- git-filter-repo, for every child ------------------------------------------------- + +def package_module(name: str, monkeypatch): + """`openreposhape.` from this checkout's `src/`, imported in this + process with no bytecode written; `sys.path` and the bytecode flag are + put back when the test ends.""" + monkeypatch.syspath_prepend(str(SOURCE)) + monkeypatch.setattr(sys, "dont_write_bytecode", True) + return importlib.import_module(f"openreposhape.{name}") + + +def fake_filter_repo(directory: Path) -> Path: + """A `git-filter-repo` in `directory` a command line would start, as pip + writes one: a script on POSIX, and on Windows a name `PATHEXT` knows.""" + if os.name == "nt": + fake = directory / "git-filter-repo.cmd" + fake.write_bytes(b"@exit /b 0\r\n") + else: + fake = directory / "git-filter-repo" + fake.write_bytes(b"#!/bin/sh\nexit 0\n") + fake.chmod(0o755) + return fake + + +def test_children_get_the_scripts_directory_first_when_it_holds_filter_repo( + tmp_path, monkeypatch): + """First on PATH when a `git-filter-repo` is there, and PATH untouched + when none is.""" + standard = package_module("standard", monkeypatch) + scripts = tmp_path / "scripts" + scripts.mkdir() + monkeypatch.setattr(standard.sysconfig, "get_path", + lambda name, *rest: str(scripts)) + before = {"PATH": os.pathsep.join(["/one", "/two"]), "OTHER": "kept"} + assert standard.child_environment(before) == before + fake_filter_repo(scripts) + assert standard.child_environment(before) == { + "PATH": os.pathsep.join([str(scripts), "/one", "/two"]), + "OTHER": "kept"} + + +def test_a_child_finds_the_git_filter_repo_of_the_packages_environment( + tmp_path, shape_standard, monkeypatch): + """THE PIPX CASE, END TO END: the package runs in an environment of its + own whose scripts directory holds `git-filter-repo` and is on nobody's + PATH. The tool it starts finds that one, first on its PATH; and before + there is one there, it finds none, and PATH is as it was.""" + venv = tmp_path / "venv" + subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(venv)], + check=True, capture_output=True) + python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + scripts = python.parent + standard = package_module("standard", monkeypatch) + elsewhere = [entry for entry in os.environ["PATH"].split(os.pathsep) + if entry and not standard.filter_repo_in(entry)] + site = installed_copy(tmp_path / "package", shape_standard.stamp()) + env = package_env(tmp_path, detached=True, env={ + "OPENREPOSHAPE_REPO": str(shape_standard.bare), + "PATH": os.pathsep.join(elsewhere)}) + + def adopt(): + result = run_package(site, "adopt_project", ["plan"], env, + cwd=tmp_path, python=python) + assert result.returncode == 0, result.stdout + result.stderr + return stub_report(result.stdout) + + before = adopt() + assert before["filter_repo"] is None + assert before["path0"] == elsewhere[0] + fake = fake_filter_repo(scripts) + after = adopt() + assert os.path.normcase(after["filter_repo"]) == os.path.normcase( + str(fake)) + assert os.path.normcase(after["path0"]) == os.path.normcase(str(scripts)) diff --git a/tests/test_repo_hygiene.py b/tests/test_repo_hygiene.py index 402d39dd..4d3184bc 100644 --- a/tests/test_repo_hygiene.py +++ b/tests/test_repo_hygiene.py @@ -43,7 +43,20 @@ REPO / "templates" / "family-root" / "scripts" / "validate-family.py", REPO / "templates" / "family-root" / "scripts" / "bootstrap.py", REPO / "templates" / "family-root" / "scripts" / "siblings.py", + REPO / "src" / "openreposhape" / "__init__.py", + REPO / "src" / "openreposhape" / "__main__.py", + REPO / "src" / "openreposhape" / "_release.py", + REPO / "src" / "openreposhape" / "cli.py", + REPO / "src" / "openreposhape" / "standard.py", ] +#: THE openreposhape PACKAGE joins them on 2026-10-15 (#563), every module of +#: it, and `test_the_package_is_every_module_shipped` below holds the list to +#: the directory. pipx installs it with one dependency, git-filter-repo, which +#: it never imports: it only puts that program on its children's PATH. A +#: package that imported anything else would make the second way in need +#: more than the first, and the first is python3 and git alone. Its modules +#: import each other relatively, which the walk above does not count. +#: #: `bootstrap` joins them on 2026-09-09 (#76): the holder's `siblings.py` #: imports the credential resolution and the member rows from the #: `bootstrap.py` beside it rather than defining either a second time, and the @@ -108,6 +121,14 @@ def test_every_shipped_script_compiles(): compile(path.read_text(encoding="utf-8"), str(path), "exec") +def test_the_package_is_every_module_shipped(): + """Every module of `src/openreposhape/` is in `SHIPPED`, so a module + added to the package is held to the standard library like the rest.""" + package = REPO / "src" / "openreposhape" + assert sorted(package.glob("*.py")) == sorted( + path for path in SHIPPED if path.parent == package) + + @pytest.mark.parametrize("template", ["assembly-root", "family-root"]) def test_the_materializers_account_for_every_template_file(template): """A file added to a root template but named in no copy list would @@ -1465,6 +1486,21 @@ def test_setup_project_py_is_pure_ascii(name): f"`grep -nP '[^\\x00-\\x7F]' {name}`") +@pytest.mark.parametrize( + "path", sorted((REPO / "src" / "openreposhape").glob("*.py")), + ids=lambda p: p.name) +def test_the_package_is_pure_ascii(path): + """The openreposhape package (#563), held to `setup-project.py`'s rule + for the same reason: its `openRepoShape` prints its usage and every + refusal to whatever console the command was typed in, and on a stock + Windows that is PowerShell 5.1's. Its wheel carries `setup-project.py` + as well, which the test above already holds.""" + name = path.relative_to(REPO).as_posix() + assert path.read_bytes().isascii(), ( + f"{name} must be pure ASCII; find the offending line with " + f"`grep -nP '[^\\x00-\\x7F]' {name}`") + + #: THE ONE LINE THAT INSTALLS THE COMMANDS THIS REPOSITORY NO LONGER SHIPS #: (#92). `park` and `resume` moved to opensoft/openRepoTools, and what is #: left here is a pointer at its installer: printed by `openRepoShape @@ -1484,8 +1520,10 @@ def test_setup_project_py_is_pure_ascii(name): #: Every file that carries it. The shim is where a person meets it, README.md #: is where they look it up, and `docs/handbook.html` is the README's designed -#: reading - a page a reader copies from as readily as either. -OPENREPOTOOLS_INSTALL_IN = ["openRepoShape", "README.md", "docs/handbook.html"] +#: reading - a page a reader copies from as readily as either. The package's +#: `openRepoShape` prints it in its usage, as the shim does (#563). +OPENREPOTOOLS_INSTALL_IN = ["openRepoShape", "README.md", "docs/handbook.html", + "src/openreposhape/cli.py"] @pytest.mark.parametrize("name", OPENREPOTOOLS_INSTALL_IN)