diff --git a/install.sh b/install.sh index c63893b9..e2b6efee 100755 --- a/install.sh +++ b/install.sh @@ -125,17 +125,30 @@ USAGE # setup.sh makes; there is no release at either, so this refuses them by # name. A control character is refused before anything is echoed, and # before it a character a terminal draws as something other than itself -# (#837), named by its code point: first, because `[![:print:]]` is the -# locale's, matching every byte past ASCII under `C` and none of those -# characters under a UTF-8 locale, where the refusal below would echo them. +# (#837), named by its code point: first, because the control check +# matches none of those characters, in any locale, and the owner/name +# refusal after it echoes the value. +# +# A CONTROL CHARACTER IS THE LAUNCHER'S RULE, #760's, which `main` holds +# the directory to (#940): under `LC_ALL=C`, in a subshell, `[[:cntrl:]]` +# or U+0080 to U+009F's two bytes (`\302`, then `\200` to `\237`), so the +# answer is the same in every locale. It was `[![:print:]]`, which under +# `C` matches every byte past ASCII: a letter past ASCII was refused as a +# control character there, and as no owner/name under a UTF-8 locale. It is +# no owner/name in either now, in those words. The trade is the launcher's +# (#867): a byte that is not UTF-8, as a lone `0x9B` or an overlong form of +# a control, passes, in every locale, and the refusal that names the value +# prints it: a terminal that reads UTF-8 draws it as a character it cannot +# read, and only one set to an 8-bit charset acts on it. `checked_version` +# and the option check in `main` hold their values to the same rule. checked_repo() { local misdrawn misdrawn="$(LC_ALL=C; misdrawn_characters "$REPO")" [ -z "$misdrawn" ] || die "\$OPENREPOSHAPE_REPO holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. It must be an owner/name on github.com, such as opensoft/openRepoShape." - case "$REPO" in - *[![:print:]]*) die "\$OPENREPOSHAPE_REPO carries a control character; it must be an owner/name on github.com, such as opensoft/openRepoShape." ;; - esac + if (LC_ALL=C; case "$REPO" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + die "\$OPENREPOSHAPE_REPO carries a control character; it must be an owner/name on github.com, such as opensoft/openRepoShape." + fi case "$REPO" in [A-Za-z0-9]*/[A-Za-z0-9._-]*) ;; *) die "\$OPENREPOSHAPE_REPO is '$REPO', which is not an owner/name on github.com, such as opensoft/openRepoShape: this installer builds github.com URLs from it." ;; @@ -151,7 +164,12 @@ checked_repo() { # shape, which is the only one that echoes it), plus the empty value, and, # between the length and the control character, a character a terminal # draws as something other than itself, as the launcher's `ref_fault` has -# it (#837). Not shared, because this file arrives alone down a pipe. +# it (#837). The control character is `checked_repo`'s rule, above, with +# its trade (#940): a byte that is not UTF-8, as a lone `0x9B` or an +# overlong form of a control, passes, and a version with a letter past +# ASCII is the shape's to refuse, in its words, in every locale. A tag the +# redirect names is printable ASCII by then (`latest_from_redirect`). Not +# shared, because this file arrives alone down a pipe. # # THE SHAPE IS GIT'S RULE PART BY PART (#874), as `ref_fault`'s and # `setup.sh`'s are: no `..`, no empty part between slashes, no part that @@ -169,9 +187,9 @@ checked_version() { misdrawn="$(LC_ALL=C; misdrawn_characters "$ref")" [ -z "$misdrawn" ] || die "$what holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. No release tag, branch or commit holds one." - case "$ref" in - *[![:print:]]*) die "$what carries a control character, which is not a release tag, branch or commit." ;; - esac + if (LC_ALL=C; case "$ref" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + die "$what carries a control character, which is not a release tag, branch or commit." + fi case "$ref" in [!A-Za-z0-9]* | *..* | */.* | *.lock | *.lock/* | *//* | */ | *. | *[!A-Za-z0-9._/-]*) die "$what is '$ref', which is not a release tag, branch or commit: a ref starts with an ASCII letter or digit, uses only ASCII letters, digits, '.', '_', '/', '-', has no '..' (git reads it as a revision range), no empty part between slashes ('//', or a '/' at the end), no part that starts with '.' or ends in '.lock' (the name git gives the lock file it writes beside a ref), and does not end in '.'." ;; esac @@ -189,7 +207,7 @@ checked_version() { # itself, never gathered into an array, so a reader and the analyser both # see them there. `tests/test_install_sh.py` holds every curl line to it. latest_from_redirect() { - local url="$1" headers line value status=0 + local url="$1" headers line value tag status=0 FOUND="" command -v curl >/dev/null 2>&1 || { FAULT="curl is not installed" @@ -213,13 +231,33 @@ latest_from_redirect() { done case "$value" in */releases/tag/?*) - FOUND="${value##*/releases/tag/}" - return 0 + # The tag is the network's too, and `checked_version`, which + # it meets next, holds it to the byte rule a person's own + # values are held to (#940), whose trade passes a byte that + # is not UTF-8 and whose shape refusal prints the tag. So it + # is taken only when it is printable ASCII, under `LC_ALL=C`, + # which gives one answer in every locale. A release tag is + # ASCII, so this refuses no real one; any other tag names no + # release, unnamed, and `gh` is asked next. + tag="${value##*/releases/tag/}" + if (LC_ALL=C; case "$tag" in *[![:print:]]*) exit 1 ;; *) exit 0 ;; esac); then + FOUND="$tag" + return 0 + fi + FAULT="it redirects to a location that names no release" + return 1 ;; esac # The location is named only when it is text a terminal draws as # it is: no control character, and no character it draws as - # something other than itself (#837). + # something other than itself (#837). It keeps the locale's + # `[![:print:]]`, not the byte rule a person's own values are + # held to (#940): it is the network's, naming it only lets the + # fault say more, and that class takes a byte that is not UTF-8, + # as a lone `0x9B` or an overlong form of a control, in no + # locale, where the byte rule's trade would print one from the + # network. The cost: under `C` a location with a letter past + # ASCII goes unnamed. FAULT="it redirects to a location that names no release" if [ -z "$(LC_ALL=C; misdrawn_characters "$value")" ]; then case "$value" in @@ -450,9 +488,12 @@ main() { misdrawn="$(LC_ALL=C; misdrawn_characters "$1")" [ -z "$misdrawn" ] || die "an option holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. The one argument is a version, such as v1.0.1; with none, the newest release." - case "$1" in - *[![:print:]]*) die "an option carries a control character. The one argument is a version, such as v1.0.1; with none, the newest release." ;; - esac + # `checked_repo`'s rule for a control character, with its + # trade: a byte that is not UTF-8, as a lone `0x9B` or an + # overlong form of a control, passes (#940). + if (LC_ALL=C; case "$1" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + die "an option carries a control character. The one argument is a version, such as v1.0.1; with none, the newest release." + fi die "unknown option '$1'. The one argument is a version, such as v1.0.1; with none, the newest release." ;; *) @@ -474,11 +515,13 @@ main() { # from the release that added the check, and only after this file had # fetched it. The `case` is `install_commands`'s in `openRepoShape`, # copied for the reason the drive-letter one below is, and the refusal - # says nothing of the value. `[[:cntrl:]]` and not `[![:print:]]`, which - # `checked_repo` uses: a directory with a letter the locale does not - # print (`café` under LC_ALL=C) is no fault of the directory's. The check - # runs under `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, - # then `\200` to `\237`), so a C1 control is refused in every locale, + # says nothing of the value. `[[:cntrl:]]` and not `[![:print:]]`: a + # directory with a letter the locale does not print (`café` under + # LC_ALL=C) is no fault of the directory's, and `checked_repo`, + # `checked_version` and the option check hold their values to these + # same bytes (#940). The check runs under `LC_ALL=C` and names U+0080 + # to U+009F's two bytes (`\302`, then `\200` to `\237`), so a C1 + # control is refused in every locale, # whatever else the value holds, unless the locale reads those bytes as # text (#858, below); U+2028 and U+2029, which only a UTF-8 locale's # `[[:cntrl:]]` matched, are the next check's (#837). diff --git a/openRepoShape b/openRepoShape index 1abd0d0d..661796d2 100755 --- a/openRepoShape +++ b/openRepoShape @@ -603,11 +603,12 @@ install_commands() { # spelling below is. The control characters go first, and their refusal # says nothing of the value, so that no refusal after it echoes one that # can forge a line of output. It is `[[:cntrl:]]` and not - # `[![:print:]]`, which is what this file holds the repository and the - # ref to: a name with a letter the locale does not print, as `café` - # under LC_ALL=C, is no fault of the directory's. The check runs under - # `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, then `\200` - # to `\237`), so a C1 control is refused in every locale, whatever else + # `[![:print:]]`: a name with a letter the locale does not print, as + # `café` under LC_ALL=C, is no fault of the directory's, and the + # repository and the ref are held to these same bytes (#940, + # `checked_repo` below). The check runs under `LC_ALL=C` and names + # U+0080 to U+009F's two bytes (`\302`, then `\200` to `\237`), so a + # C1 control is refused in every locale, whatever else # the value holds, unless the locale reads those bytes as text (#858, # below); U+2028 and U+2029, which only a UTF-8 locale's `[[:cntrl:]]` # matched, are the next check's (#837). Left to the locale, @@ -794,11 +795,27 @@ find_python() { # sent by the fetch with it. An owner/name keeps it inside the URL, before # `.git`, so only those three forms lost it. # +# A CONTROL CHARACTER IS #760's, AS `install_commands` READS ONE IN THE +# DIRECTORY (#940): under `LC_ALL=C`, in a subshell, `[[:cntrl:]]` or +# U+0080 to U+009F's two bytes (`\302`, then `\200` to `\237`), so the +# answer is the same in every locale. It was `[![:print:]]`, which is the +# locale's: under `C` and POSIX it matches every byte past ASCII, so a URL +# or a path with a letter past ASCII (`/srv/café/openRepoShape.git`) was +# refused there as carrying a control character, which it does not carry, +# where a UTF-8 locale and the package's `--version` took it. The trade is +# the directory's (#867): a byte that is not UTF-8, as a lone `0x9B` or an +# overlong form of a control, passes, in every locale, as it passes the +# package's check, and `--version` prints it: a terminal that reads UTF-8 +# draws it as a character it cannot read, and only one set to an 8-bit +# charset acts on it. `setup.sh`, which is handed the clone URL as +# `$OPENREPOSHAPE_REPO`, holds it to the same rule, and refuses U+2028 and +# U+2029 there too, which this file refuses first by their code points +# (below). +# # A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED # FIRST (#837), in words that name it by its code point and nothing else of -# the value. First, because `[![:print:]]` is the locale's: under `C` it -# matches every byte past ASCII, and under a UTF-8 locale none of these -# characters, so after it the words would change with the locale. +# the value. The control check after it matches none of these characters, +# in any locale, so this is the check that refuses them. # # install.sh's `checked_repo` is stricter, an owner/name and nothing else, # because it builds only github.com URLs from the value. @@ -808,9 +825,9 @@ checked_repo() { misdrawn="$(LC_ALL=C; misdrawn_characters "$REPO")" [ -z "$misdrawn" ] || die "OPENREPOSHAPE_REPO holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. This tool will not put it on a \`git clone\` command line. Set it to a URL or a path." - case "$url$REPO" in - -* | *[![:print:]]*) die "OPENREPOSHAPE_REPO is not a value this tool will put on a \`git clone\` command line: it starts with '-' or carries a control character. Set it to a URL or a path." ;; - esac + if (LC_ALL=C; case "$url$REPO" in -* | *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + die "OPENREPOSHAPE_REPO is not a value this tool will put on a \`git clone\` command line: it starts with '-' or carries a control character. Set it to a URL or a path." + fi } # A CHECKOUT OF THE STANDARD, made the way `setup.sh` self-bootstraps one: @@ -874,9 +891,9 @@ shape_checkout() { # A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF (#837) is a # fault between the length and the control character, named by its code # point and never echoed, as `checked_repo` refuses one: before the -# control check, whose `[![:print:]]` is the locale's, so the words do not -# change with the locale, and before the shape, which echoes the value. -# Those two are `ref_character_fault`'s, below. +# control check, which matches none of these characters in any locale +# (#940), and before the shape, which echoes the value. Those two are +# `ref_character_fault`'s, below. # # THE SHAPE IS GIT'S RULE PART BY PART (#874), as `setup.sh`'s and # `install.sh`'s are: no `..`, no empty part between slashes, no part that @@ -913,6 +930,13 @@ ref_fault() { # as it reads, and an install that does not record the ref takes one that # names no ref at all (workBenches', `record_ref`), so those two are left to # where a ref is used. +# +# THE CONTROL CHARACTER IS `checked_repo`'s RULE, #760's bytes under +# `LC_ALL=C`, with its trade (#940): a byte that is not UTF-8, as a lone +# `0x9B` or an overlong form of a control, passes. A ref with a letter past +# ASCII (`café`) carries no control character in any locale, and is the +# shape's to refuse where the ref is used, in the shape's words, as a UTF-8 +# locale had it. ref_character_fault() { local ref="$1" what="$2" misdrawn misdrawn="$(LC_ALL=C; misdrawn_characters "$ref")" @@ -920,9 +944,9 @@ ref_character_fault() { printf '%s' "$what holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. No branch, tag or commit this tool will pass to \`git checkout\` holds one." return 0 fi - case "$ref" in - *[![:print:]]*) printf '%s' "$what carries a control character, which is not a branch, tag or commit this tool will pass to \`git checkout\`." ;; - esac + if (LC_ALL=C; case "$ref" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + printf '%s' "$what carries a control character, which is not a branch, tag or commit this tool will pass to \`git checkout\`." + fi } checked_ref() { diff --git a/setup.sh b/setup.sh index b125b4bc..7f5e3416 100755 --- a/setup.sh +++ b/setup.sh @@ -203,11 +203,36 @@ fi # forged line on stdout or stderr before this script has said a word of its # own, which is why the refusal below does not echo the value back. The `--` # below guards the argv too; this is what names the fault instead of leaving -# a confusing git error. -case "$SHAPE_REPO" in --*|*[![:print:]]*) die "OPENREPOSHAPE_REPO is not a value this tool will put on a \`git clone\` command line: it starts with '-' or carries a control character. Set it to a URL or a path." ;; -*) ;; # a well-formed value; passed to `git clone` unchanged below. -esac +# a confusing git error. A value that passes goes to `git clone` unchanged. +# +# A CONTROL CHARACTER IS #760's, THE LAUNCHER'S RULE (#940): under +# `LC_ALL=C`, in a subshell, `[[:cntrl:]]` or U+0080 to U+009F's two bytes +# (`\302`, then `\200` to `\237`), so the answer is the same in every +# locale, and the launcher, which hands this script its clone URL as +# `$OPENREPOSHAPE_REPO`, and this script give one answer for one value. It +# was `[![:print:]]`, which under `C` and POSIX matches every byte past +# ASCII, so a URL or a path with a letter past ASCII +# (`/srv/café/openRepoShape.git`) was refused there as carrying a control +# character, where a UTF-8 locale cloned it. The trade is the launcher's +# (#867): a byte that is not UTF-8, as a lone `0x9B` or an overlong form of +# a control, passes, in every locale, and the clone line below prints it: +# a terminal that reads UTF-8 draws it as a character it cannot read, and +# only one set to an 8-bit charset acts on it. +# +# AND U+2028 AND U+2029, BY THEIR THREE BYTES (`\342\200`, then `\250` or +# `\251`), which the launcher leaves to its check of the characters a +# terminal draws as something other than themselves (#837), and which this +# script has no copy of. They are the line and paragraph separators, which +# end a line for a reader that keeps Unicode's line breaks, and both were +# refused here in every locale: by `C`'s `[![:print:]]` with every byte past +# ASCII, and by a UTF-8 locale's, as glibc counts them among its controls. +# So they still are, in the same words, and the launcher and this script +# still refuse them both. The rest of #837's characters (U+202E among them) +# pass here in every locale, as they passed under a UTF-8 locale before; +# the launcher refuses each first where it is the way in. +if (LC_ALL=C; case "$SHAPE_REPO" in -* | *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']* | *$'\342\200'[$'\250'$'\251']*) exit 0 ;; *) exit 1 ;; esac); then + die "OPENREPOSHAPE_REPO is not a value this tool will put on a \`git clone\` command line: it starts with '-' or carries a control character. Set it to a URL or a path." +fi # Mirrors `setup-project.py`'s `checked_ref`, which asks `REF_RE` and then # `why_not_a_ref`: a ref starts with an ASCII letter or digit, holds only ASCII @@ -232,17 +257,20 @@ esac # own guard above: (1) LENGTH first, because an over-long ref would put # 255+ characters of noise on stderr if named; (2) a CONTROL CHARACTER next # - a newline in the value could forge an extra line onto stdout or stderr -# the way the comment above OPENREPOSHAPE_REPO's guard explains. Only once -# a ref has passed both - so it is at most 255 characters and entirely -# printable - does (3) the shape `case` below echo it back, which is by -# then safe to read. +# the way the comment above OPENREPOSHAPE_REPO's guard explains, and the +# control character is that guard's rule, with its trade (a byte that is +# not UTF-8, as a lone `0x9B` or an overlong form of a control, passes) +# and its two separators (#940). Only once a ref has passed both - so it +# is at most 255 characters and holds no control character - does (3) the +# shape `case` below echo it back, which is by then safe to read. A ref +# with a letter past ASCII (`café`) is the shape's to refuse, in its +# words, under `C` as under a UTF-8 locale. if [[ "${#SHAPE_REF}" -gt 255 ]]; then die "--shape-ref is ${#SHAPE_REF} characters long; a branch, tag or commit this tool will pass to \`git checkout\` is at most 255 characters." fi -case "$SHAPE_REF" in -*[![:print:]]*) die "--shape-ref carries a control character, which is not a branch, tag or commit this tool will pass to \`git checkout\`." ;; -*) ;; # printable; the ref-syntax check below still has to pass. -esac +if (LC_ALL=C; case "$SHAPE_REF" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']* | *$'\342\200'[$'\250'$'\251']*) exit 0 ;; *) exit 1 ;; esac); then + die "--shape-ref carries a control character, which is not a branch, tag or commit this tool will pass to \`git checkout\`." +fi case "$SHAPE_REF" in [!A-Za-z0-9]*|*..*|*/.*|*.lock|*.lock/*|*//*|*/|*.|*[!A-Za-z0-9._/-]*) die "--shape-ref is '$SHAPE_REF', which is not a branch, tag or commit this tool will pass to \`git checkout\`: a ref starts with an ASCII letter or digit, uses only ASCII letters, digits, '.', '_', '/', '-', has no '..' (git reads it as a revision range), no empty part between slashes ('//', or a '/' at the end), no part that starts with '.' or ends in '.lock' (the name git gives the lock file it writes beside a ref), and does not end in '.'." ;; *) ;; # a valid ref shape; $SHAPE_REF is safe to echo back and check out. diff --git a/tests/conftest.py b/tests/conftest.py index ae9e0855..8bc458c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2542,3 +2542,108 @@ def assert_the_refusal_states_the_ref_rule(said: str, opening: str) -> None: goes on to `REF_RULE_WORDS`, the whole rule it applied.""" assert opening in said, said assert REF_RULE_WORDS in said, said + + +# --- a control character in a repository or a ref, in the shell copies (#940) + +#: What a check for a control character says of a value: it takes it, and +#: hands it to the check after it, or it refuses it as a control character. +#: A third answer is a code point, `U+XXXX`, which `misdrawn_characters` +#: names where it refuses the value first (#837). +TAKEN = "taken" +CONTROL = "a control character" + +#: How a value with a letter past ASCII ends, the row of the table a person +#: meets: a mirror at `/srv/café/...`, a ref named after one (#940). +A_LETTER_PAST_ASCII = "caf" + chr(0xE9) + +#: THE TABLE the seven checks for a control character in a repository or a +#: ref are held to, one answer each, under `C` and `C.UTF-8` alike (#940): +#: `openRepoShape`'s `checked_repo` and `ref_character_fault`, `setup.sh`'s +#: guards on `$OPENREPOSHAPE_REPO` and `--shape-ref`, and `install.sh`'s +#: `checked_repo`, `checked_version` and option check +#: (`tests/test_openreposhape_command.py`, `tests/test_setup_sh.py` and +#: `tests/test_install_sh.py` read it). Each is #760's rule, the one the +#: launcher holds `$OPENREPOSHAPE_BIN_DIR` to: under `LC_ALL=C`, +#: `[[:cntrl:]]` or U+0080 to U+009F's two bytes. Each row is how a value +#: ends, and what the checks say of it: +#: * `TAKEN`: an e with an acute accent, which `[![:print:]]` refused +#: under `C` as a control character, so a URL, a path or a ref with a +#: letter past ASCII was refused there and taken under `C.UTF-8`; a +#: lone `0x9B` byte, which is not UTF-8 and which a UTF-8 locale's +#: `[![:print:]]` refused too, the trade #867 made for the directory +#: (a terminal that reads UTF-8 draws it as a character it cannot +#: read); a lone `0xE9` byte, Latin-1's e with an acute accent; and an +#: escape in an overlong form (`0xC0 0x9B`), no UTF-8 either and the +#: same trade, as is every overlong form of a control; +#: * `CONTROL`: an escape, U+009B, DEL, a tab, U+0085, the first and the +#: last C1 control, so that an arm that stops one short at either end +#: is caught, a newline at the end, and a lone `0xE9` byte before +#: U+009B, which makes a UTF-8 locale's `[[:cntrl:]]` match byte by +#: byte and miss the control (as `C1_BIN_DIRS` has it); +#: * a code point: U+202E and U+200B, which no control check matches in +#: any locale, and U+2028 and U+2029, which a UTF-8 locale's +#: `[[:cntrl:]]` matched, each refused first by the launcher and +#: `install.sh`, by its code point (#837). `setup.sh` has no such +#: check: it refuses the two separators as control characters, by +#: their bytes, as it did in every locale before (`SEPARATORS`), and +#: reads no other such row. +#: A lone byte is spelled as Python hands one to a child: a surrogate that +#: `surrogateescape` turns back into the byte. A refused value ends in +#: `forged`, or in a newline after it, which no refusal may echo. +CONTROL_RULE = [pytest.param(end, said, id=name) for name, end, said in ( + ("an-e-acute", A_LETTER_PAST_ASCII, TAKEN), + ("a-lone-0x9b-byte", "a" + chr(0xDC9B) + "b", TAKEN), + ("a-lone-0xe9-byte", "caf" + chr(0xDCE9), TAKEN), + ("an-overlong-escape", "a" + chr(0xDCC0) + chr(0xDC9B) + "b", TAKEN), + ("an-escape", "a" + chr(0x1B) + "[31mforged", CONTROL), + ("u+009b", "a" + chr(0x9B) + "31mforged", CONTROL), + ("a-delete", "a" + chr(0x7F) + "forged", CONTROL), + ("a-tab", "a" + chr(0x09) + "forged", CONTROL), + ("u+0085", "a" + chr(0x85) + "forged", CONTROL), + ("u+0080", "a" + chr(0x80) + "forged", CONTROL), + ("u+009f", "a" + chr(0x9F) + "forged", CONTROL), + ("a-newline-at-the-end", "forged" + chr(0x0A), CONTROL), + ("a-lone-0xe9-byte-then-u+009b", + "caf" + chr(0xDCE9) + chr(0x9B) + "31mforged", CONTROL), + ("u+202e", "a" + chr(0x202E) + "forged", "U+202E"), + ("u+200b", "a" + chr(0x200B) + "forged", "U+200B"), + ("u+2028", "a" + chr(0x2028) + "forged", "U+2028"), + ("u+2029", "a" + chr(0x2029) + "forged", "U+2029"), +)] + +#: The line and paragraph separators, which `setup.sh` refuses as control +#: characters where the launcher and `install.sh` name them (#940). +SEPARATORS = ("U+2028", "U+2029") + +#: A repository the launcher's `checked_repo` and `setup.sh`'s guard both +#: refuse for its leading `-`, in the words they refuse a control character +#: in: a URL, which the launcher hands a clone as it is (`shape_clone_url`), +#: so the `-` is still first there. +REPO_WITH_A_LEADING_DASH = "-forged://acme/r" + + +def the_rules_answer(stderr: str, took: bool) -> str: + """What a check answered (#940): `TAKEN` where the run `took` the value + on to the check after it, else `CONTROL` where the refusal says it + carries a control character, else the code points the refusal names + (#837), else the refusal itself, which the table has no answer for.""" + if took: + return TAKEN + words = " ".join(stderr.split()) + named = re.search(r"draws as something other than itself \(([^)]*)\)", + words) + if named: + return named.group(1) + if "carries a control character" in words: + return CONTROL + return words + + +def assert_a_copy_answers_as_the_rule_does(copy: str, end: str, said: str, + answered: str) -> None: + """THE ONE ANSWER (#940): the check `copy` says of a value that ends in + `end` what `CONTROL_RULE` says of it.""" + assert answered == said, ( + f"{copy} answers {answered!r} for a value that ends in {end!r}, and " + f"the table says {said!r}") diff --git a/tests/test_install_sh.py b/tests/test_install_sh.py index d3e6e4a5..d0785c2e 100644 --- a/tests/test_install_sh.py +++ b/tests/test_install_sh.py @@ -48,6 +48,10 @@ from conftest import (BASH, BASH_SKIP, REF_PARTS, REPO, assert_a_copy_answers_as_git_does, assert_the_refusal_states_the_ref_rule, bash_path) +from conftest import (A_LETTER_PAST_ASCII, CONTROL, CONTROL_RULE, + REF_RULE_WORDS, TAKEN, + assert_a_copy_answers_as_the_rule_does, + the_rules_answer) from test_openreposhape_command import (BIN_DIR_SPELLINGS, COMMAND, COMMAND_REFUSAL_ENDS, CYGPATH_STATES, HOLDS_A_COLON, @@ -142,7 +146,7 @@ def curl(): sys.stdout.buffer.write(( "HTTP/2 302 \r\n%s: %s\r\ncontent-length: 0\r\n\r\n" % (NET.get("location_header", "location"), location) - ).encode("utf-8")) + ).encode("utf-8", "surrogateescape")) return 0 served = NET["files"].get(url) if served is None: @@ -248,14 +252,18 @@ def serve(self, ref: str, slug: str = SLUG, self.fake("setup.sh", served)) def run(self, *args: str, piped: bool = False, env: dict | None = None, - script: str | None = None) -> subprocess.CompletedProcess: + script: str | None = None, + errors: str | None = None) -> subprocess.CompletedProcess: """`bash install.sh `, or `… | bash -s -- ` when piped; - `script` pipes that text in place of the installer's own.""" + `script` pipes that text in place of the installer's own. `errors`, + how the output is read where it may hold a byte that is not UTF-8 + (#940).""" (self.root / "net.json").write_text(json.dumps(self.config), encoding="utf-8") gh_config = self.tmp_path / "gh-config" gh_config.mkdir(exist_ok=True) how = {"cwd": str(self.tmp_path), "start_new_session": True, + "errors": errors, "env": command_env(home=self.tmp_path, setup_sh=None, env={ "PATH": str(self.path), "GH_TOKEN": "bogus", "GH_CONFIG_DIR": str(gh_config), @@ -1073,6 +1081,9 @@ def test_the_installer_refuses_as_the_bytes_say_under_c_or_c_utf_8( # location github.com redirects to, holding U+202E (#837). Each arranges # `net` and returns the arguments, the environment, what the refusal says, # and whether the value comes from the network, so that the run asks it. +# The newest release's tag is `gh`'s: the redirect's is taken only when it +# is printable ASCII (#940), so one holding U+202E names no release there, +# as `a_location` does. def a_version(net): return (("v1" + RLO + "forged",), {}, @@ -1090,7 +1101,8 @@ def the_repository(net): def the_newest_tag(net): - net.config["redirects"][latest()] = tag_page("v1" + RLO + "forged") + net.config["redirects"][latest()] = f"https://github.com/{SLUG}/releases" + net.config["gh_latest"] = "v1" + RLO + "forged" return ((), {}, "the newest release's tag " f"{drawn_as_another('U+202E')}.", True) @@ -1125,6 +1137,142 @@ def test_the_installer_refuses_a_value_drawn_as_another_unechoed( assert bool(calls) is asks, calls +# The three values a person hands install.sh, each ending in a row of +# `CONTROL_RULE` (#940). Each takes how the value ends and what the table +# says of it, and returns the arguments, the environment, the whole +# refusal, and the words of the refusal the next check gives, which say the +# control check took the value. + +def the_repository(end: str, said: str): + """`$OPENREPOSHAPE_REPO`, by `checked_repo`: a value it takes is no + owner/name, in those words, in either locale.""" + value = "acme/r" + end + ending = "It must be an owner/name on github.com, such as " \ + "opensoft/openRepoShape." + refusal = { + TAKEN: f"$OPENREPOSHAPE_REPO is '{value}', which is not an " + "owner/name on github.com, such as opensoft/openRepoShape: " + "this installer builds github.com URLs from it.", + CONTROL: "$OPENREPOSHAPE_REPO carries a control character; it must " + "be an owner/name on github.com, such as " + "opensoft/openRepoShape.", + }.get(said, f"$OPENREPOSHAPE_REPO {drawn_as_another(said)}. {ending}") + return (), {"OPENREPOSHAPE_REPO": value}, refusal, "not an owner/name" + + +def the_version(end: str, said: str): + """The version, by `checked_version`: a value it takes is no ref, in + the shape's words.""" + value = "v1" + end + refusal = { + TAKEN: f"the version is '{value}', which is not a release tag, " + f"branch or commit: {REF_RULE_WORDS}", + CONTROL: "the version carries a control character, which is not a " + "release tag, branch or commit.", + }.get(said, f"the version {drawn_as_another(said)}. No release tag, " + "branch or commit holds one.") + return (value,), {}, refusal, REF_RULE_WORDS + + +def the_option(end: str, said: str): + """An option, by the check in `main`: a value it takes is an unknown + option, named.""" + value = "-x" + end + ending = "The one argument is a version, such as v1.0.1; with none, " \ + "the newest release." + refusal = { + TAKEN: f"unknown option '{value}'. {ending}", + CONTROL: f"an option carries a control character. {ending}", + }.get(said, f"an option {drawn_as_another(said)}. {ending}") + return (value,), {}, refusal, "unknown option '" + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("arrange", [the_repository, the_version, the_option], + ids=lambda arrange: arrange.__name__) +@pytest.mark.parametrize("end, said", CONTROL_RULE) +def test_the_installer_answers_each_value_as_the_launchers_rule_does( + net, end, said, arrange, locale): + """`checked_repo`, `checked_version` and the option check held their + values to `[![:print:]]`, which under `C` matches every byte past + ASCII: a value with a letter past ASCII was refused there as carrying a + control character, and under `C.UTF-8` by the check after it, which + names it (#940). Each now asks the launcher's rule, #760's bytes under + `LC_ALL=C`, and gives each row of `CONTROL_RULE` its answer in both + locales: a value it takes goes on to the next check, which refuses it + in the words `C.UTF-8` gave, a lone `0x9B` byte too (#867's trade); a + control character is refused, never echoed, and U+202E, U+200B, U+2028 + and U+2029 by their code points (#837). Exit 2 every time, nothing + placed, no request.""" + args, env, refusal, next_check = arrange(end, said) + result = net.run(*args, env={**env, "LC_ALL": locale}, + errors="surrogateescape") + assert_a_copy_answers_as_the_rule_does( + f"install.sh's check of {arrange.__name__}", + end, said, the_rules_answer(result.stderr, + next_check in result.stderr)) + net.refused(result) + assert net.calls() == [], net.calls() + if said == TAKEN: + assert result.stderr == f"\nREFUSED: {refusal}\n" + return + assert assert_refused_saying(result, refusal) == f"REFUSED: {refusal}" + + +def holds_a_byte_that_is_not_utf_8(end: str) -> bool: + """A test spells such a byte as a surrogate (`surrogateescape`).""" + return any(0xDC80 <= ord(character) <= 0xDCFF for character in end) + + +#: The ends of a location the fault never names (#940): each row +#: `CONTROL_RULE` refuses as a control character, and each it takes only by +#: its trade, a byte that is not UTF-8. A newline ends a header line, so the +#: row that ends in one is no row here. Then two tags the location names, +#: neither taken in any locale: one with a lone `0x9B` byte, which the +#: trade would pass on to the shape refusal that prints it, and one with a +#: letter past ASCII, which a UTF-8 locale's `[:print:]` holds. +UNNAMED_LOCATION_ENDS = [ + *(pytest.param(row.values[0], id=row.id) + for row in CONTROL_RULE + if not row.values[0].endswith("\n") + and (row.values[1] == CONTROL + or holds_a_byte_that_is_not_utf_8(row.values[0]))), + pytest.param("tag/v1" + chr(0xDC9B) + "31mforged", + id="a-tag-with-a-lone-0x9b-byte"), + pytest.param("tag/v1" + A_LETTER_PAST_ASCII + "forged", + id="a-tag-with-an-e-acute")] + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("end", UNNAMED_LOCATION_ENDS) +def test_the_redirect_names_no_location_holding_a_control_character( + tmp_path, end, locale): + """The location `releases/latest` redirects to is the network's, so its + fault names it only where it holds no character `[![:print:]]` matches, + and that class it keeps (#940): it matches a byte that is not UTF-8, as + a lone `0x9B` or an overlong form of a control, in both locales, where + the byte rule a person's own values are held to passes one (#867's + trade), so no such byte from the network is printed. The tag in it is + taken only when it is printable ASCII, under `C`, so none of those + bytes reaches `checked_version` and its shape refusal from there, and + each locale gives a tag one answer. Each end above, under `C` and + `C.UTF-8`: the fault says only that the location names no release, and + nothing of it reaches either stream.""" + net = Net(tmp_path, ("curl",)) + net.config["redirects"][latest()] = ( + f"https://github.com/{SLUG}/releases/" + end) + result = net.run(env={"LC_ALL": locale}, errors="surrogateescape") + net.refused(result) + words = " ".join(result.stderr.split()) + assert f"curl -fsSI {latest()} (it redirects to a location that names " \ + "no release)" in words, ascii(words) + for stream in (result.stdout, result.stderr): + assert "forged" not in stream, ascii(stream) + assert end not in stream, ascii(stream) + + #: A curl that RUNS, in either script: tab-indented, alone or as the value #: of an assignment. Usage text and refusals carry curl lines too, indented #: with spaces, and are printed, never run. diff --git a/tests/test_openreposhape_command.py b/tests/test_openreposhape_command.py index 6d0ee8b1..0ff1148d 100644 --- a/tests/test_openreposhape_command.py +++ b/tests/test_openreposhape_command.py @@ -39,6 +39,10 @@ assert_a_copy_answers_as_git_does, assert_the_refusal_states_the_ref_rule, bash_path, git, native_path) +from conftest import (A_LETTER_PAST_ASCII, CONTROL, CONTROL_RULE, + REPO_WITH_A_LEADING_DASH, TAKEN, + assert_a_copy_answers_as_the_rule_does, + the_rules_answer) COMMAND = REPO / "openRepoShape" SETUP = REPO / "setup.sh" @@ -121,7 +125,8 @@ def command_env(home: Path | None = None, env: dict | None = None, def run_cmd(*args: str, home: Path | None = None, env: dict | None = None, setup_sh: Path | None = SETUP, command: Path = COMMAND, - cwd: Path | None = None) -> subprocess.CompletedProcess: + cwd: Path | None = None, + errors: str | None = None) -> subprocess.CompletedProcess: """The command, run from its file, with that environment. `input=""` means stdin is a pipe rather than a terminal, which is what the @@ -133,11 +138,14 @@ def run_cmd(*args: str, home: Path | None = None, env: dict | None = None, `command` is this checkout's file unless a test names an INSTALLED copy, which is how the record `--install` leaves beside one is read (#564). `cwd`, where a relative path the command is handed must land. + `errors`, how its output is read where it may print a byte that is not + UTF-8: `surrogateescape` hands such a byte back as the surrogate a test + spelled it with (#940). """ return subprocess.run([BASH, bash_path(command), *args], capture_output=True, text=True, check=False, input="", env=command_env(home, env, setup_sh), - start_new_session=True, + start_new_session=True, errors=errors, cwd=None if cwd is None else str(cwd)) @@ -889,6 +897,47 @@ def test_install_sh_carries_the_launchers_misdrawn_check_as_it_is(): assert misdrawn_check(REPO / "install.sh") == misdrawn_check(COMMAND) +#: A CHECK FOR A CONTROL CHARACTER IN A REPOSITORY OR A REF, as the +#: launcher, `setup.sh` and `install.sh` write each (#940): a one-line `if` +#: whose subshell asks `[[:cntrl:]]`. The directory's (#760, #858) span +#: lines, so none is such a line. +CONTROL_CHECK_LINE = re.compile(r"^\s*if \((.*\[\[:cntrl:\]\].*)\); then$", + re.M) + +#: #760's rule, as each such line asks it of the value, written `$value`. +CONTROL_CHECK = ('LC_ALL=C; case "$value" in *[[:cntrl:]]* | ' + r"*$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac") + +#: `setup.sh`'s arm for U+2028 and U+2029, by their bytes (#940), in place +#: of the launcher's `misdrawn_characters`, which it has no copy of. +SEPARATORS_ARM = r" | *$'\342\200'[$'\250'$'\251']*" + + +def control_checks(path: Path) -> list[str]: + """Each such line in the file at `path`, apart from what may differ + between them: the value it asks of, the leading `-` arm of the two + guards that hand a repository to `git clone`, and `SEPARATORS_ARM`.""" + return [re.sub(r'case "[^"]*" in (-\* \| )?', 'case "$value" in ', line) + .replace(SEPARATORS_ARM, "") + for line in CONTROL_CHECK_LINE.findall( + path.read_text(encoding="utf-8"))] + + +@pytest.mark.parametrize("name, count", [("openRepoShape", 2), + ("setup.sh", 2), ("install.sh", 3)]) +def test_the_seven_control_checks_carry_one_rule_as_it_is(name, count): + """The launcher's `checked_repo` and `ref_character_fault`, `setup.sh`'s + two guards, and `install.sh`'s `checked_repo`, `checked_version` and + option check ask #760's rule, one text in all seven (#940): under + `LC_ALL=C`, in a subshell, `[[:cntrl:]]` or U+0080 to U+009F's two + bytes. No locale this suite can count on tells a check without + `LC_ALL=C` from one with it: a Latin-1 locale's `[[:cntrl:]]` holds + `0x80` to `0x9F`, so a lone `0x9B` byte would be refused there, where + `C` and `C.UTF-8` take it. So the text is held here, as + `misdrawn_characters`'s is above.""" + assert control_checks(REPO / name) == [CONTROL_CHECK] * count + + #: Every code point whose UTF-8 starts with the bytes one of `MISDRAWN` #: starts with: U+0080 to U+00BF (`0xC2`), U+0600 to U+063F (`0xD8`), #: U+2000 to U+20FF (`0xE2 0x80` to `0xE2 0x83`), U+FE00 to U+FFFF @@ -2288,18 +2337,20 @@ def test_install_from_stdin_fetches_itself_into_a_live_workdir(offline_github, ] -def cloned_for(tmp_path, repo: str, caller: str) -> list: +def cloned_for(tmp_path, repo: str, caller: str, + env: dict | None = None) -> list: """What the clone `caller` makes is handed for `$OPENREPOSHAPE_REPO`: the value setup.sh is given to clone, through a setup.sh that prints it, or the URL `--doctor`'s own `git clone` names, through a `git` that prints it and fails, from a copy of the command with no checkout of the - standard beside it. Nothing is cloned and nothing is fetched.""" + standard beside it. Nothing is cloned and nothing is fetched. `env` is + put over the rest, such as a locale (#940).""" if caller == "setup.sh": stub = tmp_path / "setup.sh" write_script(stub, '#!/usr/bin/env bash\n' 'printf "CLONES %s\\n" "$OPENREPOSHAPE_REPO"\n') result = run_cmd("--preflight", setup_sh=stub, - env={"OPENREPOSHAPE_REPO": repo}) + env={"OPENREPOSHAPE_REPO": repo, **(env or {})}) assert result.returncode == 0, result.stdout + result.stderr else: fake = tmp_path / "fake-path" @@ -2311,7 +2362,7 @@ def cloned_for(tmp_path, repo: str, caller: str) -> list: copy.write_bytes(COMMAND.read_bytes()) result = run_cmd("--doctor", str(tmp_path), command=copy, env={ "OPENREPOSHAPE_REPO": repo, "TMPDIR": str(tmp_path), - "PATH": f"{fake}{os.pathsep}{os.environ['PATH']}"}) + "PATH": f"{fake}{os.pathsep}{os.environ['PATH']}", **(env or {})}) assert result.returncode == 2, result.stdout + result.stderr assert "could not clone" in result.stderr return re.findall(r"^CLONES (.*)$", result.stdout, re.M) @@ -2370,28 +2421,32 @@ def ref_drawn_as_another(named: str) -> str: "or commit this tool will pass to `git checkout` holds one.") -#: Each variable holding a C0 control (an escape), a C1 control (U+009B), -#: U+202E and U+200B, each before a `forged` tail, with the whole refusal -#: (#866); a repository whose clone URL starts with `-`, the third thing -#: `checked_repo` refuses; and a path, a URL and an scp address that end in -#: a newline, which `$(shape_clone_url)` drops, so the check looks at the -#: value as typed too (rv866's SHOULD-1). +def refused_as(variable: str, said: str) -> str: + """The whole refusal of a value in `variable` that `CONTROL_RULE` says + `said` of, a control character or a code point: `checked_repo`'s, or + `ref_character_fault`'s for `$OPENREPOSHAPE_REF` (#940).""" + if variable == "OPENREPOSHAPE_REPO": + return (REPO_NOT_FOR_A_CLONE if said == CONTROL + else repo_drawn_as_another(said)) + return (REF_CARRIES_A_CONTROL_CHARACTER if said == CONTROL + else ref_drawn_as_another(said)) + + +#: Each variable ending in each value `CONTROL_RULE` refuses, with the whole +#: refusal (#866, #940): an escape, U+009B, U+202E and U+200B among them, +#: each before a `forged` tail, and a newline at the end; a repository +#: whose clone URL starts with `-`, the third thing `checked_repo` refuses; +#: and a path, a URL and an scp address that end in a newline, which +#: `$(shape_clone_url)` drops, so the check looks at the value as typed too +#: (rv866's SHOULD-1). UNPRINTABLE_VALUES = [ - *(pytest.param("OPENREPOSHAPE_REPO", "acme/r" + character + "forged", - said, id=f"repo-{name}") - for character, said, name in ( - (ESCAPE + "[31m", REPO_NOT_FOR_A_CLONE, "an-escape"), - (C1_CSI + "31m", REPO_NOT_FOR_A_CLONE, "a-c1-csi"), - (RLO, repo_drawn_as_another("U+202E"), "u+202e"), - (chr(0x200B), repo_drawn_as_another("U+200B"), "u+200b"))), - *(pytest.param("OPENREPOSHAPE_REF", "v1" + character + "forged", - said, id=f"ref-{name}") - for character, said, name in ( - (ESCAPE + "[31m", REF_CARRIES_A_CONTROL_CHARACTER, "an-escape"), - (C1_CSI + "31m", REF_CARRIES_A_CONTROL_CHARACTER, "a-c1-csi"), - (RLO, ref_drawn_as_another("U+202E"), "u+202e"), - (chr(0x200B), ref_drawn_as_another("U+200B"), "u+200b"))), - pytest.param("OPENREPOSHAPE_REPO", "-forged://acme/r", + *(pytest.param(variable, start + row.values[0], + refused_as(variable, row.values[1]), + id=f"{name}-{row.id}") + for variable, start, name in (("OPENREPOSHAPE_REPO", "acme/r", "repo"), + ("OPENREPOSHAPE_REF", "v1", "ref")) + for row in CONTROL_RULE if row.values[1] != TAKEN), + pytest.param("OPENREPOSHAPE_REPO", REPO_WITH_A_LEADING_DASH, REPO_NOT_FOR_A_CLONE, id="repo-a-leading-dash"), *(pytest.param("OPENREPOSHAPE_REPO", value, REPO_NOT_FOR_A_CLONE, id=f"repo-a-trailing-newline-{form}") @@ -2531,20 +2586,161 @@ def test_version_prints_every_value_the_check_passes_as_typed( @NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) @pytest.mark.parametrize("name", NOT_CONTROLS) -def test_version_prints_a_path_a_utf_8_locale_prints(name): +def test_version_prints_a_path_a_utf_8_locale_prints(name, locale): """A path holding a letter past ASCII, U+200C, U+200D or a tag is no - fault under `C.UTF-8`, whose `[![:print:]]` passes each of - `NOT_CONTROLS` (#866). Under `C` that class matches every byte past - ASCII, so there the path is refused, as setup.sh's clone and the - doctor's refuse it.""" + fault in either locale (#866, #940). The control check is #760's byte + rule, under `LC_ALL=C`, which passes each of `NOT_CONTROLS`, as the + directory's does. It was `[![:print:]]`, which `C.UTF-8` passed each + of them by, and which under `C` matches every byte past ASCII, so there + the path was refused as carrying a control character.""" path = f"/srv/{name}/openRepoShape.git" result = run_cmd("--version", env={"OPENREPOSHAPE_REPO": path, - "LC_ALL": "C.UTF-8"}) + "LC_ALL": locale}) assert result.returncode == 0, result.stdout + result.stderr assert result.stdout == f"openRepoShape ({path} @ main)\n" +#: Where a test puts how a value ends (`CONTROL_RULE`): after the start of +#: a path, a URL and an scp address, three forms of repository a clone is +#: handed as they are (`CLONE_URLS`), and after the start of a ref (#940). +VALUE_FORMS = [ + pytest.param("OPENREPOSHAPE_REPO", "/srv/", id="repo-path"), + pytest.param("OPENREPOSHAPE_REPO", "https://example.invalid/", + id="repo-url"), + pytest.param("OPENREPOSHAPE_REPO", "git@example.invalid:", id="repo-scp"), + pytest.param("OPENREPOSHAPE_REF", "v1", id="ref"), +] + + +def version_line(variable: str, value: str) -> str: + """What `--version` prints with `variable` set to `value`.""" + if variable == "OPENREPOSHAPE_REPO": + return f"openRepoShape ({value} @ main)\n" + return f"openRepoShape (opensoft/openRepoShape @ {value})\n" + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("variable, start", VALUE_FORMS) +@pytest.mark.parametrize("end, said", CONTROL_RULE) +def test_version_answers_a_repository_or_ref_as_the_control_rule_does( + end, said, variable, start, locale): + """`checked_repo` and `ref_character_fault` held the repository and the + ref to `[![:print:]]`, which under `C` matches every byte past ASCII: + `--version` refused `/srv/café` and a ref `v1café` as + carrying a control character, and printed both under `C.UTF-8` (#940). + Each now asks #760's byte rule, under `LC_ALL=C`, and gives each row of + `CONTROL_RULE` its answer in both locales: a value it takes is printed + as typed, exit 0, a lone `0x9B` byte as well, which is the trade + (#867); a control character is refused, exit 2, in the words it had, + and U+202E, U+200B, U+2028 and U+2029 by their code points (#837), + none of it echoed.""" + value = start + end + result = run_cmd("--version", env={variable: value, "LC_ALL": locale}, + errors="surrogateescape") + assert_a_copy_answers_as_the_rule_does( + f"openRepoShape's check of ${variable}", end, said, + the_rules_answer(result.stderr, result.returncode == 0)) + if said == TAKEN: + assert result.stdout == version_line(variable, value) + return + refusal = refused_as(variable, said) + assert assert_refused_saying(result, refusal) == f"REFUSED: {refusal}" + + +# The runs a repository or a ref with a letter past ASCII went through under +# `C.UTF-8`, and from #872 to #940 did not under `C` (#940). Each takes +# `tmp_path`, the variable, its value and the locale, and holds the run to +# what it does under `C.UTF-8`. + +def scaffolds(tmp_path, variable, value, locale): + """A scaffold run asks the fake `gh` for `setup.sh` with the value, + once, and runs what it is served.""" + result, calls = run_against_a_fake_github( + tmp_path, ("Atlas", "--org", "acme"), False, + {variable: value, "LC_ALL": locale}) + assert result.returncode == 0, result.stdout + result.stderr + assert STUB_MARKER in result.stdout + [call] = calls + assert call.startswith("gh api repos/"), call + assert value in call, call + + +def installs(tmp_path, variable, value, locale): + """`--install` from a file places this command.""" + result = run_cmd("--install", home=tmp_path, cwd=tmp_path, env={ + variable: value, "LC_ALL": locale, + "OPENREPOSHAPE_BIN_DIR": bash_path(tmp_path / "bin")}) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "bin" / "openRepoShape").read_bytes() \ + == COMMAND.read_bytes() + + +def clones_for_the_doctor(tmp_path, _variable, value, locale): + """`--doctor`, with no checkout beside the command, hands a fake `git` + the repository to clone.""" + assert cloned_for(tmp_path, value, "doctor", {"LC_ALL": locale}) \ + == [value] + + +def hands_setup_sh_the_repository(tmp_path, _variable, value, locale): + """`--preflight` hands `setup.sh` the repository to clone.""" + assert cloned_for(tmp_path, value, "setup.sh", {"LC_ALL": locale}) \ + == [value] + + +def runs_the_doctor_beside_a_checkout(tmp_path, variable, value, locale): + """`--doctor` beside a checkout, which a `shape-doctor.py` that says it + ran stands for, runs it: there the ref names nothing to check out.""" + copy = tmp_path / "beside" / "openRepoShape" + copy.parent.mkdir() + copy.write_bytes(COMMAND.read_bytes()) + write_script(copy.parent / "shape-doctor.py", "print('DOCTOR RAN')\n") + result = run_cmd("--doctor", str(tmp_path), command=copy, + env={variable: value, "LC_ALL": locale}) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout == "DOCTOR RAN\n" + + +def hands_setup_sh_the_ref(tmp_path, variable, value, locale): + """`--preflight` hands `setup.sh` the ref, as `--shape-ref`.""" + stub = tmp_path / "setup.sh" + write_script(stub, STUB_SETUP_SH) + result = run_cmd("--preflight", setup_sh=stub, + env={variable: value, "LC_ALL": locale}) + assert result.returncode == 0, result.stdout + result.stderr + match = re.search(rf"{STUB_MARKER} in .+ args: (.*)", result.stdout) + assert match, result.stdout + result.stderr + assert match.group(1).split() == ["--shape-ref", value, "--preflight"] + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("variable, start, run", [ + *(pytest.param(*form.values, run, id=f"{form.id}-{run.__name__}") + for form in VALUE_FORMS if form.values[0] == "OPENREPOSHAPE_REPO" + for run in (scaffolds, installs, clones_for_the_doctor, + hands_setup_sh_the_repository)), + *(pytest.param(*form.values, run, id=f"{form.id}-{run.__name__}") + for form in VALUE_FORMS if form.values[0] == "OPENREPOSHAPE_REF" + for run in (scaffolds, installs, runs_the_doctor_beside_a_checkout, + hands_setup_sh_the_ref)), +]) +def test_a_letter_past_ascii_goes_through_every_run_in_either_locale( + tmp_path, variable, start, run, locale): + """#872 checked the repository and the ref before every run but + `--help`, with `[![:print:]]`, so under `C` a URL, a path or an scp + address with a letter past ASCII, or a ref with one, stopped every run + that `C.UTF-8` let through, most of which v1.1.0 had run (#940). Now + each run does what it does under `C.UTF-8`, in both locales: a scaffold run + fetches `setup.sh`, `--install` places the command, `--doctor` clones + the repository or runs beside a checkout, and `--preflight` hands + `setup.sh` the repository or the ref.""" + run(tmp_path, variable, start + A_LETTER_PAST_ASCII, locale) + + # --- the ref `--install` records, and the installed command runs (#564) ----- #: A ref for an install to record. Nothing fetches it: every install below diff --git a/tests/test_setup_sh.py b/tests/test_setup_sh.py index 40ba53cc..59ab50a7 100644 --- a/tests/test_setup_sh.py +++ b/tests/test_setup_sh.py @@ -58,8 +58,16 @@ from conftest import (REF_PARTS, assert_a_copy_answers_as_git_does, assert_the_refusal_states_the_ref_rule) +from conftest import (CONTROL, CONTROL_RULE, REF_RULE_WORDS, + REPO_WITH_A_LEADING_DASH, SEPARATORS, + assert_a_copy_answers_as_the_rule_does, + the_rules_answer) + from test_openreposhape_command import fake_github +from test_openreposhape_command import (LOCALES, REPO_NOT_FOR_A_CLONE, + assert_refused_saying) + SETUP = REPO / "setup.sh" DEGRADE_LINE = "authority is not wallet-carried in this org" @@ -124,8 +132,8 @@ def make_outside_checkout(tmp_path: Path) -> Path: def run_bare_setup(script: Path, *args: str, cwd: Path, - extra_env: dict | None = None, - stdin: str = "") -> subprocess.CompletedProcess: + extra_env: dict | None = None, stdin: str = "", + errors: str | None = None) -> subprocess.CompletedProcess: """Like `run_setup`, but against an arbitrary script path and cwd — what the self-bootstrap tests need since `run_setup` always drives THIS checkout's own `setup.sh` from THIS checkout's own directory. @@ -133,7 +141,8 @@ def run_bare_setup(script: Path, *args: str, cwd: Path, `stdin` as `run_setup` has it: a pipe, and no controlling terminal either (#492), never the terminal pytest was started from, so a run that reaches the one `yes` refuses instead of waiting for somebody to type - it.""" + it. `errors`, how the output is read where it may hold a byte that is + not UTF-8 (#940).""" env = dict(os.environ) env.setdefault("GIT_AUTHOR_NAME", "openRepoShape tests") env.setdefault("GIT_AUTHOR_EMAIL", "tests@openreposhape.invalid") @@ -142,7 +151,7 @@ def run_bare_setup(script: Path, *args: str, cwd: Path, env.update(extra_env or {}) return subprocess.run(["bash", str(script), *args], capture_output=True, text=True, check=False, input=stdin, cwd=str(cwd), - env=env, start_new_session=True) + env=env, start_new_session=True, errors=errors) def shim_default_repo() -> str: @@ -922,6 +931,29 @@ def test_a_shape_ref_that_is_not_a_ref_is_refused_by_the_shim(tmp_path, ref): assert "--shape-ref" in result.stderr +def run_shim_under(tmp_path: Path, repo: str, *args: str, locale: str = ""): + """The shim from a directory that is no checkout, run with `args`, + `$OPENREPOSHAPE_REPO` set to `repo`, `LC_ALL` to `locale` where one is + given, a `$TMPDIR` of its own, and a `gh` and a `curl` that log and + fail first on `$PATH`. Neither is asked, and the run leaves no shape + checkout. Read with `surrogateescape`: a value the guard takes may be + printed as typed, a byte that is not UTF-8 and all (#940).""" + outside = make_outside_checkout(tmp_path) + temporary = tmp_path / "tmp" + temporary.mkdir() + network = tmp_path / "network.log" + chosen = {"LC_ALL": locale} if locale else {} + result = run_bare_setup(outside / "setup.sh", *args, + cwd=outside, errors="surrogateescape", + extra_env={"OPENREPOSHAPE_REPO": repo, **chosen, + "TMPDIR": str(temporary), + **fake_github(tmp_path, (), + log=network)}) + assert not network.exists(), network.read_text(encoding="utf-8") + assert list(temporary.iterdir()) == [], "a shape checkout was left behind" + return result + + @pytest.mark.parametrize("name, takes", REF_PARTS) def test_the_shim_takes_a_shape_ref_as_git_does_before_any_clone( tmp_path, name, takes): @@ -939,24 +971,12 @@ def test_the_shim_takes_a_shape_ref_as_git_does_before_any_clone( applied, and leaves nothing in `$TMPDIR`. A `gh` and a `curl` that log and fail stand first on `$PATH`, and neither is called. """ - outside = make_outside_checkout(tmp_path) - temporary = tmp_path / "tmp" - temporary.mkdir() - network = tmp_path / "network.log" - result = run_bare_setup(outside / "setup.sh", + result = run_shim_under(tmp_path, str(tmp_path / "no-standard-here"), "--org", "demoorg", "--project", "Sample", "--yes", "--shape-ref", name, - "--local-remote-dir", str(tmp_path / "remotes"), - cwd=outside, - extra_env={"OPENREPOSHAPE_REPO": - str(tmp_path / "no-standard-here"), - "TMPDIR": str(temporary), - **fake_github(tmp_path, (), - log=network)}) - assert not network.exists(), network.read_text(encoding="utf-8") + "--local-remote-dir", str(tmp_path / "remotes")) took = "(0) self-bootstrap" in result.stdout assert_a_copy_answers_as_git_does("setup.sh", name, takes, took) - assert list(temporary.iterdir()) == [], "a shape checkout was left behind" if took: return assert result.returncode == 2, result.stderr + result.stdout @@ -1020,6 +1040,93 @@ def test_a_shape_ref_with_a_control_character_is_refused_without_echo(tmp_path): "the newline let a forged line reach stderr:\n" + result.stderr) +# --- the launcher's rule for a control character (#940) ---------------------- + +#: `CONTROL_RULE` as this script answers it (#940): the line and paragraph +#: separators as control characters, by their bytes (`SEPARATORS`), and no +#: other row a code point answers, as this script asks nothing else of the +#: characters a terminal draws as something other than themselves (#837). +SHIM_CONTROL_RULE = [ + pytest.param(row.values[0], + CONTROL if row.values[1] in SEPARATORS else row.values[1], + id=row.id) + for row in CONTROL_RULE + if not row.values[1].startswith("U+") or row.values[1] in SEPARATORS] + +#: How `$OPENREPOSHAPE_REPO` ends (`SHIM_CONTROL_RULE`), after a path +#: nothing is at (`{d}`), and what the guard says of it; and a value that +#: starts with `-`, which it refuses in the words of a control character. +SHIM_REPOSITORIES = [ + *(pytest.param("{d}/" + row.values[0], row.values[1], id=row.id) + for row in SHIM_CONTROL_RULE), + pytest.param(REPO_WITH_A_LEADING_DASH, CONTROL, id="a-leading-dash"), +] + +#: The guard's refusal of a `--shape-ref` holding a control character. +SHAPE_REF_CARRIES_A_CONTROL_CHARACTER = ( + "--shape-ref carries a control character, which is not a branch, tag or " + "commit this tool will pass to `git checkout`.") + + +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("typed, said", SHIM_REPOSITORIES) +def test_the_repository_guard_answers_as_the_launchers_rule_does( + tmp_path, typed, said, locale): + """The guard held `$OPENREPOSHAPE_REPO` to `[![:print:]]`, which under + `C` matches every byte past ASCII: a URL or a path with a letter past + ASCII was refused as carrying a control character, where `C.UTF-8` + cloned it, and the launcher hands this script the URL it fetched from + (#940). It now asks the launcher's rule, #760's bytes under + `LC_ALL=C`, and gives each row of `SHIM_CONTROL_RULE` its answer in + both locales: a value it takes goes to `git clone`, which finds nothing + there, a lone `0x9B` byte too (#867's trade); a control character, the + line and paragraph separators, which it refused in both locales before, + and a leading `-` are refused, exit 2, before any clone, none of it + echoed.""" + repo = typed.format(d=tmp_path / "no-standard-here") + result = run_shim_under(tmp_path, repo, "--preflight", locale=locale) + took = "(0) self-bootstrap" in result.stdout + assert_a_copy_answers_as_the_rule_does( + "setup.sh's guard on $OPENREPOSHAPE_REPO", repo, said, + the_rules_answer(result.stderr, took)) + if took: + assert f" git clone --quiet -- {repo} " in result.stdout + return + assert assert_refused_saying(result, REPO_NOT_FOR_A_CLONE) \ + == f"REFUSED: {REPO_NOT_FOR_A_CLONE}" + + +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("end, said", SHIM_CONTROL_RULE) +def test_the_shape_ref_guard_answers_as_the_launchers_rule_does( + tmp_path, end, said, locale): + """The guard held `--shape-ref` to `[![:print:]]` as well, so under `C` + a ref with a letter past ASCII was refused as carrying a control + character, and under `C.UTF-8` by the shape `case`, which names it + (#940). It now asks the launcher's rule: a ref it takes goes on to the + shape, which refuses each of these, exit 2, naming it, in the words + `C.UTF-8` gave, a lone `0x9B` byte included (#867's trade); a control + character, and either separator, is refused, exit 2, and not echoed. + No clone either way.""" + ref = "v1" + end + result = run_shim_under(tmp_path, str(tmp_path / "no-standard-here"), + "--preflight", "--shape-ref", ref, locale=locale) + took = REF_RULE_WORDS in result.stderr + assert_a_copy_answers_as_the_rule_does( + "setup.sh's guard on --shape-ref", end, said, + the_rules_answer(result.stderr, took)) + assert "git clone" not in result.stdout + if took: + assert result.returncode == 2, result.stderr + result.stdout + assert_the_refusal_states_the_ref_rule( + result.stderr, + f"REFUSED: --shape-ref is '{ref}', which is not a branch, tag or " + "commit this tool will pass to `git checkout`: ") + return + refusal = SHAPE_REF_CARRIES_A_CONTROL_CHARACTER + assert assert_refused_saying(result, refusal) == f"REFUSED: {refusal}" + + def test_the_shim_refuses_before_cloning_when_git_is_missing(tmp_path): """A PATH holding only `dirname` (needed to compute `$SCRIPT_DIR` before the probe can run at all) and a `python3` symlink pointing at THIS