diff --git a/install.sh b/install.sh index 09f3170f..1428e0b8 100755 --- a/install.sh +++ b/install.sh @@ -45,7 +45,8 @@ # command already there as a symbolic link (pipx's, #563), a # `$OPENREPOSHAPE_BIN_DIR` spelled for Windows where no `cygpath` can # spell it the way bash does (#667), or a directory holding a control -# character or a `:`, which no line about `$PATH` can name (#760). +# character or a `:`, which no line about `$PATH` can name (#760), or +# a character a terminal draws as something other than itself (#837). # git-filter-repo never changes it: only adopting needs it, and # `adopt-project.py` refuses with its own install command where it is missing. # What it changes is the summary (#595): `adoption:` says whether it is there @@ -122,8 +123,16 @@ USAGE # THE REPOSITORY, AS AN OWNER/NAME, because every URL below is built from it. # The launcher also takes a URL or a path in this variable, for the clone # setup.sh makes; there is no release at either, so this refuses them by -# name. A control character is refused before anything is echoed. +# name. A control character is refused before anything is echoed, and +# before it a character a terminal draws as something other than itself +# (#837), named by its code point: first, because `[![:print:]]` is the +# locale's, matching every byte past ASCII under `C` and none of those +# characters under a UTF-8 locale, where the refusal below would echo them. checked_repo() { + local misdrawn + misdrawn="$(LC_ALL=C; misdrawn_characters "$REPO")" + [ -z "$misdrawn" ] || + die "\$OPENREPOSHAPE_REPO holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. It must be an owner/name on github.com, such as opensoft/openRepoShape." case "$REPO" in *[![:print:]]*) die "\$OPENREPOSHAPE_REPO carries a control character; it must be an owner/name on github.com, such as opensoft/openRepoShape." ;; esac @@ -139,14 +148,19 @@ checked_repo() { # THE LAUNCHER'S `checked_ref`, TRANSCRIBED: the version reaches a URL here # and `git checkout` once the command runs, so it is held to the same three # checks in the same order (length, then a control character, then the -# shape, which is the only one that echoes it), plus the empty value. Not -# shared, because this file arrives alone down a pipe. +# shape, which is the only one that echoes it), plus the empty value, and, +# between the length and the control character, a character a terminal +# draws as something other than itself, as the launcher's `ref_fault` has +# it (#837). Not shared, because this file arrives alone down a pipe. checked_version() { - local ref="$1" what="$2" + local ref="$1" what="$2" misdrawn [ -n "$ref" ] || die "$what is empty. Name a release, such as v1.0.1, or none for the newest." if [ "${#ref}" -gt 255 ]; then die "$what is ${#ref} characters long; a release tag, branch or commit is at most 255 characters." fi + misdrawn="$(LC_ALL=C; misdrawn_characters "$ref")" + [ -z "$misdrawn" ] || + die "$what holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. No release tag, branch or commit holds one." case "$ref" in *[![:print:]]*) die "$what carries a control character, which is not a release tag, branch or commit." ;; esac @@ -195,10 +209,16 @@ latest_from_redirect() { return 0 ;; esac - case "$value" in - *[![:print:]]*) FAULT="it redirects to a location that names no release" ;; - *) FAULT="it redirects to $value, which names no release" ;; - esac + # The location is named only when it is text a terminal draws as + # it is: no control character, and no character it draws as + # something other than itself (#837). + FAULT="it redirects to a location that names no release" + if [ -z "$(LC_ALL=C; misdrawn_characters "$value")" ]; then + case "$value" in + *[![:print:]]*) ;; + *) FAULT="it redirects to $value, which names no release" ;; + esac + fi return 1 ;; esac @@ -336,8 +356,52 @@ filter_repo_step() { adoption_not_ready "it is not installed" "$how $found" } +# THE CHARACTERS A TERMINAL DRAWS AS SOMETHING OTHER THAN THEMSELVES, AND +# WHICH OF THEM "$1" HOLDS (#837): the launcher's `MISDRAWN_CHARACTERS` and +# `misdrawn_characters`, copied as they are, for the reason the control +# check in `main` is copied: this file arrives alone down a pipe. The +# launcher says why these characters and no others, and which forms of +# UTF-8 the function builds (two, three and four bytes), and +# `tests/test_openreposhape_command.py` holds the two copies to one text. +# `main` asks it of the directory, of an option, `checked_repo` of the +# repository, `checked_version` of a version and `latest_from_redirect` +# of the location it reads, each before any line prints the value. +MISDRAWN_CHARACTERS=(00AD 061C 200B 200E-200F 2028-202E 2060-206F FEFF FFF9-FFFB 1D173-1D17A E0001) + +misdrawn_characters() { + local value="$1" entry code last bytes name found="" + for entry in "${MISDRAWN_CHARACTERS[@]}"; do + code=$((0x${entry%-*})) + last=$((0x${entry#*-})) + while [ "$code" -le "$last" ]; do + if [ "$code" -lt 2048 ]; then + printf -v bytes '\\0%o\\0%o' $((0xC0 | (code >> 6))) \ + $((0x80 | (code & 63))) + elif [ "$code" -lt 65536 ]; then + printf -v bytes '\\0%o\\0%o\\0%o' $((0xE0 | (code >> 12))) \ + $((0x80 | ((code >> 6) & 63))) $((0x80 | (code & 63))) + else + printf -v bytes '\\0%o\\0%o\\0%o\\0%o' \ + $((0xF0 | (code >> 18))) $((0x80 | ((code >> 12) & 63))) \ + $((0x80 | ((code >> 6) & 63))) $((0x80 | (code & 63))) + fi + printf -v bytes '%b' "$bytes" + case "$value" in + *"$bytes"*) + printf -v name 'U+%04X' "$code" + found="${found:+$found, }$name" + ;; + *) ;; + esac + code=$((code + 1)) + done + done + printf '%s' "$found" +} + main() { local version="" asked launcher_url bin_dir spelled target shown status=0 pinned=1 + local misdrawn local machine="this machine is ready" # NOTHING A CHILD READS IS THIS SCRIPT. Piped, stdin is the pipe the @@ -350,6 +414,9 @@ main() { -h | --help) usage; exit 0 ;; --) shift; continue ;; -*) + misdrawn="$(LC_ALL=C; misdrawn_characters "$1")" + [ -z "$misdrawn" ] || + die "an option holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. The one argument is a version, such as v1.0.1; with none, the newest release." case "$1" in *[![:print:]]*) die "an option carries a control character. The one argument is a version, such as v1.0.1; with none, the newest release." ;; esac @@ -379,7 +446,8 @@ main() { # print (`café` under LC_ALL=C) is no fault of the directory's. The check # runs under `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, # then `\200` to `\237`), so a C1 control is refused in every locale, - # whatever else the value holds; U+2028 and U+2029 are refused in none. + # whatever else the value holds; U+2028 and U+2029, which only a UTF-8 + # locale's `[[:cntrl:]]` matched, are the next check's (#837). # Left to the locale, `[[:cntrl:]]` matches a C1 control only in a value # that is valid UTF-8: one byte that is not (a lone 0xE9) makes bash # match byte by byte, and then U+009B's two bytes are not controls, as @@ -389,6 +457,14 @@ main() { if (LC_ALL=C; case "$bin_dir" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then die "the directory to install into holds a control character, such as a newline, a carriage return, a tab or an escape, and no line about \$PATH can print such a name as it is: a terminal acts on it. Nothing was installed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this line again." fi + # A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED + # NEXT (#837), as the command refuses it, and before any request: the + # name would read as another directory's in every line below. The + # refusal names each one by its code point, because nobody can see it, + # and says nothing else of the value. + misdrawn="$(LC_ALL=C; misdrawn_characters "$bin_dir")" + [ -z "$misdrawn" ] || + die "the directory to install into holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it, so no line about \$PATH can print the name as it reads. Nothing was installed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this line again." # A WINDOWS SPELLING IS SPELLED THE WAY BASH SPELLS IT, THE COMMAND'S OWN # RULE (#667, after #649), before it is named in a line or looked at: a # value typed `D:\tools\bin` was echoed as typed, in the `install:`, diff --git a/openRepoShape b/openRepoShape index b5cd1dd7..a94e5192 100755 --- a/openRepoShape +++ b/openRepoShape @@ -413,8 +413,9 @@ record_ref() { # README says `export PATH=…`, and a space or an apostrophe is safe in # double quotes. A `:` splits a `$PATH` whatever the quoting, and a control # character (a newline breaks the printed line in two, an escape is the -# terminal's to act on) is no name a line can print as it is, so -# `install_commands` refuses both before this is asked. +# terminal's to act on) is no name a line can print as it is, and nor is a +# character a terminal draws as something other than itself (#837), so +# `install_commands` refuses all three before this is asked. export_path_line() { local dir="$1" backslash='\' quote="'" rest quoted="" case "$dir" in @@ -442,9 +443,104 @@ export_path_line() { esac } +# THE CHARACTERS A TERMINAL DRAWS AS SOMETHING OTHER THAN THEMSELVES (#837), +# by code point, a range as its first and last. A value holding one is +# refused wherever this command would print it, without printing it: +# `install_commands` refuses such a directory, `shape_checkout` such a +# repository and `ref_fault` such a ref. Every line that printed it would +# show another value: laid out as a terminal lays out a line (fribidi), +# `~/` U+202E `.local/bin` reads `~/nib/lacol.`, `/opt/tools-1` U+200F +# `/2/bin` reads `/opt/tools-12//bin`, and `~/.local` U+200B `/bin` reads +# `~/.local/bin`, which is another directory's name. +# 00AD the soft hyphen, drawn as a hyphen in a terminal cell and as +# nothing elsewhere, never as what it is; +# 061C, 200E-200F, 202A-202E, 2066-2069 +# the twelve bidirectional controls, which reorder the text +# around them, and which `spelled` in scripts/repo_shape.py +# escapes for that reason (#391): a mark reorders the +# characters beside it, an embedding, override or isolate the +# rest of the line; +# 2028-2029 the line and paragraph separators, which end a line for any +# reader that keeps Unicode's line breaks (Python's +# `str.splitlines` among them); +# 200B, 2060-206F, FEFF, FFF9-FFFB, 1D173-1D17A, E0001 +# the format characters of no width that nothing needs: the +# zero-width space and no-break space, the word joiner, the +# invisible operators, the deprecated format controls, the +# interlinear annotation characters, the musical beam, tie, +# slur and phrase controls, and the deprecated language tag. +# Unicode makes every code point from U+2060 to U+206F +# default-ignorable, so that range has no gap. +# Left out, each for its reason: +# * U+200C and U+200D (ZERO WIDTH NON-JOINER and JOINER): a name in +# Persian, in an Indic script or with an emoji sequence holds them; and +# U+180E (MONGOLIAN VOWEL SEPARATOR), which Mongolian needs as Persian +# needs U+200C; +# * the other format characters of no width, with which a script builds +# its signs: the Egyptian hieroglyph format controls (U+13430 to +# U+1343F), which join signs into a group, and the shorthand format +# controls (U+1BCA0 to U+1BCA3), which overlap and step Duployan signs; +# * the tags (U+E0020 to U+E007F), which make an emoji subdivision flag, +# the variation selectors (U+FE00 to U+FE0F, U+E0100 to U+E01EF), which +# choose how the character before them is drawn, and U+034F (COMBINING +# GRAPHEME JOINER), which keeps the characters around it one grapheme: +# each is drawn as part of the character before it; +# * U+1160 (HANGUL JUNGSEONG FILLER), which forms a syllable block with +# the consonant before it and is drawn as that block; +# * the format characters drawn over the digits after them (U+0600 to +# U+0605, U+06DD, U+070F, U+0890, U+0891, U+08E2, U+110BD, U+110CD), +# which take a cell and are drawn as themselves; +# * every space (U+00A0, U+202F, U+3000) and the blank Hangul fillers +# (U+115F, U+3164, U+FFA0): each is drawn as a blank cell, as a space +# is. +# Every entry is U+0080 or above: `misdrawn_characters` builds the two-, +# three- and four-byte forms of UTF-8 (U+0080 to U+10FFFF), and no other. +MISDRAWN_CHARACTERS=(00AD 061C 200B 200E-200F 2028-202E 2060-206F FEFF FFF9-FFFB 1D173-1D17A E0001) + +# WHICH OF THEM "$1" HOLDS, as `U+XXXX`, comma-separated, in that order, or +# nothing (#837). Run under `LC_ALL=C`, in a subshell, as the control check +# is, so the match is byte by byte in every locale: each character's UTF-8 +# is built from its code point with printf's octal escapes (`%b` reads +# `\0NNN`, as POSIX defines it), in two bytes below U+0800, three below +# U+10000 and four above, and looked for in the value as bytes, and no byte +# is read back as a number. So one list says both what is refused and what +# the refusal names. install.sh carries the list and this function as they +# are here. +misdrawn_characters() { + local value="$1" entry code last bytes name found="" + for entry in "${MISDRAWN_CHARACTERS[@]}"; do + code=$((0x${entry%-*})) + last=$((0x${entry#*-})) + while [ "$code" -le "$last" ]; do + if [ "$code" -lt 2048 ]; then + printf -v bytes '\\0%o\\0%o' $((0xC0 | (code >> 6))) \ + $((0x80 | (code & 63))) + elif [ "$code" -lt 65536 ]; then + printf -v bytes '\\0%o\\0%o\\0%o' $((0xE0 | (code >> 12))) \ + $((0x80 | ((code >> 6) & 63))) $((0x80 | (code & 63))) + else + printf -v bytes '\\0%o\\0%o\\0%o\\0%o' \ + $((0xF0 | (code >> 18))) $((0x80 | ((code >> 12) & 63))) \ + $((0x80 | ((code >> 6) & 63))) $((0x80 | (code & 63))) + fi + printf -v bytes '%b' "$bytes" + case "$value" in + *"$bytes"*) + printf -v name 'U+%04X' "$code" + found="${found:+$found, }$name" + ;; + *) ;; + esac + code=$((code + 1)) + done + done + printf '%s' "$found" +} + install_commands() { local dir="${OPENREPOSHAPE_BIN_DIR:-$HOME/.local/bin}" local name target source verb placed=0 recording=0 record spelled="" + local misdrawn # NO LINE ABOUT `$PATH` CAN NAME A DIRECTORY HOLDING A CONTROL CHARACTER # OR A `:` (#667, #760): the colon is what splits a `$PATH`; a newline # breaks the line the note prints in two; a carriage return rewrites the @@ -459,7 +555,8 @@ install_commands() { # under LC_ALL=C, is no fault of the directory's. The check runs under # `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, then `\200` # to `\237`), so a C1 control is refused in every locale, whatever else - # the value holds; U+2028 and U+2029 are refused in none. Left to the + # the value holds; U+2028 and U+2029, which only a UTF-8 locale's + # `[[:cntrl:]]` matched, are the next check's (#837). Left to the # locale, `[[:cntrl:]]` matches a C1 control only in a value that is # valid UTF-8: one byte that is not (a lone 0xE9) makes bash match byte # by byte, and then U+009B's two bytes are not controls, as under `C` @@ -471,6 +568,14 @@ install_commands() { if (LC_ALL=C; case "$dir" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then die "--install would place this command in a directory whose name holds a control character, such as a newline, a carriage return, a tab or an escape, and no line about \$PATH can print such a name as it is: a terminal acts on it. Nothing was placed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this again." fi + # A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED + # NEXT (#837), for the control characters' reason and before the same + # lines: the name would read as another directory's. The refusal names + # each one by its code point, because nobody can see it, and says + # nothing else of the value. `misdrawn_characters` says which they are. + misdrawn="$(LC_ALL=C; misdrawn_characters "$dir")" + [ -z "$misdrawn" ] || + die "--install would place this command in a directory whose name holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it, so no line about \$PATH can print the name as it reads. Nothing was placed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this again." # A WINDOWS SPELLING IS SPELLED AGAIN THE WAY BASH SPELLS IT (#649), with # `cygpath -u`, which Git Bash has, before anything is checked or placed: # `D:\tools\bin` becomes `/d/tools/bin`. Kept as typed, every line below @@ -607,9 +712,19 @@ find_python() { # a `git clone` command line, where a leading `-` is an option rather than a # repository and a control character could forge a line of output before this # script has said a word. The refusal does not echo the value back. +# +# A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED +# FIRST (#837), before the clone's refusal names `$REPO`, and in words that +# name it by its code point and nothing else of the value. First, because +# `[![:print:]]` is the locale's: under `C` it matches every byte past ASCII, +# and under a UTF-8 locale none of these characters, so after it the words +# would change with the locale. shape_checkout() { - local url + local url misdrawn url="$(shape_clone_url)" + misdrawn="$(LC_ALL=C; misdrawn_characters "$url")" + [ -z "$misdrawn" ] || + die "OPENREPOSHAPE_REPO holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. This tool will not put it on a \`git clone\` command line. Set it to a URL or a path." case "$url" in -* | *[![:print:]]*) die "OPENREPOSHAPE_REPO is not a value this tool will put on a \`git clone\` command line: it starts with '-' or carries a control character. Set it to a URL or a path." ;; esac @@ -646,13 +761,24 @@ shape_checkout() { # refusing spelling. An EMPTY ref is a fault too, which no caller could reach # before the record: an empty variable is no ref at all, while an empty file # is one. +# +# A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF (#837) is a +# fault between the length and the control character, named by its code +# point and never echoed, as `shape_checkout` refuses one: before the +# control check, whose `[![:print:]]` is the locale's, so the words do not +# change with the locale, and before the shape, which echoes the value. ref_fault() { - local ref="$1" what="$2" + local ref="$1" what="$2" misdrawn if [ -z "$ref" ]; then printf '%s' "$what is empty, which is not a branch, tag or commit this tool will pass to \`git checkout\`." elif [ "${#ref}" -gt 255 ]; then printf '%s' "$what is ${#ref} characters long; a branch, tag or commit this tool will pass to \`git checkout\` is at most 255 characters." else + misdrawn="$(LC_ALL=C; misdrawn_characters "$ref")" + if [ -n "$misdrawn" ]; then + printf '%s' "$what holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it. No branch, tag or commit this tool will pass to \`git checkout\` holds one." + return 0 + fi case "$ref" in *[![:print:]]*) printf '%s' "$what carries a control character, which is not a branch, tag or commit this tool will pass to \`git checkout\`." ;; [!A-Za-z0-9]* | *..* | *.lock | *[!A-Za-z0-9._/-]*) printf '%s' "$what is '$ref', which is not a branch, tag or commit this tool will pass to \`git checkout\`: a ref starts with a letter or digit, uses only letters, digits, '.', '_', '/', '-', has no '..' and does not end in '.lock'." ;; diff --git a/tests/test_install_sh.py b/tests/test_install_sh.py index 19604b80..d35ece00 100644 --- a/tests/test_install_sh.py +++ b/tests/test_install_sh.py @@ -46,15 +46,18 @@ import pytest from conftest import BASH, BASH_SKIP, REPO, bash_path -from test_openreposhape_command import (BIN_DIR_SPELLINGS, C1_BIN_DIRS, - COMMAND, COMMAND_REFUSAL_ENDS, +from test_openreposhape_command import (BIN_DIR_SPELLINGS, COMMAND, + COMMAND_REFUSAL_ENDS, CYGPATH_STATES, HOLDS_A_COLON, - HOLDS_A_CONTROL_CHARACTER, - INSTALL_NEEDS, LOCALES, - NO_FILE_NAMED_SO, + INSTALL_NEEDS, LINKED_BIN_DIRS, + LOCALES, NO_FILE_NAMED_SO, NO_NON_ASCII_INTO_GIT_BASH, - NOT_CONTROLS, UNNAMEABLE_BIN_DIRS, + NOT_CONTROLS, + REFUSED_IN_EVERY_LOCALE, RLO, + UNNAMEABLE_BIN_DIRS, assert_a_bin_dir_is_refused, + assert_refused_saying, + drawn_as_another, command_env, machine_path, put_cygpath, put_on_path, run_cmd, typed_bin_dir, write_script) @@ -905,21 +908,21 @@ def test_the_installer_holds_the_default_directory_to_the_colon_refusal( @NO_FILE_NAMED_SO -def test_a_control_character_is_refused_before_the_symlink_check_names_it( - tmp_path): +@pytest.mark.parametrize("name, said", LINKED_BIN_DIRS) +def test_a_bin_dir_is_refused_before_the_symlink_check_names_it( + tmp_path, name, said): """The symlink refusal (#563) names `$target`, so a directory holding a - carriage return that has pipx's link in it would have install.sh print - the value as typed (#760): the control character is refused first, and - the link stays as it was.""" + carriage return or U+202E that has pipx's link in it would have + install.sh print the value as typed (#760, #837): the value is refused + first, and the link stays as it was.""" net = Net(tmp_path) - bin_dir = tmp_path / "a\rforged" / "bin" + bin_dir = tmp_path / name / "bin" bin_dir.mkdir(parents=True) link = bin_dir / "openRepoShape" link.symlink_to(tmp_path / "nowhere") result = net.run(env={"OPENREPOSHAPE_BIN_DIR": str(bin_dir)}) assert net.calls() == [], net.calls() - assert_a_bin_dir_is_refused(result, str(bin_dir), - HOLDS_A_CONTROL_CHARACTER, + assert_a_bin_dir_is_refused(result, str(bin_dir), said, INSTALLER_REFUSAL_ENDS) assert link.is_symlink() @@ -935,9 +938,10 @@ def test_the_installer_places_a_directory_with_an_unprinted_letter( directory called `caf` and that letter for a fault it has not got. It is installed into, in either locale, and named as typed, and so is a no-break space, whose first byte is the C1 controls' (`0xC2`) and - whose second is the one past them, and so are U+2028 and U+2029 (see - `NOT_CONTROLS`). The line is looked for among the lines split at a - newline and nothing else: `splitlines` splits at those two as well.""" + whose second is the one past them, and so are the zero-width + non-joiner and joiner and the other names in `NOT_CONTROLS` (#837). + The line is looked for among the lines split at a newline and nothing + else, as `splitlines` splits at other characters too.""" net = Net(tmp_path) net.bin = tmp_path / name result = net.run(env={"OPENREPOSHAPE_BIN_DIR": bash_path(net.bin), @@ -950,25 +954,82 @@ def test_the_installer_places_a_directory_with_an_unprinted_letter( @NO_NON_ASCII_INTO_GIT_BASH @pytest.mark.parametrize("locale", LOCALES) -@pytest.mark.parametrize("value", C1_BIN_DIRS) -def test_the_installer_refuses_a_c1_control_in_every_locale_before_any_request( - tmp_path, value, locale): - """A C1 control (U+0080 to U+009F) in the directory, under `C` and - under `C.UTF-8`, alone, after a byte that is not UTF-8, and after a - colon (#760): exit 2 with the control-character refusal, no request, - and nothing of the value in either stream. Under a UTF-8 locale a byte - that is not UTF-8 anywhere in the value makes bash match byte by byte, - and `[[:cntrl:]]` then matches neither byte of U+009B, as it matches - neither under `C`.""" +@pytest.mark.parametrize("value, said", REFUSED_IN_EVERY_LOCALE) +def test_the_installer_refuses_in_every_locale_before_any_request( + tmp_path, value, said, locale): + """A C1 control (U+0080 to U+009F) in the directory (#760), or a + character a terminal draws as something other than itself (#837), + under `C` and under `C.UTF-8`, alone, after a byte that is not UTF-8, + and beside a colon: exit 2 with the refusal that says what it holds, + no request, and nothing of the value in either stream. Under a UTF-8 + locale a byte that is not UTF-8 anywhere in the value makes bash match + byte by byte, and `[[:cntrl:]]` then matches neither byte of U+009B, as + it matches neither under `C`; the second check looks for bytes in any + locale, and names what it found by its code point.""" net = Net(tmp_path) typed = value.format(d=bash_path(tmp_path)) result = net.run(env={"OPENREPOSHAPE_BIN_DIR": typed, "LC_ALL": locale}) assert net.calls() == [], net.calls() - assert_a_bin_dir_is_refused(result, typed, HOLDS_A_CONTROL_CHARACTER, - INSTALLER_REFUSAL_ENDS) + assert_a_bin_dir_is_refused(result, typed, said, INSTALLER_REFUSAL_ENDS) net.refused(result) +# A version, an option, the repository, the newest release's tag or the +# location github.com redirects to, holding U+202E (#837). Each arranges +# `net` and returns the arguments, the environment, what the refusal says, +# and whether the value comes from the network, so that the run asks it. + +def a_version(net): + return (("v1" + RLO + "forged",), {}, + f"the version {drawn_as_another('U+202E')}.", False) + + +def an_option(net): + return (("-x" + RLO + "forged",), {}, + f"an option {drawn_as_another('U+202E')}.", False) + + +def the_repository(net): + return ((), {"OPENREPOSHAPE_REPO": "acme/x" + RLO + "forged"}, + f"$OPENREPOSHAPE_REPO {drawn_as_another('U+202E')}.", False) + + +def the_newest_tag(net): + net.config["redirects"][latest()] = tag_page("v1" + RLO + "forged") + return ((), {}, "the newest release's tag " + f"{drawn_as_another('U+202E')}.", True) + + +def a_location(net): + net.config["redirects"][latest()] = ( + f"https://github.com/{SLUG}/releases/" + RLO + "forged") + return ((), {}, "it redirects to a location that names no release", + True) + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("arrange", [a_version, an_option, the_repository, + the_newest_tag, a_location], + ids=lambda arrange: arrange.__name__) +def test_the_installer_refuses_a_value_drawn_as_another_unechoed( + net, arrange, locale): + """`checked_repo`, `checked_version`, the option check and the + redirect's reading held their value to `[![:print:]]`, which a UTF-8 + locale's U+202E passes, and the refusal or fault after it printed the + value as typed (#837). Each now looks for a character of `MISDRAWN` + first and names it by its code point, under `C` and `C.UTF-8` alike: + exit 2, nothing placed, nothing fetched, no request at all where the + value is not the network's, and neither stream holds the value.""" + args, env, said, asks = arrange(net) + result = net.run(*args, env={**env, "LC_ALL": locale}) + assert_refused_saying(result, said) + net.refused(result) + calls = net.calls() + assert not any("-fsSL" in call for call in calls), calls + assert bool(calls) is asks, calls + + #: A curl that RUNS, in either script: tab-indented, alone or as the value #: of an assignment. Usage text and refusals carry curl lines too, indented #: with spaces, and are printed, never run. diff --git a/tests/test_openreposhape_command.py b/tests/test_openreposhape_command.py index 49b6247d..fe481d32 100644 --- a/tests/test_openreposhape_command.py +++ b/tests/test_openreposhape_command.py @@ -607,9 +607,39 @@ def test_a_pasted_path_note_reaches_an_interactive_bash_as_printed( #: child's environment (the byte comes back through `surrogateescape`). NOT_UTF8 = chr(0xDCE9) +#: THE CHARACTERS A TERMINAL DRAWS AS SOMETHING OTHER THAN THEMSELVES (#837), +#: by code point, as the launcher's `MISDRAWN_CHARACTERS` and install.sh's +#: copy list them: the soft hyphen, the twelve bidirectional controls, the +#: line and paragraph separators, and the format characters of no width +#: that nothing needs, three of them past U+FFFF. Written out here, not read +#: from either file, so that a change to that list is a change to this one +#: too. +MISDRAWN = (0x00AD, 0x061C, 0x200B, 0x200E, 0x200F, *range(0x2028, 0x202F), + *range(0x2060, 0x2070), 0xFEFF, *range(0xFFF9, 0xFFFC), + *range(0x1D173, 0x1D17B), 0xE0001) + +#: U+202E RIGHT-TO-LEFT OVERRIDE, the one of `MISDRAWN` the tests of each +#: place it is refused hold, after a `forged` tail's start. +RLO = chr(0x202E) + HOLDS_A_COLON = "holds a ':'" HOLDS_A_CONTROL_CHARACTER = "holds a control character" + +def drawn_as_another(named: str) -> str: + """What every refusal of a value holding `named` says of it, in both + files: the `U+XXXX` of each character of `MISDRAWN` in it, + comma-separated (#837). Each place goes on in words of its own.""" + return ("holds a character a terminal draws as something other than " + f"itself ({named}): one that is invisible, is drawn as another " + "character, ends a line or reorders the text around it") + + +def bin_dir_drawn_as_another(named: str) -> str: + """The directory's refusal of `named`, in both files (#837).""" + return (drawn_as_another(named) + ", so no line about $PATH can print " + "the name as it reads.") + #: How `--install`'s two refusals end, and how install.sh's do (#760): the #: remedy is the same sentence in both, less the verb. COMMAND_REFUSAL_ENDS = ( @@ -659,24 +689,35 @@ def test_a_pasted_path_note_reaches_an_interactive_bash_as_printed( "Git Bash is not run here") -def assert_a_bin_dir_is_refused(result, typed: str, said: str, - ending: str) -> None: - """`result` is a refusal, exit 2, of `typed`, a directory no line about - `$PATH` can name (#667, #760), saying `said` and ending as `ending` - does. A colon's refusal names the directory; nothing else does, and no - control character, and no `forged` tail, reaches either stream.""" +def assert_refused_saying(result, said: str) -> str: + """`result` is a refusal, exit 2, that says `said` on stderr and + nothing on stdout, and no control character, no character of + `MISDRAWN` and no `forged` tail reaches either stream (#760, #837). + The refusal, its whitespace collapsed, is returned.""" assert result.returncode == 2, result.stdout + result.stderr said_in_full = " ".join(result.stderr.split()) assert said in said_in_full, said_in_full - assert ending in said_in_full, said_in_full - if said == HOLDS_A_COLON: - assert f"'{typed}', which {said}" in said_in_full, said_in_full for stream in (result.stdout, result.stderr): assert "forged" not in stream, stream assert ESCAPE not in stream, stream assert DELETE not in stream, stream assert C1_CSI not in stream, stream + for code in MISDRAWN: + assert chr(code) not in stream, (f"U+{code:04X}", stream) assert result.stdout == "" + return said_in_full + + +def assert_a_bin_dir_is_refused(result, typed: str, said: str, + ending: str) -> None: + """`result` is a refusal, exit 2, of `typed`, a directory no line about + `$PATH` can name (#667, #760, #837), saying `said` and ending as + `ending` does. A colon's refusal names the directory; nothing else + does (see `assert_refused_saying`).""" + said_in_full = assert_refused_saying(result, said) + assert ending in said_in_full, said_in_full + if said == HOLDS_A_COLON: + assert f"'{typed}', which {said}" in said_in_full, said_in_full @pytest.mark.parametrize("value, said", UNNAMEABLE_BIN_DIRS) @@ -697,17 +738,30 @@ def test_a_bin_dir_no_path_line_can_name_is_refused_before_it_places_anything( assert list(tmp_path.iterdir()) == [] -#: Names no arm refuses, in either locale (#760): an e with an acute accent; -#: U+00A0, a no-break space, whose bytes `0xC2 0xA0` come right after the -#: C1 range's last (`0xC2 0x9F`); and U+2028 and U+2029, which `[[:cntrl:]]` -#: alone matches under a UTF-8 locale and not under `C`. The arm runs under -#: `C`, so it answers alike in both, and neither is a control a terminal -#: acts on (a line that holds one prints as it is). +#: Names no check refuses, in either locale (#760, #837): an e with an acute +#: accent; U+00A0, a no-break space, whose bytes `0xC2 0xA0` come right +#: after the C1 range's last (`0xC2 0x9F`); a Persian word (letters) whose +#: two parts U+200C ZERO WIDTH NON-JOINER keeps apart, and an emoji sequence +#: (a woman technologist) that U+200D ZERO WIDTH JOINER makes, which that +#: script and that sequence need; U+2026 and U+FF21, which start with the +#: bytes `0xE2` and `0xEF` that most of `MISDRAWN` start with; and a +#: subdivision flag (England's), which the tags U+E0020 to U+E007F make, +#: whose first three bytes are U+E0001's (`0xF3 0xA0 0x80` and +#: `0xF3 0xA0 0x81`). U+2028 and U+2029 were here until #837 refused them +#: with the rest of `MISDRAWN`. NOT_CONTROLS = [ pytest.param("caf" + chr(0xE9), id="an-e-acute"), pytest.param("a" + chr(0xA0) + "b", id="a-no-break-space"), - pytest.param("a" + chr(0x2028) + "b", id="a-line-separator"), - pytest.param("a" + chr(0x2029) + "b", id="a-paragraph-separator"), + pytest.param("".join(map(chr, (0x0646, 0x0627, 0x0645, 0x0647, 0x200C, + 0x0647, 0x0627))), + id="a-persian-word-and-u+200c"), + pytest.param(chr(0x1F469) + chr(0x200D) + chr(0x1F4BB), + id="an-emoji-sequence-and-u+200d"), + pytest.param("a" + chr(0x2026) + "b", id="u+2026"), + pytest.param("a" + chr(0xFF21) + "b", id="u+ff21"), + pytest.param("".join(map(chr, (0x1F3F4, 0xE0067, 0xE0062, 0xE0065, + 0xE006E, 0xE0067, 0xE007F))), + id="a-subdivision-flag"), ] @@ -721,8 +775,10 @@ def test_a_bin_dir_with_an_unprinted_letter_is_placed( printable, so the second would refuse a directory called `caf` and that letter for a fault it has not got. It is placed, in either locale, and so is a no-break space, whose first byte is the C1 - controls' (`0xC2`) and whose second is the one past them, and so are - U+2028 and U+2029, which only a UTF-8 locale's `[[:cntrl:]]` matches.""" + controls' (`0xC2`) and whose second is the one past them; and so are + the zero-width non-joiner and joiner, which a script or an emoji + needs, and two characters drawn as themselves whose first bytes are + those of the characters #837 refuses (see `NOT_CONTROLS`).""" result = run_cmd("--install", home=tmp_path, cwd=tmp_path, env={ "OPENREPOSHAPE_BIN_DIR": f"{bash_path(tmp_path)}/{name}", "LC_ALL": locale}) @@ -749,43 +805,219 @@ def test_a_bin_dir_with_an_unprinted_letter_is_placed( id="a-colon-then-a-byte-that-is-not-utf-8-and-a-c1-csi"), ] +#: Each of `MISDRAWN` in a directory, refused by the words that name it by +#: its code point (#837); then two at once, which are both named, in +#: `MISDRAWN`'s order; one after a byte that is not UTF-8; and one beside a +#: `:` and one in a drive spelling, whose own refusals name the directory, +#: so this one has to come before them. +MISDRAWN_BIN_DIRS = [ + *(pytest.param("{d}/a" + chr(code) + "forged/bin", + bin_dir_drawn_as_another(f"U+{code:04X}"), + id=f"u+{code:04x}") + for code in MISDRAWN), + pytest.param("{d}/a" + RLO + "b" + chr(0x200B) + "forged/bin", + bin_dir_drawn_as_another("U+200B, U+202E"), + id="two-at-once"), + pytest.param("{d}/caf" + NOT_UTF8 + RLO + "forged/bin", + bin_dir_drawn_as_another("U+202E"), + id="a-byte-that-is-not-utf-8-then-u+202e"), + pytest.param("{d}/a:b" + RLO + "forged/bin", + bin_dir_drawn_as_another("U+202E"), + id="a-colon-and-u+202e"), + pytest.param("D:\\tools" + RLO + "forged\\bin", + bin_dir_drawn_as_another("U+202E"), + id="a-drive-spelling-and-u+202e"), +] + +#: Every value refused alike under `C` and under `C.UTF-8`, with what its +#: refusal says: each of `C1_BIN_DIRS` as a control character (#760), each +#: of `MISDRAWN_BIN_DIRS` by what it holds (#837), and a tab before U+202E +#: as a control character, because the control check comes first. +REFUSED_IN_EVERY_LOCALE = [ + *(pytest.param(*value.values, HOLDS_A_CONTROL_CHARACTER, id=value.id) + for value in C1_BIN_DIRS), + *MISDRAWN_BIN_DIRS, + pytest.param("{d}/a\t" + RLO + "forged/bin", HOLDS_A_CONTROL_CHARACTER, + id="a-tab-then-u+202e"), +] + @NO_NON_ASCII_INTO_GIT_BASH @pytest.mark.parametrize("locale", LOCALES) -@pytest.mark.parametrize("value", C1_BIN_DIRS) -def test_a_c1_control_is_refused_in_every_locale_whatever_else_the_value_holds( - tmp_path, value, locale): - """Refused as an escape is, exit 2, nothing placed, and never echoed: - under `C`, where `[[:cntrl:]]` reads the two bytes one at a time and - matches neither, and under `C.UTF-8`, where a byte that is not UTF-8 - anywhere in the value makes bash match byte by byte (#760).""" +@pytest.mark.parametrize("value, said", REFUSED_IN_EVERY_LOCALE) +def test_a_bin_dir_is_refused_in_every_locale_whatever_else_it_holds( + tmp_path, value, said, locale): + """Refused, exit 2, nothing placed, and never echoed, under `C` and + under `C.UTF-8`. A C1 control is refused as an escape is (#760): under + `C`, `[[:cntrl:]]` reads its two bytes one at a time and matches + neither, and under `C.UTF-8` a byte that is not UTF-8 anywhere in the + value makes bash match byte by byte. A character a terminal draws as + something other than itself is refused in words that name it by its + code point, because nobody can see it (#837), and its bytes are looked + for as bytes, so the locale changes nothing there either.""" typed = value.format(d=bash_path(tmp_path)) result = run_cmd("--install", home=tmp_path, cwd=tmp_path, env={"OPENREPOSHAPE_BIN_DIR": typed, "LC_ALL": locale}) - assert_a_bin_dir_is_refused(result, typed, HOLDS_A_CONTROL_CHARACTER, - COMMAND_REFUSAL_ENDS) + assert_a_bin_dir_is_refused(result, typed, said, COMMAND_REFUSAL_ENDS) assert list(tmp_path.iterdir()) == [] +def misdrawn_check(path: Path) -> str: + """`MISDRAWN_CHARACTERS` and `misdrawn_characters`, as the file at + `path` carries them (#837).""" + text = path.read_text(encoding="utf-8") + [listed] = re.findall(r"^MISDRAWN_CHARACTERS=\(.*?\)\n", text, re.M) + [function] = re.findall(r"^misdrawn_characters\(\) \{\n.*?^\}\n", text, + re.M | re.S) + return listed + function + + +def test_install_sh_carries_the_launchers_misdrawn_check_as_it_is(): + """install.sh arrives alone down a pipe, so it carries a copy of the + launcher's list and function, as it carries the control check (#837). + The two are one text, so neither file can refuse a directory the other + places, or name it otherwise.""" + assert misdrawn_check(REPO / "install.sh") == misdrawn_check(COMMAND) + + +#: Every code point whose UTF-8 starts with the bytes one of `MISDRAWN` +#: starts with: U+0080 to U+00BF (`0xC2`), U+0600 to U+063F (`0xD8`), +#: U+2000 to U+20FF (`0xE2 0x80` to `0xE2 0x83`), U+FE00 to U+FFFF +#: (`0xEF 0xB8` to `0xEF 0xBF`), U+1D100 to U+1D1FF (`0xF0 0x9D 0x84` to +#: `0xF0 0x9D 0x87`) and U+E0000 to U+E007F (`0xF3 0xA0 0x80` and +#: `0xF3 0xA0 0x81`); and a sample of the other planes past U+FFFF: their +#: first and last code points, an emoji, a CJK ideograph, the variation +#: selectors past U+FFFF, and the format characters a script needs, which +#: stay allowed (the Kaithi number signs, the Egyptian hieroglyph and the +#: shorthand format controls). +SHARING_ITS_BYTES = (*range(0x80, 0xC0), *range(0x600, 0x640), + *range(0x2000, 0x2100), *range(0xFE00, 0x10000), + *range(0x1D100, 0x1D200), *range(0xE0000, 0xE0080), + 0x10000, 0x110BD, 0x110CD, *range(0x13430, 0x13440), + *range(0x1BCA0, 0x1BCA4), 0x1F600, 0x20000, 0xE0100, + 0xE01EF, 0xFFFFD, 0x10FFFF) + + +@pytest.mark.parametrize("name", ["openRepoShape", "install.sh"]) +def test_misdrawn_characters_names_its_own_and_nothing_else(tmp_path, name): + """The function alone, lifted out of the file and run under `LC_ALL=C`, + as both call sites run it, over one directory name for each code point + that starts with the bytes one it refuses starts with (#837): it names + exactly the code points of `MISDRAWN`, each as `U+XXXX`, and nothing + for the rest, so a name that merely holds a byte such as `0xE2`, `0xEF` + or `0xF3` is never refused, and the four-byte form is built right. The + names travel on stdin, as bytes, so this runs under Git Bash too, where + no such name crosses the environment.""" + script = tmp_path / "misdrawn.sh" + write_script(script, "LC_ALL=C\n" + misdrawn_check(REPO / name) + + 'while IFS= read -r -d "" value; do\n' + '\tmisdrawn_characters "$value"\n' + "\tprintf '\\n'\n" + "done\n") + ran = subprocess.run( + [BASH, bash_path(script)], capture_output=True, check=False, + input=b"".join(f"/a{chr(code)}b".encode("utf-8") + b"\0" + for code in SHARING_ITS_BYTES), + env=command_env(setup_sh=None)) + assert ran.returncode == 0, ran.stderr.decode("utf-8", "replace") + named = ran.stdout.decode("ascii").split("\n")[:-1] + assert dict(zip(SHARING_ITS_BYTES, named)) == { + code: f"U+{code:04X}" if code in MISDRAWN else "" + for code in SHARING_ITS_BYTES} + + +#: A directory that already holds pipx's link (#563), whose name holds a +#: carriage return (#760) or U+202E (#837), and what its refusal says. +LINKED_BIN_DIRS = [ + pytest.param("a\rforged", HOLDS_A_CONTROL_CHARACTER, + id="a-carriage-return"), + pytest.param("a" + RLO + "forged", bin_dir_drawn_as_another("U+202E"), + id="u+202e"), +] + + @NO_FILE_NAMED_SO -def test_a_control_character_is_refused_before_a_link_in_it_names_it( - tmp_path): +@pytest.mark.parametrize("name, said", LINKED_BIN_DIRS) +def test_a_bin_dir_is_refused_before_a_link_in_it_names_it( + tmp_path, name, said): """The symlink refusal (#563) names `$dir/openRepoShape`, so a directory - holding a carriage return that already has pipx's link in it would - have the refusal print the value as typed (#760): the control character + holding a carriage return or U+202E that already has pipx's link in it + would have the refusal print the value as typed (#760, #837): the value is refused first, and the link stays as it was.""" - bin_dir = tmp_path / "a\rforged" / "bin" + bin_dir = tmp_path / name / "bin" bin_dir.mkdir(parents=True) link = bin_dir / "openRepoShape" link.symlink_to(tmp_path / "nowhere") result = run_cmd("--install", home=tmp_path, cwd=tmp_path, env={"OPENREPOSHAPE_BIN_DIR": str(bin_dir)}) - assert_a_bin_dir_is_refused(result, str(bin_dir), - HOLDS_A_CONTROL_CHARACTER, + assert_a_bin_dir_is_refused(result, str(bin_dir), said, COMMAND_REFUSAL_ENDS) assert link.is_symlink() +# A ref or a repository the command would print, holding U+202E (#837). Each +# arranges its run in `tmp_path` and returns the arguments, the +# environment, the command to run, whose value the refusal names, and a +# path the run must not make (or None). + +def a_ref_to_record(tmp_path): + """`$OPENREPOSHAPE_REF`, which `--install` is asked to record (#564).""" + return (("--install",), + {"OPENREPOSHAPE_BIN_DIR": bash_path(tmp_path / "bin"), + "OPENREPOSHAPE_REF": "v1" + RLO + "forged", + "OPENREPOSHAPE_RECORD_REF": "1"}, + COMMAND, "$OPENREPOSHAPE_REF", tmp_path / "bin") + + +def a_recorded_ref(tmp_path): + """The record beside an installed command, which every run reads.""" + _result, installed = install_at(tmp_path, PINNED) + record = record_of(installed) + record.write_bytes(("v1" + RLO + "forged\n").encode("utf-8")) + return (("--version",), {}, installed, + f"the ref recorded in {bash_path(record)}", None) + + +def a_repository_to_clone(tmp_path): + """`$OPENREPOSHAPE_REPO`, which `--doctor` would clone, run from a copy + of the command with no checkout beside it, and a `git` that leaves a + mark if it runs.""" + fake = tmp_path / "fake-path" + fake.mkdir() + ran = tmp_path / "git-ran" + write_script(fake / "git", f"#!/bin/sh\n: > '{ran}'\nexit 1\n") + (fake / "git").chmod(0o755) + copy = tmp_path / "copy" / "openRepoShape" + copy.parent.mkdir() + copy.write_bytes(COMMAND.read_bytes()) + return (("--doctor", str(tmp_path)), + {"OPENREPOSHAPE_REPO": "acme/x" + RLO + "forged", + "TMPDIR": str(tmp_path), + "PATH": f"{fake}{os.pathsep}{os.environ['PATH']}"}, + copy, "OPENREPOSHAPE_REPO", ran) + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("arrange", [a_ref_to_record, a_recorded_ref, + a_repository_to_clone], + ids=lambda arrange: arrange.__name__) +def test_a_ref_or_repository_drawn_as_another_is_refused_unechoed( + tmp_path, arrange, locale): + """`ref_fault` and `shape_checkout` held a ref and a repository to + `[![:print:]]`, which a UTF-8 locale's U+202E passes: the shape + refusal then printed the ref as typed, and the clone's refusal the + repository (#837). Both now refuse a character of `MISDRAWN` first, by + its code point, under `C` and `C.UTF-8` alike: exit 2, nothing placed, + no clone, and neither stream holds the value.""" + args, env, command, what, untouched = arrange(tmp_path) + result = run_cmd(*args, home=tmp_path, cwd=tmp_path, command=command, + env={**env, "LC_ALL": locale}) + assert_refused_saying(result, f"{what} {drawn_as_another('U+202E')}.") + if untouched is not None: + assert not untouched.exists(), untouched + + @POSIX_NAME def test_the_default_directory_is_held_to_the_same_refusal(tmp_path): """No variable at all, and a `$HOME` that holds a `:`: the directory is