diff --git a/install.sh b/install.sh index 1428e0b8..cfc84568 100755 --- a/install.sh +++ b/install.sh @@ -399,9 +399,34 @@ misdrawn_characters() { printf '%s' "$found" } +# THE VALUE AS THE PERSON'S LOCALE READS IT, IN UTF-8, WHERE THAT LOCALE +# READS TWO OR MORE OF ITS BYTES AS ONE CHARACTER (#858): the launcher's +# `locale_reading`, copied as it is, for the reason the control check in +# `main` is copied. The launcher says when it reads the value again and +# when the bytes decide, and `tests/test_openreposhape_command.py` holds the +# two copies to one text. `main` asks it of a directory the bytes refuse. +locale_reading() { + local charmap reading + charmap="$(locale charmap 2>/dev/null)" || return 1 + case "$charmap" in + "" | [Uu][Tt][Ff]-8 | [Uu][Tt][Ff]8 | ANSI_X3.4-1968 | *ASCII*) return 1 ;; + *) ;; + esac + reading="$(printf '%s' "$1" | iconv -f "$charmap" -t UTF-8 2>/dev/null)" || + return 1 + [ "$(printf '%s' "$reading" | iconv -f UTF-8 -t "$charmap" 2>/dev/null)" = "$1" ] || + return 1 + ( + LC_ALL=C + letters=${reading//[$'\200'-$'\277']/} + [ "${#letters}" -lt "${#1}" ] || exit 1 + printf '%s' "$reading" + ) +} + main() { local version="" asked launcher_url bin_dir spelled target shown status=0 pinned=1 - local misdrawn + local misdrawn seen local machine="this machine is ready" # NOTHING A CHILD READS IS THIS SCRIPT. Piped, stdin is the pipe the @@ -446,23 +471,42 @@ main() { # print (`café` under LC_ALL=C) is no fault of the directory's. The check # runs under `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, # then `\200` to `\237`), so a C1 control is refused in every locale, - # whatever else the value holds; U+2028 and U+2029, which only a UTF-8 - # locale's `[[:cntrl:]]` matched, are the next check's (#837). + # whatever else the value holds, unless the locale reads those bytes as + # text (#858, below); U+2028 and U+2029, which only a UTF-8 locale's + # `[[:cntrl:]]` matched, are the next check's (#837). # Left to the locale, `[[:cntrl:]]` matches a C1 control only in a value # that is valid UTF-8: one byte that is not (a lone 0xE9) makes bash # match byte by byte, and then U+009B's two bytes are not controls, as # under `C` (measured with bash 5.2.21). The locale is `C` in a subshell, # so that the match is byte by byte whatever a libc says of a byte in a # UTF-8 locale, and the locale of every line below is as it was. - if (LC_ALL=C; case "$bin_dir" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + # WHERE THE BYTES ARE REFUSED, THE PERSON'S LOCALE IS ASKED HOW IT READS + # THEM (#858), as the command asks it: a multibyte charset that is not + # UTF-8 may read them as text, and then the C1 bytes and + # `misdrawn_characters` are looked for in `locale_reading`'s UTF-8 + # instead. Only a value the bytes refuse is read again, so no name + # placed before is refused, and where nothing reads it again the bytes + # decide, as before. `[[:cntrl:]]` is asked of the bytes as typed. The + # trade: such a locale, left in a terminal that reads UTF-8, has C1 + # bytes it reads as letters installed into and printed in the lines + # below, and that terminal acts on them. + seen="$bin_dir" + if (LC_ALL=C; case "$bin_dir" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac) || + [ -n "$(LC_ALL=C; misdrawn_characters "$bin_dir")" ]; then + seen="$(locale_reading "$bin_dir")" || seen="$bin_dir" + fi + if (LC_ALL=C + case "$bin_dir" in *[[:cntrl:]]*) exit 0 ;; *) ;; esac + case "$seen" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then die "the directory to install into holds a control character, such as a newline, a carriage return, a tab or an escape, and no line about \$PATH can print such a name as it is: a terminal acts on it. Nothing was installed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this line again." fi # A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED # NEXT (#837), as the command refuses it, and before any request: the # name would read as another directory's in every line below. The # refusal names each one by its code point, because nobody can see it, - # and says nothing else of the value. - misdrawn="$(LC_ALL=C; misdrawn_characters "$bin_dir")" + # and says nothing else of the value. It is asked of the locale's + # reading where there is one (#858). + misdrawn="$(LC_ALL=C; misdrawn_characters "$seen")" [ -z "$misdrawn" ] || die "the directory to install into holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it, so no line about \$PATH can print the name as it reads. Nothing was installed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this line again." # A WINDOWS SPELLING IS SPELLED THE WAY BASH SPELLS IT, THE COMMAND'S OWN diff --git a/openRepoShape b/openRepoShape index a94e5192..2ed8e2e5 100755 --- a/openRepoShape +++ b/openRepoShape @@ -415,7 +415,10 @@ record_ref() { # character (a newline breaks the printed line in two, an escape is the # terminal's to act on) is no name a line can print as it is, and nor is a # character a terminal draws as something other than itself (#837), so -# `install_commands` refuses all three before this is asked. +# `install_commands` refuses all three before this is asked. The C1 +# controls and those characters are refused as the person's locale reads +# the name (#858): a multibyte locale that is not UTF-8 can hand this line +# C1 bytes it reads as letters, which a terminal that reads UTF-8 acts on. export_path_line() { local dir="$1" backslash='\' quote="'" rest quoted="" case "$dir" in @@ -537,10 +540,60 @@ misdrawn_characters() { printf '%s' "$found" } +# THE VALUE AS THE PERSON'S LOCALE READS IT, IN UTF-8, WHERE THAT LOCALE +# READS TWO OR MORE OF ITS BYTES AS ONE CHARACTER (#858). The control check +# and `misdrawn_characters` look for UTF-8's bytes under `LC_ALL=C`, and a +# multibyte charset that is not UTF-8 reads some of those bytes as text: +# GBK, GB18030 and CP936 read `0xC2 0x80` to `0xC2 0x9F` as 32 hanzi; GBK, +# Big5, EUC-JP and EUC-KR read U+00AD's two bytes as a hanzi, a kanji or a +# Hangul syllable; and in Shift_JIS a kanji whose second byte is `0xC2`, +# before one of the common kanji (first byte `0x88` to `0x98`), puts a C1 +# control's two bytes side by side (measured with Python's codecs). +# `install_commands` asks this of a value those checks refuse, and asks them +# again of what it prints. It prints the reading, and succeeds, only where: +# * `locale charmap`, asked under the person's own `LC_ALL`, `LC_CTYPE` +# and `LANG`, answers with a charset that is neither UTF-8, whose +# reading is the bytes, nor ASCII, which reads no byte past 0x7F; +# * `iconv` reads the whole value in that charset; +# * the reading, written back into that charset by `iconv`, is the value +# again, byte for byte, so an `iconv` that drops or changes a byte, or +# prints nothing at all and exits 0, gives no reading; and +# * the reading has fewer characters than the value has bytes, counted +# under `LC_ALL=C` as the bytes of UTF-8 that start a character (0x80 +# to 0xBF continue one). A single-byte charset reads each byte alone, +# and there the bytes decide, as before: under a Latin-1 `LANG` left in +# a terminal that reads UTF-8, U+FEFF's three bytes would read as three +# letters. +# Otherwise it prints nothing and fails, and the bytes decide: no `locale` +# or no `iconv` (a minimal container), a charset either does not know, or +# a value the charset cannot read. The locale is taken at its word, and +# that is the trade: a multibyte locale that is not UTF-8, left in a +# terminal that reads UTF-8, is handed the C1 bytes and the bytes of +# `MISDRAWN_CHARACTERS` it reads as letters, and that terminal acts on +# them. install.sh carries this function as it is here. +locale_reading() { + local charmap reading + charmap="$(locale charmap 2>/dev/null)" || return 1 + case "$charmap" in + "" | [Uu][Tt][Ff]-8 | [Uu][Tt][Ff]8 | ANSI_X3.4-1968 | *ASCII*) return 1 ;; + *) ;; + esac + reading="$(printf '%s' "$1" | iconv -f "$charmap" -t UTF-8 2>/dev/null)" || + return 1 + [ "$(printf '%s' "$reading" | iconv -f UTF-8 -t "$charmap" 2>/dev/null)" = "$1" ] || + return 1 + ( + LC_ALL=C + letters=${reading//[$'\200'-$'\277']/} + [ "${#letters}" -lt "${#1}" ] || exit 1 + printf '%s' "$reading" + ) +} + install_commands() { local dir="${OPENREPOSHAPE_BIN_DIR:-$HOME/.local/bin}" local name target source verb placed=0 recording=0 record spelled="" - local misdrawn + local misdrawn seen # NO LINE ABOUT `$PATH` CAN NAME A DIRECTORY HOLDING A CONTROL CHARACTER # OR A `:` (#667, #760): the colon is what splits a `$PATH`; a newline # breaks the line the note prints in two; a carriage return rewrites the @@ -555,9 +608,10 @@ install_commands() { # under LC_ALL=C, is no fault of the directory's. The check runs under # `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, then `\200` # to `\237`), so a C1 control is refused in every locale, whatever else - # the value holds; U+2028 and U+2029, which only a UTF-8 locale's - # `[[:cntrl:]]` matched, are the next check's (#837). Left to the - # locale, `[[:cntrl:]]` matches a C1 control only in a value that is + # the value holds, unless the locale reads those bytes as text (#858, + # below); U+2028 and U+2029, which only a UTF-8 locale's `[[:cntrl:]]` + # matched, are the next check's (#837). Left to the locale, + # `[[:cntrl:]]` matches a C1 control only in a value that is # valid UTF-8: one byte that is not (a lone 0xE9) makes bash match byte # by byte, and then U+009B's two bytes are not controls, as under `C` # (measured with bash 5.2.21). The locale is `C` in a subshell, so that @@ -565,15 +619,40 @@ install_commands() { # locale, and the locale of every line below is as it was. The colon is # checked after the Windows spelling, which holds one by definition, is # respelled without one, and refuses what it cannot respell. - if (LC_ALL=C; case "$dir" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then + # WHERE THE BYTES ARE REFUSED, THE PERSON'S LOCALE IS ASKED HOW IT READS + # THEM (#858): a multibyte charset that is not UTF-8 may read them as + # text, and then the C1 bytes and `misdrawn_characters` are looked for + # in `locale_reading`'s UTF-8 instead. Only a value the bytes refuse is + # read again, so no name placed before is refused, and where nothing + # reads it again the bytes decide, as before. `[[:cntrl:]]` is asked of + # the bytes as typed: those characters are ASCII's in every such + # charset, and `$(…)` drops a newline at the end of the reading. The + # trade: such a locale, left in a terminal that reads UTF-8, has C1 bytes + # it reads as letters placed and printed in the lines below, and that + # terminal acts on them. A lone C1 byte that is not UTF-8 (`0x9B`) is + # placed, as before: a terminal that reads UTF-8 draws it as a character + # it cannot read, and only one set to an 8-bit charset acts on it. Under + # an ISO 8859 charmap, the one such a terminal goes with, the charset is + # known and reads that byte as U+009B, but nothing here reads the name + # again, so `a`, `0x9B`, `b` is placed there too: a candidate, left + # (#858). + seen="$dir" + if (LC_ALL=C; case "$dir" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac) || + [ -n "$(LC_ALL=C; misdrawn_characters "$dir")" ]; then + seen="$(locale_reading "$dir")" || seen="$dir" + fi + if (LC_ALL=C + case "$dir" in *[[:cntrl:]]*) exit 0 ;; *) ;; esac + case "$seen" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then die "--install would place this command in a directory whose name holds a control character, such as a newline, a carriage return, a tab or an escape, and no line about \$PATH can print such a name as it is: a terminal acts on it. Nothing was placed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this again." fi # A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED # NEXT (#837), for the control characters' reason and before the same # lines: the name would read as another directory's. The refusal names # each one by its code point, because nobody can see it, and says - # nothing else of the value. `misdrawn_characters` says which they are. - misdrawn="$(LC_ALL=C; misdrawn_characters "$dir")" + # nothing else of the value. `misdrawn_characters` says which they are, + # in the locale's reading where there is one (#858). + misdrawn="$(LC_ALL=C; misdrawn_characters "$seen")" [ -z "$misdrawn" ] || die "--install would place this command in a directory whose name holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it, so no line about \$PATH can print the name as it reads. Nothing was placed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this again." # A WINDOWS SPELLING IS SPELLED AGAIN THE WAY BASH SPELLS IT (#649), with diff --git a/tests/test_install_sh.py b/tests/test_install_sh.py index d35ece00..86c781a2 100644 --- a/tests/test_install_sh.py +++ b/tests/test_install_sh.py @@ -53,13 +53,19 @@ LOCALES, NO_FILE_NAMED_SO, NO_NON_ASCII_INTO_GIT_BASH, NOT_CONTROLS, + HONEST, MARKS, + PLACED_AS_THE_LOCALE_READS, + REFUSED_AS_THE_BYTES_SAY, + REFUSED_UNDER_C_OR_UTF_8, REFUSED_IN_EVERY_LOCALE, RLO, UNNAMEABLE_BIN_DIRS, assert_a_bin_dir_is_refused, assert_refused_saying, drawn_as_another, command_env, machine_path, - put_cygpath, put_on_path, run_cmd, + put_a_locale, put_cygpath, + put_on_path, run_cmd, + the_persons_locale, typed_bin_dir, write_script) INSTALLER = REPO / "install.sh" @@ -974,6 +980,67 @@ def test_the_installer_refuses_in_every_locale_before_any_request( net.refused(result) +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("charmap, iconv, name", PLACED_AS_THE_LOCALE_READS) +def test_the_installer_places_a_directory_a_multibyte_locale_reads_as_text( + tmp_path, charmap, iconv, name): + """A name the bytes refuse, read as text by the locale's multibyte + charset (#858): install.sh asks `locale charmap` under the person's own + `LANG` and reads the name through `iconv`, as the command does, and the + command it fetched places itself there, under the same `locale` and + `iconv`, with the release recorded. Neither refuses what the other + places, and neither asks `iconv` of a name the bytes do not refuse.""" + net = Net(tmp_path) + put_a_locale(net.path, HONEST, iconv) + net.bin = tmp_path / name + result = net.run(env={"OPENREPOSHAPE_BIN_DIR": bash_path(net.bin), + **the_persons_locale(charmap)}) + assert result.returncode == 0, result.stdout + result.stderr + net.installed(result) + assert not (net.path / "iconv-ran").exists() + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("charmap, kind, iconv, name, said", + REFUSED_AS_THE_BYTES_SAY) +def test_the_installer_refuses_as_the_bytes_or_the_reading_say( + tmp_path, charmap, kind, iconv, name, said): + """Where the locale's reading still holds a C1 control or a character + drawn as another, or where nothing reads the name again (#858), + install.sh refuses it in #804's or #837's words, before any request: + exit 2, nothing placed, nothing of the value in either stream. With no + `locale` on this `$PATH` at all, every other test of the directory here + runs that way too.""" + net = Net(tmp_path) + put_a_locale(net.path, kind, iconv) + typed = f"{bash_path(tmp_path)}/{name}" + result = net.run(env={"OPENREPOSHAPE_BIN_DIR": typed, + **the_persons_locale(charmap)}) + assert net.calls() == [], net.calls() + assert_a_bin_dir_is_refused(result, typed, said, INSTALLER_REFUSAL_ENDS) + net.refused(result) + assert not (net.path / "iconv-ran").exists() + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("name, said", REFUSED_UNDER_C_OR_UTF_8) +def test_the_installer_refuses_as_the_bytes_say_under_c_or_c_utf_8( + tmp_path, name, said, locale): + """The person's locale is `C` or `C.UTF-8`, and `locale charmap` + answers as glibc's does: install.sh refuses as the bytes say, in #804's + and #837's words, before any request, and asks `iconv` nothing + (#858).""" + net = Net(tmp_path) + put_a_locale(net.path, HONEST, MARKS) + typed = f"{bash_path(tmp_path)}/{name}" + result = net.run(env={"OPENREPOSHAPE_BIN_DIR": typed, "LC_ALL": locale}) + assert net.calls() == [], net.calls() + assert_a_bin_dir_is_refused(result, typed, said, INSTALLER_REFUSAL_ENDS) + net.refused(result) + assert not (net.path / "iconv-ran").exists() + + # A version, an option, the repository, the newest release's tag or the # location github.com redirects to, holding U+202E (#837). Each arranges # `net` and returns the arguments, the environment, what the refusal says, diff --git a/tests/test_openreposhape_command.py b/tests/test_openreposhape_command.py index fe481d32..a0516af1 100644 --- a/tests/test_openreposhape_command.py +++ b/tests/test_openreposhape_command.py @@ -862,14 +862,20 @@ def test_a_bin_dir_is_refused_in_every_locale_whatever_else_it_holds( assert list(tmp_path.iterdir()) == [] +def shell_function(path: Path, name: str) -> str: + """The function `name`, from its first line to its closing brace, as + the file at `path` carries it.""" + [function] = re.findall(rf"^{name}\(\) \{{\n.*?^\}}\n", + path.read_text(encoding="utf-8"), re.M | re.S) + return function + + def misdrawn_check(path: Path) -> str: """`MISDRAWN_CHARACTERS` and `misdrawn_characters`, as the file at `path` carries them (#837).""" text = path.read_text(encoding="utf-8") [listed] = re.findall(r"^MISDRAWN_CHARACTERS=\(.*?\)\n", text, re.M) - [function] = re.findall(r"^misdrawn_characters\(\) \{\n.*?^\}\n", text, - re.M | re.S) - return listed + function + return listed + shell_function(path, "misdrawn_characters") def test_install_sh_carries_the_launchers_misdrawn_check_as_it_is(): @@ -926,6 +932,280 @@ def test_misdrawn_characters_names_its_own_and_nothing_else(tmp_path, name): for code in SHARING_ITS_BYTES} +def test_install_sh_carries_the_launchers_locale_reading_as_it_is(): + """install.sh carries the launcher's `locale_reading` as well (#858), + so the two read a directory the bytes refuse alike, and neither places + one the other refuses.""" + assert shell_function(REPO / "install.sh", "locale_reading") \ + == shell_function(COMMAND, "locale_reading") + + +#: HOW A `locale` OR AN `iconv` A TEST PUTS FIRST ON `$PATH` BEHAVES (#858). +#: Every `locale` answers `locale charmap` as glibc's does, from the first of +#: `$LC_ALL`, `$LC_CTYPE` and `$LANG` that is set and not empty: a marker +#: locale `xx_XX.` gives that charmap, `C.UTF-8` gives `UTF-8`, and +#: anything else `ANSI_X3.4-1968`. `HONEST` stops there; `FAILS` answers and +#: then exits 1; `MISSING` exits 127, as a shell answers a name it cannot +#: find. An `iconv` is this machine's (None), or `MISSING`, or `EMPTY`, one +#: that prints nothing and exits 0, as no working one does, or +#: `READS_THEN_FAILS`, this machine's run in full and then exit 1, so that +#: the whole reading is printed by an `iconv` that failed, or `MARKS`, one +#: that leaves `iconv-ran` beside itself and exits 1, so that a run which +#: must not ask it shows whether it did. +HONEST, FAILS, MISSING, EMPTY, READS_THEN_FAILS, MARKS = ( + "honest", "fails", "missing", "empty", "reads-then-fails", "marks") + +#: What a `locale` stub answers `locale charmap` with, as described above. +CHARMAP_ANSWER = ( + '[ "$1" = charmap ] || exit 2\n' + 'name=${LC_ALL:-${LC_CTYPE:-$LANG}}\n' + 'case "$name" in\n' + 'xx_XX.*) printf \'%s\\n\' "${name#xx_XX.}" ;;\n' + "C.UTF-8) printf 'UTF-8\\n' ;;\n" + "*) printf 'ANSI_X3.4-1968\\n' ;;\n" + "esac\n") + + +def put_a_locale(directory: Path, kind: str = HONEST, + iconv: str | None = None) -> None: + """A `locale` in `directory` that behaves as `kind` says, and an + `iconv` beside it that behaves as `iconv` says (#858). No GB18030, GBK + or Shift_JIS locale is installed here or on CI, and none can be built, + so the person's locale is a marker name the stub alone knows, and bash + itself runs in `C` under it.""" + write_script(directory / "locale", "#!/bin/sh\n" + { + HONEST: CHARMAP_ANSWER, FAILS: CHARMAP_ANSWER + "exit 1\n", + MISSING: "exit 127\n"}[kind]) + (directory / "locale").chmod(0o755) + real = shutil.which("iconv") + assert real, "this machine has no iconv, which these cases read with" + if iconv is None: + put_on_path(directory, "iconv", real) + return + write_script(directory / "iconv", "#!/bin/sh\n" + { + MISSING: "exit 127\n", EMPTY: "cat >/dev/null\nexit 0\n", + READS_THEN_FAILS: f'{shlex.quote(real)} "$@"\nexit 1\n', + MARKS: f": > {shlex.quote(str(directory / 'iconv-ran'))}\nexit 1\n", + }[iconv]) + (directory / "iconv").chmod(0o755) + + +def the_persons_locale(charmap: str) -> dict: + """The environment of a person whose locale reads `charmap`: `LANG` the + marker locale, and `LC_ALL` and `LC_CTYPE` empty, as unset (#858). bash + says nothing of a `LANG` that is not installed, and runs in `C`.""" + return {"LANG": f"xx_XX.{charmap}", "LC_ALL": "", "LC_CTYPE": ""} + + +#: One of the 32 hanzi that GBK, GB18030 and CP936 read `0xC2 0x80` to +#: `0xC2 0x9F` as, which in UTF-8 are a C1 control's two bytes: Python +#: hands `chr(0x9B)` to a child as `0xC2 0x9B`, U+8078 in GB18030 (#858). +A_HANZI_OF_C1_BYTES = "a" + C1_CSI + "b" + +#: Names a multibyte charset reads as text, each with that charset and its +#: `iconv` (#858): the first, the issue's and the last of the 32 hanzi, so +#: that a C1 range that stops one short at either end, where it decides +#: whether to ask the locale, is caught; U+00AD's two bytes, U+9885 in GBK; +#: U+2028's three, a kanji and a half-width katakana in Shift_JIS; in +#: Shift_JIS a half-width katakana (`0xC2`) before a character whose first +#: byte is `0x81`, read as U+FF82 and then U+3005 or U+FF3F, whose UTF-8 +#: holds a `0x80` or a `0xBF`, the first and the last byte that continues a +#: character, which the count of the reading's characters must not count; +#: and a name the bytes do not refuse, which is never read again: the +#: `iconv` there would leave its mark. +PLACED_AS_THE_LOCALE_READS = [ + pytest.param("GB18030", None, "a" + chr(0x80) + "b", + id="gb18030-0xc2-0x80"), + pytest.param("GB18030", None, A_HANZI_OF_C1_BYTES, + id="gb18030-0xc2-0x9b"), + pytest.param("GB18030", None, "a" + chr(0x9F) + "b", + id="gb18030-0xc2-0x9f"), + pytest.param("GBK", None, "a" + chr(0xAD) + "b", id="gbk-u+00ad-bytes"), + pytest.param("SHIFT_JIS", None, "a" + chr(0x2028) + "b", + id="shift_jis-u+2028-bytes"), + pytest.param("SHIFT_JIS", None, "a" + chr(0x81) + "Xb", + id="shift_jis-0xc2-0x81-0x58"), + pytest.param("SHIFT_JIS", None, "a" + chr(0x81) + "Qb", + id="shift_jis-0xc2-0x81-0x51"), + pytest.param("GB18030", MARKS, "caf" + chr(0xE9), + id="no-byte-refused-never-read-again"), +] + +#: Names refused under a marker locale, and what the refusal says (#858). +#: The locale's reading is checked itself: GB18030 writes U+009B as +#: `0x81 0x30 0x83 0x37` and U+202E as `0x81 0x36 0xA7 0x31`, which are no +#: UTF-8, so a name holding either beside a hanzi the bytes refuse is +#: refused as its reading says. And the bytes decide, in #804's and #837's +#: words, where `locale` fails or is not there; where `iconv` is not there, +#: prints nothing, or prints the whole reading and then fails, which the +#: round trip alone would let through; where GB18030 cannot read a byte +#: (`0xFF`); under a single-byte charset, as Latin-1 reads U+FEFF's three +#: bytes, each alone; with a newline at the end, which `$(…)` drops from a +#: reading; and where `locale` answers nothing, UTF-8 or ASCII, under which +#: `iconv` is never asked. +REFUSED_AS_THE_BYTES_SAY = [ + pytest.param("GB18030", HONEST, None, + A_HANZI_OF_C1_BYTES + "\udc81" + "0" + "\udc83" + "7", + HOLDS_A_CONTROL_CHARACTER, id="gb18030-reads-u+009b"), + pytest.param("GB18030", HONEST, None, + A_HANZI_OF_C1_BYTES + "\udc81" + "6" + "\udca7" + "1", + bin_dir_drawn_as_another("U+202E"), + id="gb18030-reads-u+202e"), + pytest.param("GB18030", FAILS, None, A_HANZI_OF_C1_BYTES, + HOLDS_A_CONTROL_CHARACTER, id="a-locale-that-fails"), + pytest.param("GB18030", MISSING, None, A_HANZI_OF_C1_BYTES, + HOLDS_A_CONTROL_CHARACTER, id="no-locale"), + pytest.param("GB18030", HONEST, MISSING, A_HANZI_OF_C1_BYTES, + HOLDS_A_CONTROL_CHARACTER, id="no-iconv"), + pytest.param("GB18030", HONEST, EMPTY, A_HANZI_OF_C1_BYTES, + HOLDS_A_CONTROL_CHARACTER, id="an-iconv-that-prints-nothing"), + pytest.param("GB18030", HONEST, READS_THEN_FAILS, A_HANZI_OF_C1_BYTES, + HOLDS_A_CONTROL_CHARACTER, + id="an-iconv-that-reads-then-fails"), + pytest.param("GB18030", HONEST, None, A_HANZI_OF_C1_BYTES + chr(0xDCFF), + HOLDS_A_CONTROL_CHARACTER, id="a-byte-gb18030-cannot-read"), + pytest.param("ISO-8859-1", HONEST, None, "a" + chr(0xFEFF) + "b", + bin_dir_drawn_as_another("U+FEFF"), + id="a-single-byte-charset"), + pytest.param("GB18030", HONEST, None, A_HANZI_OF_C1_BYTES + "\n", + HOLDS_A_CONTROL_CHARACTER, id="a-newline-at-the-end"), + *(pytest.param(answer, HONEST, MARKS, A_HANZI_OF_C1_BYTES, + HOLDS_A_CONTROL_CHARACTER, + id=f"iconv-unasked-under-{answer or 'no-answer'}") + for answer in ("", "UTF-8", "utf8", "ANSI_X3.4-1968", "US-ASCII")), +] + +#: Names refused when the person's locale is `C` or `C.UTF-8` themselves +#: (#858), in #804's and #837's words, `iconv` never asked: three of +#: `PLACED_AS_THE_LOCALE_READS`. +REFUSED_UNDER_C_OR_UTF_8 = [ + pytest.param(A_HANZI_OF_C1_BYTES, HOLDS_A_CONTROL_CHARACTER, + id="0xc2-0x9b"), + pytest.param("a" + chr(0xAD) + "b", bin_dir_drawn_as_another("U+00AD"), + id="u+00ad"), + pytest.param("a" + chr(0x2028) + "b", bin_dir_drawn_as_another("U+2028"), + id="u+2028"), +] + + +def install_under_a_locale(tmp_path: Path, typed: str, environment: dict, + kind: str, iconv: str | None): + """`--install` into `typed`, with `put_a_locale`'s `locale` and `iconv` + first on `$PATH` in `tmp_path/tools`, and `environment` naming the + person's locale (#858).""" + tools = tmp_path / "tools" + tools.mkdir() + put_a_locale(tools, kind, iconv) + return run_cmd("--install", home=tmp_path, cwd=tmp_path, env={ + "OPENREPOSHAPE_BIN_DIR": typed, **environment, + "PATH": f"{tools}{os.pathsep}{os.environ['PATH']}"}) + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("charmap, iconv, name", PLACED_AS_THE_LOCALE_READS) +def test_a_bin_dir_a_multibyte_locale_reads_as_text_is_placed( + tmp_path, charmap, iconv, name): + """The control check and `misdrawn_characters` look for UTF-8's bytes, + and GB18030 reads `0xC2 0x9B` as a hanzi, not as U+009B: the directory + was refused as holding a control character, in a locale where it holds + none (#858). Where the bytes are refused, `locale charmap` is asked + under the person's own `LANG`, and a multibyte charset's reading of the + name, through `iconv`, has the last word. So a name GB18030, GBK or + Shift_JIS reads as text is placed; and a name the bytes do not refuse + is placed without `iconv` being asked.""" + result = install_under_a_locale( + tmp_path, f"{bash_path(tmp_path)}/{name}", + the_persons_locale(charmap), HONEST, iconv) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / name / "openRepoShape").read_bytes() \ + == COMMAND.read_bytes() + assert not (tmp_path / "tools" / "iconv-ran").exists() + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("charmap, kind, iconv, name, said", + REFUSED_AS_THE_BYTES_SAY) +def test_a_bin_dir_is_refused_as_its_bytes_or_its_reading_say( + tmp_path, charmap, kind, iconv, name, said): + """Where the locale's reading still holds a C1 control or a character + drawn as another, it is refused as the reading says; and where the + locale's charset is UTF-8, ASCII, single-byte or not said, where + `iconv` is not there, prints nothing or cannot read the name, the bytes + decide, as before (#858): refused, exit 2, nothing placed, never + echoed, in #804's or #837's words. A newline at the end is looked for + in the bytes as typed, not in the reading.""" + typed = f"{bash_path(tmp_path)}/{name}" + result = install_under_a_locale(tmp_path, typed, + the_persons_locale(charmap), kind, iconv) + assert_a_bin_dir_is_refused(result, typed, said, COMMAND_REFUSAL_ENDS) + assert [path.name for path in tmp_path.iterdir()] == ["tools"] + assert not (tmp_path / "tools" / "iconv-ran").exists() + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("name, said", REFUSED_UNDER_C_OR_UTF_8) +def test_a_bin_dir_is_refused_as_its_bytes_say_under_c_or_c_utf_8( + tmp_path, name, said, locale): + """The person's locale is `C` or `C.UTF-8`, as `LC_ALL` says, and + `locale charmap` answers `ANSI_X3.4-1968` or `UTF-8`, as glibc's does: + the bytes decide, in #804's and #837's words, and `iconv` is never + asked (#858).""" + typed = f"{bash_path(tmp_path)}/{name}" + result = install_under_a_locale(tmp_path, typed, {"LC_ALL": locale}, + HONEST, MARKS) + assert_a_bin_dir_is_refused(result, typed, said, COMMAND_REFUSAL_ENDS) + assert not (tmp_path / "tools" / "iconv-ran").exists() + + +#: The Python codec that reads as each charset of +#: `PLACED_AS_THE_LOCALE_READS` does, and Latin-1's. +CODECS = {"GB18030": "gb18030", "GBK": "gbk", "SHIFT_JIS": "shift_jis", + "ISO-8859-1": "latin-1"} + + +@NO_NON_ASCII_INTO_GIT_BASH +@pytest.mark.parametrize("locale", LOCALES) +@pytest.mark.parametrize("name", ["openRepoShape", "install.sh"]) +def test_locale_reading_counts_bytes_whatever_bash_itself_reads( + tmp_path, name, locale): + """`locale_reading` alone, lifted out of each file, with a `locale` + function that answers each name's charset, under bash's own `C` and + `C.UTF-8` (#858). In a GB18030 locale bash itself reads GB18030, and + no such locale can be installed here, so `C.UTF-8` stands in for a + multibyte locale of bash's own. For each name of + `PLACED_AS_THE_LOCALE_READS` the bytes refuse, the reading is the one + Python's codec gives; for Latin-1's U+FEFF bytes and a byte GB18030 + cannot read there is none. So the count is of bytes in either.""" + rows = [(param.values[0], param.values[2]) + for param in PLACED_AS_THE_LOCALE_READS + if param.values[1] is None] + rows += [("ISO-8859-1", "a" + chr(0xFEFF) + "b"), + ("GB18030", A_HANZI_OF_C1_BYTES + chr(0xDCFF))] + script = tmp_path / "reading.sh" + write_script(script, f"LC_ALL={locale}\n" + "locale() { printf '%s\\n' \"$CHARMAP\"; }\n" + + shell_function(REPO / name, "locale_reading") + + 'while IFS= read -r -d "" CHARMAP && ' + 'IFS= read -r -d "" value; do\n' + '\tif reading="$(locale_reading "$value")"; then\n' + "\t\tprintf 'read %s\\0' \"$reading\"\n" + "\telse\n" + "\t\tprintf 'none\\0'\n" + "\tfi\n" + "done\n") + ran = subprocess.run( + [BASH, bash_path(script)], capture_output=True, check=False, + input=b"".join(charmap.encode("ascii") + b"\0" + + value.encode("utf-8", "surrogateescape") + b"\0" + for charmap, value in rows), + env=command_env(setup_sh=None)) + assert ran.returncode == 0, ran.stderr.decode("utf-8", "replace") + expected = [b"read " + value.encode("utf-8", "surrogateescape").decode( + CODECS[charmap]).encode("utf-8") for charmap, value in rows[:-2]] + assert ran.stdout.split(b"\0")[:-1] == [*expected, b"none", b"none"] + + #: A directory that already holds pipx's link (#563), whose name holds a #: carriage return (#760) or U+202E (#837), and what its refusal says. LINKED_BIN_DIRS = [