install.sh and openRepoShape: a $OPENREPOSHAPE_BIN_DIR the bytes refuse is read again in the locale's own charset, so a GB18030, GBK or Shift_JIS name that holds no control character and nothing drawn as another is placed, and where nothing can read it the bytes decide as before

install.sh

@@ -399,9 +399,34 @@ misdrawn_characters() {
 	printf '%s' "$found"
 }
 
+# THE VALUE AS THE PERSON'S LOCALE READS IT, IN UTF-8, WHERE THAT LOCALE
+# READS TWO OR MORE OF ITS BYTES AS ONE CHARACTER (#858): the launcher's
+# `locale_reading`, copied as it is, for the reason the control check in
+# `main` is copied. The launcher says when it reads the value again and
+# when the bytes decide, and `tests/test_openreposhape_command.py` holds the
+# two copies to one text. `main` asks it of a directory the bytes refuse.
+locale_reading() {
+	local charmap reading
+	charmap="$(locale charmap 2>/dev/null)" || return 1
+	case "$charmap" in
+	"" | [Uu][Tt][Ff]-8 | [Uu][Tt][Ff]8 | ANSI_X3.4-1968 | *ASCII*) return 1 ;;
+	*) ;;
+	esac
+	reading="$(printf '%s' "$1" | iconv -f "$charmap" -t UTF-8 2>/dev/null)" ||
+		return 1
+	[ "$(printf '%s' "$reading" | iconv -f UTF-8 -t "$charmap" 2>/dev/null)" = "$1" ] ||
+		return 1
+	(
+		LC_ALL=C
+		letters=${reading//[$'\200'-$'\277']/}
+		[ "${#letters}" -lt "${#1}" ] || exit 1
+		printf '%s' "$reading"
+	)
+}
+
 main() {
 	local version="" asked launcher_url bin_dir spelled target shown status=0 pinned=1
-	local misdrawn
+	local misdrawn seen
 	local machine="this machine is ready"
 
 	# NOTHING A CHILD READS IS THIS SCRIPT. Piped, stdin is the pipe the
@@ -446,23 +471,42 @@ main() {
 	# print (`café` under LC_ALL=C) is no fault of the directory's. The check
 	# runs under `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`,
 	# then `\200` to `\237`), so a C1 control is refused in every locale,
-	# whatever else the value holds; U+2028 and U+2029, which only a UTF-8
-	# locale's `[[:cntrl:]]` matched, are the next check's (#837).
+	# whatever else the value holds, unless the locale reads those bytes as
+	# text (#858, below); U+2028 and U+2029, which only a UTF-8 locale's
+	# `[[:cntrl:]]` matched, are the next check's (#837).
 	# Left to the locale, `[[:cntrl:]]` matches a C1 control only in a value
 	# that is valid UTF-8: one byte that is not (a lone 0xE9) makes bash
 	# match byte by byte, and then U+009B's two bytes are not controls, as
 	# under `C` (measured with bash 5.2.21). The locale is `C` in a subshell,
 	# so that the match is byte by byte whatever a libc says of a byte in a
 	# UTF-8 locale, and the locale of every line below is as it was.
-	if (LC_ALL=C; case "$bin_dir" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then
+	# WHERE THE BYTES ARE REFUSED, THE PERSON'S LOCALE IS ASKED HOW IT READS
+	# THEM (#858), as the command asks it: a multibyte charset that is not
+	# UTF-8 may read them as text, and then the C1 bytes and
+	# `misdrawn_characters` are looked for in `locale_reading`'s UTF-8
+	# instead. Only a value the bytes refuse is read again, so no name
+	# placed before is refused, and where nothing reads it again the bytes
+	# decide, as before. `[[:cntrl:]]` is asked of the bytes as typed. The
+	# trade: such a locale, left in a terminal that reads UTF-8, has C1
+	# bytes it reads as letters installed into and printed in the lines
+	# below, and that terminal acts on them.
+	seen="$bin_dir"
+	if (LC_ALL=C; case "$bin_dir" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac) ||
+		[ -n "$(LC_ALL=C; misdrawn_characters "$bin_dir")" ]; then
+		seen="$(locale_reading "$bin_dir")" || seen="$bin_dir"
+	fi
+	if (LC_ALL=C
+		case "$bin_dir" in *[[:cntrl:]]*) exit 0 ;; *) ;; esac
+		case "$seen" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then
 		die "the directory to install into holds a control character, such as a newline, a carriage return, a tab or an escape, and no line about \$PATH can print such a name as it is: a terminal acts on it. Nothing was installed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this line again."
 	fi
 	# A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED
 	# NEXT (#837), as the command refuses it, and before any request: the
 	# name would read as another directory's in every line below. The
 	# refusal names each one by its code point, because nobody can see it,
-	# and says nothing else of the value.
-	misdrawn="$(LC_ALL=C; misdrawn_characters "$bin_dir")"
+	# and says nothing else of the value. It is asked of the locale's
+	# reading where there is one (#858).
+	misdrawn="$(LC_ALL=C; misdrawn_characters "$seen")"
 	[ -z "$misdrawn" ] ||
 		die "the directory to install into holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it, so no line about \$PATH can print the name as it reads. Nothing was installed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this line again."
 	# A WINDOWS SPELLING IS SPELLED THE WAY BASH SPELLS IT, THE COMMAND'S OWN

openRepoShape

@@ -415,7 +415,10 @@ record_ref() {
 # character (a newline breaks the printed line in two, an escape is the
 # terminal's to act on) is no name a line can print as it is, and nor is a
 # character a terminal draws as something other than itself (#837), so
-# `install_commands` refuses all three before this is asked.
+# `install_commands` refuses all three before this is asked. The C1
+# controls and those characters are refused as the person's locale reads
+# the name (#858): a multibyte locale that is not UTF-8 can hand this line
+# C1 bytes it reads as letters, which a terminal that reads UTF-8 acts on.
 export_path_line() {
 	local dir="$1" backslash='\' quote="'" rest quoted=""
 	case "$dir" in
@@ -537,10 +540,60 @@ misdrawn_characters() {
 	printf '%s' "$found"
 }
 
+# THE VALUE AS THE PERSON'S LOCALE READS IT, IN UTF-8, WHERE THAT LOCALE
+# READS TWO OR MORE OF ITS BYTES AS ONE CHARACTER (#858). The control check
+# and `misdrawn_characters` look for UTF-8's bytes under `LC_ALL=C`, and a
+# multibyte charset that is not UTF-8 reads some of those bytes as text:
+# GBK, GB18030 and CP936 read `0xC2 0x80` to `0xC2 0x9F` as 32 hanzi; GBK,
+# Big5, EUC-JP and EUC-KR read U+00AD's two bytes as a hanzi, a kanji or a
+# Hangul syllable; and in Shift_JIS a kanji whose second byte is `0xC2`,
+# before one of the common kanji (first byte `0x88` to `0x98`), puts a C1
+# control's two bytes side by side (measured with Python's codecs).
+# `install_commands` asks this of a value those checks refuse, and asks them
+# again of what it prints. It prints the reading, and succeeds, only where:
+#   * `locale charmap`, asked under the person's own `LC_ALL`, `LC_CTYPE`
+#     and `LANG`, answers with a charset that is neither UTF-8, whose
+#     reading is the bytes, nor ASCII, which reads no byte past 0x7F;
+#   * `iconv` reads the whole value in that charset;
+#   * the reading, written back into that charset by `iconv`, is the value
+#     again, byte for byte, so an `iconv` that drops or changes a byte, or
+#     prints nothing at all and exits 0, gives no reading; and
+#   * the reading has fewer characters than the value has bytes, counted
+#     under `LC_ALL=C` as the bytes of UTF-8 that start a character (0x80
+#     to 0xBF continue one). A single-byte charset reads each byte alone,
+#     and there the bytes decide, as before: under a Latin-1 `LANG` left in
+#     a terminal that reads UTF-8, U+FEFF's three bytes would read as three
+#     letters.
+# Otherwise it prints nothing and fails, and the bytes decide: no `locale`
+# or no `iconv` (a minimal container), a charset either does not know, or
+# a value the charset cannot read. The locale is taken at its word, and
+# that is the trade: a multibyte locale that is not UTF-8, left in a
+# terminal that reads UTF-8, is handed the C1 bytes and the bytes of
+# `MISDRAWN_CHARACTERS` it reads as letters, and that terminal acts on
+# them. install.sh carries this function as it is here.
+locale_reading() {
+	local charmap reading
+	charmap="$(locale charmap 2>/dev/null)" || return 1
+	case "$charmap" in
+	"" | [Uu][Tt][Ff]-8 | [Uu][Tt][Ff]8 | ANSI_X3.4-1968 | *ASCII*) return 1 ;;
+	*) ;;
+	esac
+	reading="$(printf '%s' "$1" | iconv -f "$charmap" -t UTF-8 2>/dev/null)" ||
+		return 1
+	[ "$(printf '%s' "$reading" | iconv -f UTF-8 -t "$charmap" 2>/dev/null)" = "$1" ] ||
+		return 1
+	(
+		LC_ALL=C
+		letters=${reading//[$'\200'-$'\277']/}
+		[ "${#letters}" -lt "${#1}" ] || exit 1
+		printf '%s' "$reading"
+	)
+}
+
 install_commands() {
 	local dir="${OPENREPOSHAPE_BIN_DIR:-$HOME/.local/bin}"
 	local name target source verb placed=0 recording=0 record spelled=""
-	local misdrawn
+	local misdrawn seen
 	# NO LINE ABOUT `$PATH` CAN NAME A DIRECTORY HOLDING A CONTROL CHARACTER
 	# OR A `:` (#667, #760): the colon is what splits a `$PATH`; a newline
 	# breaks the line the note prints in two; a carriage return rewrites the
@@ -555,25 +608,51 @@ install_commands() {
 	# under LC_ALL=C, is no fault of the directory's. The check runs under
 	# `LC_ALL=C` and names U+0080 to U+009F's two bytes (`\302`, then `\200`
 	# to `\237`), so a C1 control is refused in every locale, whatever else
-	# the value holds; U+2028 and U+2029, which only a UTF-8 locale's
-	# `[[:cntrl:]]` matched, are the next check's (#837). Left to the
-	# locale, `[[:cntrl:]]` matches a C1 control only in a value that is
+	# the value holds, unless the locale reads those bytes as text (#858,
+	# below); U+2028 and U+2029, which only a UTF-8 locale's `[[:cntrl:]]`
+	# matched, are the next check's (#837). Left to the locale,
+	# `[[:cntrl:]]` matches a C1 control only in a value that is
 	# valid UTF-8: one byte that is not (a lone 0xE9) makes bash match byte
 	# by byte, and then U+009B's two bytes are not controls, as under `C`
 	# (measured with bash 5.2.21). The locale is `C` in a subshell, so that
 	# the match is byte by byte whatever a libc says of a byte in a UTF-8
 	# locale, and the locale of every line below is as it was. The colon is
 	# checked after the Windows spelling, which holds one by definition, is
 	# respelled without one, and refuses what it cannot respell.
-	if (LC_ALL=C; case "$dir" in *[[:cntrl:]]* | *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then
+	# WHERE THE BYTES ARE REFUSED, THE PERSON'S LOCALE IS ASKED HOW IT READS
+	# THEM (#858): a multibyte charset that is not UTF-8 may read them as
+	# text, and then the C1 bytes and `misdrawn_characters` are looked for
+	# in `locale_reading`'s UTF-8 instead. Only a value the bytes refuse is
+	# read again, so no name placed before is refused, and where nothing
+	# reads it again the bytes decide, as before. `[[:cntrl:]]` is asked of
+	# the bytes as typed: those characters are ASCII's in every such
+	# charset, and `$(…)` drops a newline at the end of the reading. The
+	# trade: such a locale, left in a terminal that reads UTF-8, has C1 bytes
+	# it reads as letters placed and printed in the lines below, and that
+	# terminal acts on them. A lone C1 byte that is not UTF-8 (`0x9B`) is
+	# placed, as before: a terminal that reads UTF-8 draws it as a character
+	# it cannot read, and only one set to an 8-bit charset acts on it. Under
+	# an ISO 8859 charmap, the one such a terminal goes with, the charset is
+	# known and reads that byte as U+009B, but nothing here reads the name
+	# again, so `a`, `0x9B`, `b` is placed there too: a candidate, left
+	# (#858).
+	seen="$dir"
+	if (LC_ALL=C; case "$dir" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac) ||
+		[ -n "$(LC_ALL=C; misdrawn_characters "$dir")" ]; then
+		seen="$(locale_reading "$dir")" || seen="$dir"
+	fi
+	if (LC_ALL=C
+		case "$dir" in *[[:cntrl:]]*) exit 0 ;; *) ;; esac
+		case "$seen" in *$'\302'[$'\200'-$'\237']*) exit 0 ;; *) exit 1 ;; esac); then
 		die "--install would place this command in a directory whose name holds a control character, such as a newline, a carriage return, a tab or an escape, and no line about \$PATH can print such a name as it is: a terminal acts on it. Nothing was placed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this again."
 	fi
 	# A CHARACTER A TERMINAL DRAWS AS SOMETHING OTHER THAN ITSELF IS REFUSED
 	# NEXT (#837), for the control characters' reason and before the same
 	# lines: the name would read as another directory's. The refusal names
 	# each one by its code point, because nobody can see it, and says
-	# nothing else of the value. `misdrawn_characters` says which they are.
-	misdrawn="$(LC_ALL=C; misdrawn_characters "$dir")"
+	# nothing else of the value. `misdrawn_characters` says which they are,
+	# in the locale's reading where there is one (#858).
+	misdrawn="$(LC_ALL=C; misdrawn_characters "$seen")"
 	[ -z "$misdrawn" ] ||
 		die "--install would place this command in a directory whose name holds a character a terminal draws as something other than itself ($misdrawn): one that is invisible, is drawn as another character, ends a line or reorders the text around it, so no line about \$PATH can print the name as it reads. Nothing was placed. Set \$OPENREPOSHAPE_BIN_DIR to a directory without one, then run this again."
 	# A WINDOWS SPELLING IS SPELLED AGAIN THE WAY BASH SPELLS IT (#649), with

tests/test_install_sh.py

@@ -53,13 +53,19 @@
                                         LOCALES, NO_FILE_NAMED_SO,
                                         NO_NON_ASCII_INTO_GIT_BASH,
                                         NOT_CONTROLS,
+                                        HONEST, MARKS,
+                                        PLACED_AS_THE_LOCALE_READS,
+                                        REFUSED_AS_THE_BYTES_SAY,
+                                        REFUSED_UNDER_C_OR_UTF_8,
                                         REFUSED_IN_EVERY_LOCALE, RLO,
                                         UNNAMEABLE_BIN_DIRS,
                                         assert_a_bin_dir_is_refused,
                                         assert_refused_saying,
                                         drawn_as_another,
                                         command_env, machine_path,
-                                        put_cygpath, put_on_path, run_cmd,
+                                        put_a_locale, put_cygpath,
+                                        put_on_path, run_cmd,
+                                        the_persons_locale,
                                         typed_bin_dir, write_script)
 
 INSTALLER = REPO / "install.sh"
@@ -974,6 +980,67 @@ def test_the_installer_refuses_in_every_locale_before_any_request(
     net.refused(result)
 
 
+@NO_NON_ASCII_INTO_GIT_BASH
+@pytest.mark.parametrize("charmap, iconv, name", PLACED_AS_THE_LOCALE_READS)
+def test_the_installer_places_a_directory_a_multibyte_locale_reads_as_text(
+        tmp_path, charmap, iconv, name):
+    """A name the bytes refuse, read as text by the locale's multibyte
+    charset (#858): install.sh asks `locale charmap` under the person's own
+    `LANG` and reads the name through `iconv`, as the command does, and the
+    command it fetched places itself there, under the same `locale` and
+    `iconv`, with the release recorded. Neither refuses what the other
+    places, and neither asks `iconv` of a name the bytes do not refuse."""
+    net = Net(tmp_path)
+    put_a_locale(net.path, HONEST, iconv)
+    net.bin = tmp_path / name
+    result = net.run(env={"OPENREPOSHAPE_BIN_DIR": bash_path(net.bin),
+                          **the_persons_locale(charmap)})
+    assert result.returncode == 0, result.stdout + result.stderr
+    net.installed(result)
+    assert not (net.path / "iconv-ran").exists()
+
+
+@NO_NON_ASCII_INTO_GIT_BASH
+@pytest.mark.parametrize("charmap, kind, iconv, name, said",
+                         REFUSED_AS_THE_BYTES_SAY)
+def test_the_installer_refuses_as_the_bytes_or_the_reading_say(
+        tmp_path, charmap, kind, iconv, name, said):
+    """Where the locale's reading still holds a C1 control or a character
+    drawn as another, or where nothing reads the name again (#858),
+    install.sh refuses it in #804's or #837's words, before any request:
+    exit 2, nothing placed, nothing of the value in either stream. With no
+    `locale` on this `$PATH` at all, every other test of the directory here
+    runs that way too."""
+    net = Net(tmp_path)
+    put_a_locale(net.path, kind, iconv)
+    typed = f"{bash_path(tmp_path)}/{name}"
+    result = net.run(env={"OPENREPOSHAPE_BIN_DIR": typed,
+                          **the_persons_locale(charmap)})
+    assert net.calls() == [], net.calls()
+    assert_a_bin_dir_is_refused(result, typed, said, INSTALLER_REFUSAL_ENDS)
+    net.refused(result)
+    assert not (net.path / "iconv-ran").exists()
+
+
+@NO_NON_ASCII_INTO_GIT_BASH
+@pytest.mark.parametrize("locale", LOCALES)
+@pytest.mark.parametrize("name, said", REFUSED_UNDER_C_OR_UTF_8)
+def test_the_installer_refuses_as_the_bytes_say_under_c_or_c_utf_8(
+        tmp_path, name, said, locale):
+    """The person's locale is `C` or `C.UTF-8`, and `locale charmap`
+    answers as glibc's does: install.sh refuses as the bytes say, in #804's
+    and #837's words, before any request, and asks `iconv` nothing
+    (#858)."""
+    net = Net(tmp_path)
+    put_a_locale(net.path, HONEST, MARKS)
+    typed = f"{bash_path(tmp_path)}/{name}"
+    result = net.run(env={"OPENREPOSHAPE_BIN_DIR": typed, "LC_ALL": locale})
+    assert net.calls() == [], net.calls()
+    assert_a_bin_dir_is_refused(result, typed, said, INSTALLER_REFUSAL_ENDS)
+    net.refused(result)
+    assert not (net.path / "iconv-ran").exists()
+
+
 # A version, an option, the repository, the newest release's tag or the
 # location github.com redirects to, holding U+202E (#837). Each arranges
 # `net` and returns the arguments, the environment, what the refusal says,