Add an sshfs backend to eval-under

README.md

@@ -28,6 +28,7 @@ itself is both backend- and suite-agnostic: new filesystems drop in as
 | BeeGFS 7.4.6 | [![BeeGFS 7.4.6 / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-git-annex.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-git-annex) | [![BeeGFS 7.4.6 / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-git.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-git) | [![BeeGFS 7.4.6 / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-stress-ng.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-stress-ng) | [![BeeGFS 7.4.6 / mtime stability](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-mtime-stability.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-mtime-stability) | [![BeeGFS 7.4.6 / git-annex linkAnnex loop](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-git-annex-linkannex.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-git-annex-linkannex) | [![BeeGFS 7.4.6 / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-7.4.6-pjdfstest.svg)](https://con.github.io/eval-under/#beegfs-7.4.6-pjdfstest) |
 | BeeGFS 8.1.0 | [![BeeGFS 8.1.0 / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-git-annex.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-git-annex) | [![BeeGFS 8.1.0 / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-git.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-git) | [![BeeGFS 8.1.0 / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-stress-ng.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-stress-ng) | [![BeeGFS 8.1.0 / mtime stability](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-mtime-stability.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-mtime-stability) | [![BeeGFS 8.1.0 / git-annex linkAnnex loop](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-git-annex-linkannex.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-git-annex-linkannex) | [![BeeGFS 8.1.0 / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/beegfs-8.1.0-pjdfstest.svg)](https://con.github.io/eval-under/#beegfs-8.1.0-pjdfstest) |
 | NFS (localhost) | [![NFS (localhost) / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-git-annex.svg)](https://con.github.io/eval-under/#nfs-git-annex) | [![NFS (localhost) / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-git.svg)](https://con.github.io/eval-under/#nfs-git) | [![NFS (localhost) / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-stress-ng.svg)](https://con.github.io/eval-under/#nfs-stress-ng) | [![NFS (localhost) / mtime stability](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-mtime-stability.svg)](https://con.github.io/eval-under/#nfs-mtime-stability) | [![NFS (localhost) / git-annex linkAnnex loop](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-git-annex-linkannex.svg)](https://con.github.io/eval-under/#nfs-git-annex-linkannex) | [![NFS (localhost) / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/nfs-pjdfstest.svg)](https://con.github.io/eval-under/#nfs-pjdfstest) |
+| sshfs (loopback) | [![sshfs (loopback) / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/sshfs-git-annex.svg)](https://con.github.io/eval-under/#sshfs-git-annex) | [![sshfs (loopback) / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/sshfs-git.svg)](https://con.github.io/eval-under/#sshfs-git) | n/a | [![sshfs (loopback) / mtime stability](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/sshfs-mtime-stability.svg)](https://con.github.io/eval-under/#sshfs-mtime-stability) | [![sshfs (loopback) / git-annex linkAnnex loop](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/sshfs-git-annex-linkannex.svg)](https://con.github.io/eval-under/#sshfs-git-annex-linkannex) | n/a |
 | Loop vfat | [![Loop vfat / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-git-annex.svg)](https://con.github.io/eval-under/#loop-vfat-git-annex) | [![Loop vfat / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-git.svg)](https://con.github.io/eval-under/#loop-vfat-git) | [![Loop vfat / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-stress-ng.svg)](https://con.github.io/eval-under/#loop-vfat-stress-ng) | [![Loop vfat / mtime stability](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-mtime-stability.svg)](https://con.github.io/eval-under/#loop-vfat-mtime-stability) | [![Loop vfat / git-annex linkAnnex loop](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-git-annex-linkannex.svg)](https://con.github.io/eval-under/#loop-vfat-git-annex-linkannex) | [![Loop vfat / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-vfat-pjdfstest.svg)](https://con.github.io/eval-under/#loop-vfat-pjdfstest) |
 | Loop ext4 | [![Loop ext4 / git-annex test](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-git-annex.svg)](https://con.github.io/eval-under/#loop-ext4-git-annex) | [![Loop ext4 / git testsuite](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-git.svg)](https://con.github.io/eval-under/#loop-ext4-git) | [![Loop ext4 / stress-ng](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-stress-ng.svg)](https://con.github.io/eval-under/#loop-ext4-stress-ng) | [![Loop ext4 / mtime stability](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-mtime-stability.svg)](https://con.github.io/eval-under/#loop-ext4-mtime-stability) | [![Loop ext4 / git-annex linkAnnex loop](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-git-annex-linkannex.svg)](https://con.github.io/eval-under/#loop-ext4-git-annex-linkannex) | [![Loop ext4 / pjdfstest](https://raw.githubusercontent.com/con/eval-under/gh-pages/badges/loop-ext4-pjdfstest.svg)](https://con.github.io/eval-under/#loop-ext4-pjdfstest) |
 <!-- END CI MATRIX -->
@@ -167,6 +168,14 @@ sudo bin/eval-under nfs --set-home -- bash -c 'cd "$HOME" && git annex test'
 sudo bin/eval-under nfs --set-home --mount-opts actimeo=0 -- \
   bash -c 'cd "$HOME" && git annex test'
 
+# Under sshfs, with the attribute cache off
+sudo bin/eval-under sshfs --no-cache --set-home -- \
+  bash -c 'cd "$HOME" && git annex test'
+
+# ...or against the reporter's own server, with their mount options
+sudo bin/eval-under sshfs --host store.example.org --remote-dir /data/scratch \
+  --workaround rename --set-home -- git annex fsck
+
 # Skip teardown to poke around after a failure
 sudo bin/eval-under beegfs --set-home --keep -- some-failing-command
 
@@ -210,6 +219,7 @@ into `bin/eval-under`, bumped with each release tag.
 | `bin/eval-under`                         | Dispatcher: routes to `bin/eval-under-<backend>`                                   |
 | `bin/eval-under-beegfs`                  | BeeGFS backend (containerised cluster + kernel client mount)                       |
 | `bin/eval-under-nfs`                     | NFS backend (localhost loopback export)                                            |
+| `bin/eval-under-sshfs`                   | sshfs backend (throwaway loopback sshd, or a remote you name)                       |
 | `bin/eval-under-loop`                    | Loop-device backend (dd + losetup + mkfs.<fs> + mount)                             |
 | `fixtures/beegfs/docker-compose-v7.yml`  | BeeGFS v7 test cluster (mgmtd + meta + storage), `network_mode: host`              |
 | `fixtures/beegfs/docker-compose-v8.yml`  | Same, for BeeGFS v8.x (different mgmtd command style / gRPC control plane)         |

bin/ci/dump-failure-logs.sh

@@ -50,6 +50,24 @@ case "$BACKEND" in
             | grep -iE "loop|nfs|${VERSION:-nomatch}" \
             | tail -30 || true
         ;;
+    sshfs)
+        # bin/eval-under-sshfs removes its scratch dir on teardown, so
+        # sshd.log is only still here when teardown was skipped (a crash,
+        # or --keep). Print it when it is: a mount that died mid-suite
+        # says so there and nowhere else.
+        for log in /tmp/eval-under-sshfs-*.scratch/sshd.log; do
+            [ -f "$log" ] || continue
+            echo "=== $log (last 50) ==="
+            sudo tail -50 "$log" || true
+        done
+        # A leaked mount means teardown did not finish, which is itself
+        # the finding when the suite timed out.
+        mounts="$(mount -t fuse.sshfs 2>/dev/null)"
+        echo "=== fuse.sshfs mounts still present ==="
+        echo "${mounts:-(none)}"
+        echo "=== dmesg (fuse-tagged, last 30) ==="
+        sudo dmesg 2>/dev/null | grep -iE 'fuse|sshfs' | tail -30 || true
+        ;;
     *)
         echo "unknown backend: $BACKEND" >&2
         ;;
@@ -74,10 +92,19 @@ if [ "$TARGET" = "git" ]; then
     echo "=== git testsuite failures ($results) ==="
     if [ -d "$results" ]; then
         # Pass 1: which scripts failed, and how badly.
+        #
+        # `not ok N ... # TODO known breakage` is git's test_expect_failure:
+        # a TAP TODO directive, which prove counts as an expected result and
+        # not as a failure -- a script whose only "not ok" lines are TODOs is
+        # reported ok by the harness. Counting them here inflated every git
+        # cell (the sshfs cell read 351 failed assertions where prove saw
+        # 166, and two scripts prove called ok appeared as the worst
+        # offenders at 104 and 54), which sends triage after failures that do
+        # not exist. Exclude the directive, and match prove.
         names=() counts=()
         for out in "$results"/*.out; do
             [ -e "$out" ] || continue
-            n="$(grep -c '^not ok ' "$out" 2>/dev/null || true)"
+            n="$(grep '^not ok ' "$out" 2>/dev/null | grep -vc '# TODO' || true)"
             [ "${n:-0}" -gt 0 ] || continue
             names+=("$(basename "${out%.out}")")
             counts+=("$n")
@@ -94,7 +121,7 @@ if [ "$TARGET" = "git" ]; then
                 shown=$((shown + 1))
                 out="$results/${names[$i]}.out"
                 echo "--- ${names[$i]}: ${counts[$i]} failed ---"
-                grep '^not ok ' "$out" | head -40 || true
+                grep '^not ok ' "$out" | grep -v '# TODO' | head -40 || true
                 echo "  ... last $GIT_DUMP_TAIL_LINES lines of ${names[$i]}.out:"
                 tail -"$GIT_DUMP_TAIL_LINES" "$out" | sed 's/^/  | /' || true
                 echo

bin/ci/evals.py

@@ -31,12 +31,33 @@ def load_matrix(path: Path = MATRIX_FILE) -> dict:
         return yaml.safe_load(fh)
 
 
+def cell_enabled(b: dict, t: dict) -> bool:
+    """Is this backend x target pair a cell the matrix actually defines?
+
+    The grid is deliberately not fully populated. A backend marked
+    `no-root` cannot hand the wrapped suite privilege -- a FUSE mount
+    belongs to whoever mounted it, and there is no --no-root-squash
+    equivalent the way there is for NFS -- so it has no cell for a
+    `needs-root` target: the cell could only report on privilege rather
+    than on the filesystem. bin/ci/run-under.sh refuses the same pair
+    outright. Keep in sync with cell_enabled() in bin/ci/matrix.sh.
+    """
+    return not (b.get("no-root") and t.get("needs-root"))
+
+
 def matrix_cells(m: dict) -> dict[str, dict]:
-    """slug -> cell metadata, in matrix (row, column) order."""
+    """slug -> cell metadata, in matrix (row, column) order.
+
+    Skips the pairs cell_enabled() rules out, so every consumer -- the
+    status file, the badges, the report page -- agrees on which cells
+    exist instead of publishing a permanently-unknown one.
+    """
     cells = {}
     for b in m["backends"]:
         bslug = backend_slug(b["backend"], b["version"])
         for t in m["targets"]:
+            if not cell_enabled(b, t):
+                continue
             cells[f"{bslug}-{t['name']}"] = {
                 "backend": b["backend"],
                 "version": b["version"],

bin/ci/gen-readme-matrix.sh

@@ -75,6 +75,12 @@ table_md() {
         IFS='|' read -r backend version label <<< "$cell"
         printf '| %s |' "$label"
         for target in "${EVAL_UNDER_TARGETS[@]}"; do
+            # A pair that is not a cell gets no badge: there is nothing to
+            # report, and a badge would imply a result we never measure.
+            if ! cell_enabled "$backend" "$version" "$target"; then
+                printf ' n/a |'
+                continue
+            fi
             slug="$(cell_slug "$backend" "$version" "$target")"
             printf ' [![%s / %s](%s)](%s) |' \
                 "$label" "$(target_label "$target")" \
@@ -103,4 +109,11 @@ if [ "$CHECK" = 1 ]; then
 fi
 
 cp "$new_readme" "$README"
-echo "refreshed the README CI matrix ($((${#EVAL_UNDER_BACKENDS[@]} * ${#EVAL_UNDER_TARGETS[@]})) cells)"
+cells=0
+for cell in "${EVAL_UNDER_BACKENDS[@]}"; do
+    IFS='|' read -r backend version _ <<< "$cell"
+    for target in "${EVAL_UNDER_TARGETS[@]}"; do
+        cell_enabled "$backend" "$version" "$target" && cells=$((cells + 1))
+    done
+done
+echo "refreshed the README CI matrix ($cells cells)"

bin/ci/install-backend.sh

@@ -9,17 +9,18 @@
 # usage:
 #   bin/ci/install-backend.sh <backend> <version>
 #
-#   backend  = beegfs | nfs | loop
+#   backend  = beegfs | nfs | loop | sshfs
 #   version  = for beegfs: point release (e.g. 7.4.6, 8.1.0)
 #              for loop:   filesystem type (e.g. vfat, ext4, xfs, btrfs)
 #              for nfs:    literal "n/a"
+#              for sshfs:  literal "n/a"
 #
 # Idempotent enough for CI re-runs; not a full package manager.
 
 set -euo pipefail
 export DEBIAN_FRONTEND=noninteractive
 
-BACKEND="${1:?backend required (beegfs|nfs|loop)}"
+BACKEND="${1:?backend required (beegfs|nfs|loop|sshfs)}"
 VERSION="${2:?version required (BeeGFS version | loop fs name | 'n/a' for nfs)}"
 
 # Give unattended-upgrades a moment on ubuntu-22.04 runners rather than
@@ -72,6 +73,18 @@ install_nfs() {
     command -v exportfs
 }
 
+install_sshfs() {
+    apt_update
+    # openssh-sftp-server is what actually serves the mount; on Ubuntu it
+    # is pulled in by openssh-server, but name it so a slimmer image
+    # cannot leave us without an sftp-server binary.
+    apt_install sshfs openssh-server openssh-sftp-server
+    command -v sshfs
+    # The backend starts its own sshd, so the system one need not run --
+    # but its privilege-separation directory must exist.
+    sudo mkdir -p /run/sshd
+}
+
 install_loop() {
     local pkg
     case "$VERSION" in
@@ -90,5 +103,6 @@ case "$BACKEND" in
     beegfs) install_beegfs ;;
     nfs)    install_nfs ;;
     loop)   install_loop ;;
-    *) echo "unknown backend: $BACKEND (expected beegfs|nfs|loop)" >&2; exit 1 ;;
+    sshfs)  install_sshfs ;;
+    *) echo "unknown backend: $BACKEND (expected beegfs|nfs|loop|sshfs)" >&2; exit 1 ;;
 esac

bin/ci/matrix-json.sh

@@ -42,6 +42,9 @@ entries=()
 for cell in "${EVAL_UNDER_BACKENDS[@]}"; do
     IFS='|' read -r backend version label <<< "$cell"
     for target in "${EVAL_UNDER_TARGETS[@]}"; do
+        # Not every backend x target pair is a cell -- see cell_enabled()
+        # in matrix.sh.
+        cell_enabled "$backend" "$version" "$target" || continue
         needs_ga=0
         target_needs_git_annex "$target" && needs_ga=1
         entries+=("$backend|$version|$label|$target|$(target_label "$target")|$(cell_slug "$backend" "$version" "$target")|$needs_ga")

bin/ci/matrix.sh

@@ -65,6 +65,14 @@ out.append("declare -A _EU_NEEDS_ROOT=(%s)" % " ".join(
 out.append("declare -A _EU_NEEDS_GA=(%s)" % " ".join(
     "[%s]=%s" % (q(t["name"]), q(int(bool(t["needs-git-annex"])))) for t in targets))
 
+def bslug(b):
+    return b["backend"] if b["version"] == "n/a" else "%s-%s" % (b["backend"], b["version"])
+
+# Optional per-backend flag, defaulting to false, so existing rows need
+# no edit.
+out.append("declare -A _EU_BACKEND_NO_ROOT=(%s)" % " ".join(
+    "[%s]=%s" % (q(bslug(b)), q(int(bool(b.get("no-root", False))))) for b in backends))
+
 # Env overrides win, so these are defaults only.
 out.append(": \"${EVAL_UNDER_REPO_SLUG:=%s}\"" % q(d["repo-slug"]))
 out.append(": \"${EVAL_UNDER_SRC_DIR:=%s}\"" % q(d["src-dir"]))
@@ -138,3 +146,23 @@ target_needs_git_annex() { [ "${_EU_NEEDS_GA[$1]:-0}" = 1 ]; }
 cell_output_dir() {
     echo "${EVAL_UNDER_OUTPUT_DIR:-/tmp/eval-under-output/$(cell_slug "$1" "$2" "$3")}"
 }
+
+# Can this backend hand the wrapped suite privilege at all? Takes a
+# backend *slug* (as backend_slug prints it), not a bare backend name.
+backend_no_root() { [ "${_EU_BACKEND_NO_ROOT[$1]:-0}" = 1 ]; }
+
+# Is <backend> <version> x <target> a cell the matrix actually defines?
+# The grid is deliberately not fully populated: a backend that cannot run
+# as root has no cell for a target that needs root, because such a cell
+# would measure privilege rather than the filesystem. Every consumer of
+# the matrix asks this rather than assuming backends x targets, so the
+# workflow, the README grid, the badges and the report page agree on
+# which cells exist.
+cell_enabled() {
+    local backend="$1" version="$2" target="$3"
+    if backend_no_root "$(backend_slug "$backend" "$version")" \
+       && target_needs_root "$target"; then
+        return 1
+    fi
+    return 0
+}

bin/ci/render-report.py

@@ -28,7 +28,7 @@
 from pathlib import Path
 
 import known_issues
-from evals import backend_slug, load_matrix
+from evals import backend_slug, cell_enabled, load_matrix
 
 HERE = Path(__file__).resolve().parent
 
@@ -167,8 +167,8 @@ def main() -> int:
 
     # Matrix order, so the page reads like the README grid.
     m = load_matrix()
-    backends = [(backend_slug(b["backend"], b["version"]), b["label"]) for b in m["backends"]]
-    targets = [(t["name"], t["label"]) for t in m["targets"]]
+    backends = [(backend_slug(b["backend"], b["version"]), b["label"], b) for b in m["backends"]]
+    targets = [(t["name"], t["label"], t) for t in m["targets"]]
     _, issues = known_issues.load_valid()
 
     def state(c: dict) -> str:
@@ -186,9 +186,15 @@ def state(c: dict) -> str:
                  f"{npass}/{total} passing" + (f", {nnew} unexpected" if nnew else ""))
 
     rows = []
-    for bslug, blabel in backends:
+    for bslug, blabel, bdef in backends:
         tds = [f"<th scope=row>{html.escape(blabel)}</th>"]
-        for tname, tlabel in targets:
+        for tname, tlabel, tdef in targets:
+            # A pair the matrix does not define (see cell_enabled) is an
+            # explicit gap, not an "unknown" badge: nothing was measured
+            # here and nothing ever will be.
+            if not cell_enabled(bdef, tdef):
+                tds.append('<td class=cell><span class=meta>n/a</span></td>')
+                continue
             slug = f"{bslug}-{tname}"
             c = cells.get(slug, {"conclusion": "unknown"})
             st = state(c)
@@ -214,7 +220,7 @@ def state(c: dict) -> str:
             tds.append(f'<td class=cell id="{slug}">{body}{meta}{cell_notes(c, st, fixed)}</td>')
         rows.append("<tr>" + "".join(tds) + "</tr>")
 
-    head = "".join(f"<th>{html.escape(l)}</th>" for _, l in targets)
+    head = "".join(f"<th>{html.escape(l)}</th>" for _, l, _ in targets)
     repo = m.get("repo-slug", "con/eval-under")
     doc = f"""<!doctype html>
 <html lang=en>