From 872643f793265389abd860d894e31bfef97f2232 Mon Sep 17 00:00:00 2001 From: Arpit Jain Date: Wed, 3 Jun 2026 09:35:02 +0900 Subject: [PATCH 1/3] tests: Add end-to-end test for the examples Add an integration test that runs the repository, uploader and client examples against each other, the way a user runs them from the command line: * start examples/repository/repo as a live HTTP server * run examples/uploader/uploader to TOFU, claim a delegation and upload target files * run examples/client/client to TOFU and download the uploaded targets, which verifies them against the repository metadata The test uploads a second target after the client is already initialized and then downloads it, so the client has to update from the metadata version it trusts (N) to a newer version (N+M). This covers the still-open "examples" part of issue #2228; the tuf.repository interface tests requested there landed earlier in PR #2651. The test is hermetic: it points HOME at a temporary directory so the client and uploader trust stores do not touch the real home directory, runs the server on a free port, and polls for readiness instead of sleeping. Signed-off-by: Arpit Jain --- tests/test_examples_e2e.py | 232 +++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 tests/test_examples_e2e.py diff --git a/tests/test_examples_e2e.py b/tests/test_examples_e2e.py new file mode 100644 index 0000000000..3782918bf2 --- /dev/null +++ b/tests/test_examples_e2e.py @@ -0,0 +1,232 @@ +# Copyright 2026, the TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""End-to-end test that runs the repository, uploader and client examples. + +Unlike test_examples.py (which execs the self-contained 'manual_repo' +scripts), this test wires the three interacting examples together the way +a user would run them from the command line: + +* 'examples/repository/repo' is started as a live HTTP server, +* 'examples/uploader/uploader' performs Trust-On-First-Use, claims a + delegation and uploads target files to that server, +* 'examples/client/client' performs Trust-On-First-Use and downloads the + uploaded targets, which verifies them against the repository metadata. + +The test also uploads a second target after the client has already been +initialized, then downloads it: this exercises the case from issue #2228 +where a client with metadata version N must update to version N+M. +""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from typing import ClassVar +from urllib.parse import quote + +import urllib3 + +from tests import utils + +# The repository example serves metadata under this URL prefix. +_METADATA_URL = "metadata" + + +def _get_free_port() -> int: + """Return a currently-free localhost TCP port. + + The port is handed to the example repository server via '-p'. There is a + small race between closing this socket and the server binding the port, + but the readiness poll in setUp() tolerates it. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((utils.TEST_HOST_ADDRESS, 0)) + return int(sock.getsockname()[1]) + + +class TestExamplesEnd2End(unittest.TestCase): + """Run the repository, uploader and client examples against each other.""" + + examples_dir: ClassVar[Path] + + @classmethod + def setUpClass(cls) -> None: + cls.examples_dir = Path(__file__).resolve().parents[1] / "examples" + + def setUp(self) -> None: + # The examples import their private helper modules by name (e.g. + # "from _simplerepo import SimpleRepository"), so each example dir + # must be importable. The client and uploader also write trusted + # metadata and keys under Path.home(): point HOME at a throwaway dir + # so the test does not touch the real home directory and starts from + # a clean slate every run. + self.test_dir = tempfile.mkdtemp() + self.addCleanup(self._rmtree, self.test_dir) + + self.env = os.environ.copy() + self.env["HOME"] = self.test_dir + self.env["PYTHONPATH"] = os.pathsep.join( + str(self.examples_dir / name) + for name in ("repository", "uploader", "client") + ) + + self.port = _get_free_port() + self.base_url = f"http://{utils.TEST_HOST_ADDRESS}:{self.port}" + + # Start the repository example as a live server subprocess. + repo_script = self.examples_dir / "repository" / "repo" + self.server = subprocess.Popen( + [sys.executable, "-u", str(repo_script), "-p", str(self.port)], + env=self.env, + cwd=self.test_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.addCleanup(self._stop_server) + + utils.wait_for_server( + utils.TEST_HOST_ADDRESS, "repository example", self.port + ) + # wait_for_server only confirms the port is open; also wait until the + # repository has published its initial root metadata. + self._wait_for_root() + + def _rmtree(self, path: str) -> None: + # shutil.rmtree wrapper registered with addCleanup. + shutil.rmtree(path, ignore_errors=True) + + def _stop_server(self) -> None: + self.server.terminate() + try: + self.server.wait(timeout=5) + except subprocess.TimeoutExpired: + self.server.kill() + self.server.wait() + if self.server.stdout is not None: + self.server.stdout.close() + + def _wait_for_root(self, timeout: int = 10) -> None: + """Poll until the server serves 1.root.json (or time out).""" + deadline = time.monotonic() + timeout + url = f"{self.base_url}/{_METADATA_URL}/1.root.json" + last_error: Exception | None = None + while time.monotonic() < deadline: + if self.server.poll() is not None: + self.fail( + "repository example exited early with code " + f"{self.server.returncode}:\n{self._server_output()}" + ) + try: + response = urllib3.request("GET", url) + if response.status == 200: + return + except urllib3.exceptions.HTTPError as e: + last_error = e + time.sleep(0.05) + self.fail( + f"repository example never served root metadata: {last_error}" + ) + + def _server_output(self) -> str: + if self.server.stdout is None: + return "" + return self.server.stdout.read().decode("utf-8", errors="replace") + + def _run_example(self, script: str, *args: str) -> None: + """Run an example script as a subprocess and assert it succeeds.""" + script_path = self.examples_dir / script + result = subprocess.run( + [sys.executable, str(script_path), "-u", self.base_url, *args], + env=self.env, + cwd=self.test_dir, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + self.assertEqual( + result.returncode, + 0, + f"{script} {' '.join(args)} failed (rc={result.returncode}):\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}", + ) + + def _snapshot_version(self) -> int: + """Return the version of the repository's current snapshot metadata.""" + url = f"{self.base_url}/{_METADATA_URL}/snapshot.json" + response = urllib3.request("GET", url) + self.assertEqual(response.status, 200) + return int(json.loads(response.data)["signed"]["version"]) + + def _downloaded_target(self, targetpath: str) -> bytes: + """Read a target the client downloaded into its downloads dir. + + The client writes targets into a 'downloads' dir at its CWD, using the + URL-quoted targetpath as filename (see Updater file path generation). + """ + path = Path(self.test_dir) / "downloads" / quote(targetpath, "") + self.assertTrue(path.exists(), f"missing downloaded target {path}") + return path.read_bytes() + + def test_repository_uploader_client(self) -> None: + """Upload targets via the uploader and download them via the client. + + Covers the end-to-end "examples" slice of issue #2228: a client can + download a target and can update across metadata versions (N -> N+M). + """ + uploader = "uploader/uploader" + client = "client/client" + role = "myrole" + first_target = f"{role}/first" + second_target = f"{role}/second" + + # Maintainer side: trust the repo, claim a delegation, add a target. + self._run_example(uploader, "tofu") + self._run_example(uploader, "add-delegation", role) + self._run_example(uploader, "add-target", role, first_target) + + version_after_first = self._snapshot_version() + + # Client side: trust the repo and download the first target. The + # Updater verifies the target against the (signed) repository metadata + # before download_target() returns, so a successful download means the + # metadata chain and the target hash both verified. + self._run_example(client, "tofu") + self._run_example(client, "download", first_target) + + # The example generates target content equal to the targetpath. + self.assertEqual( + self._downloaded_target(first_target), + first_target.encode(), + ) + + # Maintainer adds a second target. This publishes new targets, + # snapshot and timestamp metadata, so the repository advances past the + # version the client currently trusts. + self._run_example(uploader, "add-target", role, second_target) + version_after_second = self._snapshot_version() + self.assertGreater(version_after_second, version_after_first) + + # Client downloads the second target. download() refreshes metadata + # first, so the already-initialized client must update from the + # version it trusts (N) to the newer version (N+M) to find and verify + # the new target. + self._run_example(client, "download", second_target) + self.assertEqual( + self._downloaded_target(second_target), + second_target.encode(), + ) + + +if __name__ == "__main__": + utils.configure_test_logging(sys.argv) + unittest.main() From 1d2994162888b796251b90cd4f8d20379916de91 Mon Sep 17 00:00:00 2001 From: Arpit Jain Date: Thu, 4 Jun 2026 07:23:21 +0900 Subject: [PATCH 2/3] tests: retry examples server start on lost port race The examples e2e test picks a free port with _get_free_port(), closes the socket, then starts the repository example which binds that port. If another process grabs the port in that window, the example exits immediately with OSError: Address already in use. The previous code then waited the full readiness timeout and reported a misleading 'Could not connect to the repository example on port ...' error, which is the flake seen in CI. Start the server inside a retry loop: detect when the subprocess exits before serving root metadata, and if the exit was a port bind failure, retry on a fresh port. On a genuine readiness failure the test now fails fast and prints the server's captured output instead of a bare connection timeout. _wait_for_root() now returns a bool so the caller can decide whether to retry. Signed-off-by: Arpit Jain --- tests/test_examples_e2e.py | 117 ++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/tests/test_examples_e2e.py b/tests/test_examples_e2e.py index 3782918bf2..4c4df7265c 100644 --- a/tests/test_examples_e2e.py +++ b/tests/test_examples_e2e.py @@ -41,12 +41,18 @@ _METADATA_URL = "metadata" +# Number of times to retry starting the server on a fresh port. A retry is +# only needed if another process grabs the port in the small window between +# _get_free_port() releasing it and the server binding it, which is rare. +_MAX_SERVER_STARTS = 5 + + def _get_free_port() -> int: """Return a currently-free localhost TCP port. The port is handed to the example repository server via '-p'. There is a - small race between closing this socket and the server binding the port, - but the readiness poll in setUp() tolerates it. + small race between closing this socket and the server binding the port; + _start_server() retries on a fresh port if the server loses that race. """ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind((utils.TEST_HOST_ADDRESS, 0)) @@ -58,6 +64,9 @@ class TestExamplesEnd2End(unittest.TestCase): examples_dir: ClassVar[Path] + server: subprocess.Popen[bytes] + _stopped_output: str | None + @classmethod def setUpClass(cls) -> None: cls.examples_dir = Path(__file__).resolve().parents[1] / "examples" @@ -79,26 +88,57 @@ def setUp(self) -> None: for name in ("repository", "uploader", "client") ) - self.port = _get_free_port() - self.base_url = f"http://{utils.TEST_HOST_ADDRESS}:{self.port}" + self._stopped_output = None + self._start_server() - # Start the repository example as a live server subprocess. - repo_script = self.examples_dir / "repository" / "repo" - self.server = subprocess.Popen( - [sys.executable, "-u", str(repo_script), "-p", str(self.port)], - env=self.env, - cwd=self.test_dir, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - self.addCleanup(self._stop_server) + def _start_server(self) -> None: + """Start the repository example and wait until it serves metadata. - utils.wait_for_server( - utils.TEST_HOST_ADDRESS, "repository example", self.port - ) - # wait_for_server only confirms the port is open; also wait until the - # repository has published its initial root metadata. - self._wait_for_root() + The server is started as a subprocess on a free port. If it loses the + race for that port (another process binds it first) the subprocess + exits with "Address already in use"; in that case we pick a new port + and try again rather than reporting a misleading connection timeout. + """ + repo_script = self.examples_dir / "repository" / "repo" + for attempt in range(_MAX_SERVER_STARTS): + self.port = _get_free_port() + self.base_url = f"http://{utils.TEST_HOST_ADDRESS}:{self.port}" + + self._stopped_output = None + self.server = subprocess.Popen( + [sys.executable, "-u", str(repo_script), "-p", str(self.port)], + env=self.env, + cwd=self.test_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + if self._wait_for_root(): + # Tear the server down at the end of the test. + self.addCleanup(self._stop_server) + return + + # The server never served root metadata. Stop it (which drains its + # output) before deciding whether to retry. + exited_early = self.server.poll() is not None + self._stop_server() + output = self._server_output() + + # A lost race for the port shows up as a bind failure: retry on a + # fresh port rather than reporting a misleading connection timeout. + if ( + exited_early + and "Address already in use" in output + and attempt + 1 < _MAX_SERVER_STARTS + ): + continue + + reason = ( + f"exited early with code {self.server.returncode}" + if exited_early + else "did not serve root metadata in time" + ) + self.fail(f"repository example {reason}:\n{output}") def _rmtree(self, path: str) -> None: # shutil.rmtree wrapper registered with addCleanup. @@ -111,32 +151,43 @@ def _stop_server(self) -> None: except subprocess.TimeoutExpired: self.server.kill() self.server.wait() + # Stash any remaining output before closing the pipe so it stays + # available to _server_output() after the server has been stopped. if self.server.stdout is not None: + self._stopped_output = self.server.stdout.read().decode( + "utf-8", errors="replace" + ) self.server.stdout.close() - def _wait_for_root(self, timeout: int = 10) -> None: - """Poll until the server serves 1.root.json (or time out).""" + def _wait_for_root(self, timeout: int = 10) -> bool: + """Poll until the server serves 1.root.json. + + Returns True once the server answers with the initial root metadata. + Returns False if the server process exits before then (so the caller + can retry on a new port) or if the timeout is reached without the + server ever exiting (a genuine readiness failure). + """ deadline = time.monotonic() + timeout url = f"{self.base_url}/{_METADATA_URL}/1.root.json" - last_error: Exception | None = None while time.monotonic() < deadline: if self.server.poll() is not None: - self.fail( - "repository example exited early with code " - f"{self.server.returncode}:\n{self._server_output()}" - ) + # Server exited; let the caller inspect its output. + return False try: response = urllib3.request("GET", url) if response.status == 200: - return - except urllib3.exceptions.HTTPError as e: - last_error = e + return True + except urllib3.exceptions.HTTPError: + # Connection refused/reset while the server is still starting. + pass time.sleep(0.05) - self.fail( - f"repository example never served root metadata: {last_error}" - ) + return False def _server_output(self) -> str: + # If the server has been stopped, _stop_server() already drained the + # pipe; otherwise read whatever it has produced so far. + if self._stopped_output is not None: + return self._stopped_output if self.server.stdout is None: return "" return self.server.stdout.read().decode("utf-8", errors="replace") From 82a8f4b20d82fc7d0781d376d9ed38490c2d8759 Mon Sep 17 00:00:00 2001 From: Arpit Jain Date: Sun, 6 Sep 2026 08:10:34 -0400 Subject: [PATCH 3/3] tests: keep the existing PYTHONPATH when launching the examples setUp replaced PYTHONPATH with just the three example directories. Where the test runner uses PYTHONPATH to make tuf importable, that left the repository subprocess unable to import tuf, and the failure surfaced as the opaque 'repository example exited early with code 1' with ModuleNotFoundError: No module named 'tuf' in the captured output. The example directories are now prepended to any existing PYTHONPATH instead of replacing it. Reproduced by uninstalling tuf and putting the checkout on PYTHONPATH: the test fails with that exact error before the change and passes after. Signed-off-by: Arpit Jain --- tests/test_examples_e2e.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_examples_e2e.py b/tests/test_examples_e2e.py index 4c4df7265c..961900d524 100644 --- a/tests/test_examples_e2e.py +++ b/tests/test_examples_e2e.py @@ -83,10 +83,18 @@ def setUp(self) -> None: self.env = os.environ.copy() self.env["HOME"] = self.test_dir - self.env["PYTHONPATH"] = os.pathsep.join( + # The example scripts import their sibling modules, so the example + # directories go on PYTHONPATH. Prepend rather than replace: the test + # runner may already be using PYTHONPATH to make tuf itself importable, + # and dropping it leaves the subprocess unable to import tuf at all. + example_paths = [ str(self.examples_dir / name) for name in ("repository", "uploader", "client") - ) + ] + existing_pythonpath = self.env.get("PYTHONPATH") + if existing_pythonpath: + example_paths.append(existing_pythonpath) + self.env["PYTHONPATH"] = os.pathsep.join(example_paths) self._stopped_output = None self._start_server()