From 930deb27464c940a7c8e8bf80432bab2b0a21604 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 7 Nov 2026 16:59:14 +0100 Subject: [PATCH] image: Write complete UAPI.16 sequences The manifest uses the wrong timestamp field and lacks the trailer that marks a complete sequence. Let's use mTimeNSec and require the trailer in readers and writers. Let's also sort the keys of each record so the bytes don't depend on insertion order, and fail on unnormalized names and on mtimes before 1970, which the format can't represent. Finally, let's move the format out of the image driver into a uapi16 package of its own. Other packages can then read and write manifests without depending on image/. Signed-off-by: Daan De Meyer --- docs/design/architecture.md | 2 + examples/image-secureboot/BUCK | 16 +- examples/image/expected.json | 12 +- examples/image/manifest_check.py | 3 +- image/BUCK | 17 +- image/ddi.py | 6 +- image/manifest.py | 275 +--------------- ruff.toml | 1 + uapi16/BUCK | 20 ++ uapi16/pyproject.toml | 4 + uapi16/uapi16.py | 302 ++++++++++++++++++ .../manifest_test.py => uapi16/uapi16_test.py | 95 ++++-- 12 files changed, 421 insertions(+), 332 deletions(-) create mode 100644 uapi16/BUCK create mode 100644 uapi16/pyproject.toml create mode 100644 uapi16/uapi16.py rename image/manifest_test.py => uapi16/uapi16_test.py (81%) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index c7a4e71c..fcbae251 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -53,6 +53,8 @@ tine//box/ box bootstrap and sandbox command construction tine//rootfs/ bind/overlay mounting and stored-delta translation tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and drivers, and VM runners +tine//uapi16/ UAPI.16 manifests +tine//archive/ tar writer, cpio reader and writer tine//git/ pinned Git repositories with local checkout overrides tine//cargo/ vendored crate trees and offline Rust source builds tine//go/ go.sum-verified module fetches and offline Go source builds diff --git a/examples/image-secureboot/BUCK b/examples/image-secureboot/BUCK index 17266ad2..a36ea053 100644 --- a/examples/image-secureboot/BUCK +++ b/examples/image-secureboot/BUCK @@ -314,13 +314,15 @@ box.sh_test( name = "disk-manifest-test", labels = ["image"], box = "@tine//tools:dev.box", - test = "@tine//tests:contains.sh", - args = [ - "$(location :image[disk-manifest])", - '"name":"usr/bin/bash"', - '"name":"boot/tine-tmpfiles","type":"dir","mode":448', - '"name":"boot/EFI/Linux', - ], + args = ["$(location :image[disk-manifest])"], + script = """ +# Authored tmpfiles must set the boot directory mode in the disk's filesystem listing. +jq --seq -es ' + any(.[]; .name == "usr/bin/bash") and + any(.[]; .name == "boot/tine-tmpfiles" and .type == "dir" and .mode == 448) and + any(.[]; .name | strings | startswith("boot/EFI/Linux")) +' "$1" +""", ) # The kernel install hook is bypassed, it didn't generate its own initrd. diff --git a/examples/image/expected.json b/examples/image/expected.json index a73b5a37..56c4e769 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -16,13 +16,13 @@ "sha256": "64b78fbe92ff5fcaefb77ef10923c964e402304a54b0a776a8ca11c9b43c85f7" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "c95655cea99dc0f17d5d4c6211595a8ba18728f94c3513a1695220de04c14e89" + "sha256": "b4f4144aed4ede4dec269912df3de2f1b93d8259eb293c8558822f8853e8f10f" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "f81e7f963bbe9cfaf07ac4495a0d8e47392206e9ddb561943304bbb0ae3aa38a" + "sha256": "1894ce1b990d880e6dff777433eeb282359225199b853ee16431485de0321906" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "0c28ac6ca9ee9ded432b4385ed5ce5400ea349324bcbff17323b61fd48a9f841" + "sha256": "b5d0bc2a053559673d2bbc5fd4262d5f2086c44bc1786a72bce76a8b086e6720" }, "boot-demo.fedora[partitions][usr-verity]": { "sha256": "bdff57265eca86d1a93ef53c19d88681e715bed6385ddca2a2f13cfff8766175" @@ -54,13 +54,13 @@ "sha256": "84ab38be46148a1d40243767cdc749b0dffa32bd789814b160339efd2d3d8046" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "7fbf96d7c1c343d54d0e1ef6b8c7fd4209fb6ea8656f72639fd699808ae79780" + "sha256": "eaa180ad12781b3ec5bb94a47595443191c7bcb19842b549e120200ee12a1976" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "549e4e6ccd6e57b8b553dc7d7fabbee1be80b89889f9d65edbb50cdb91c806d3" + "sha256": "9e413eccc8894a4f9c30ac16d690b577156c703b9a87c5b4ca664e912493a2ba" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "f6450f8f8a1e832c420e7409809a7b2ff924be69e0eb0155ed7fe4233d437cc7" + "sha256": "d43c2e1f98613d48a86dbf412ed8c051ed434890ca7fa73d1ae8dd9dbff4aeff" }, "boot-demo.fedora[partitions][usr-verity]": { "sha256": "6636ed1748dfdf35d0692b9c8d4ab0c50e5112e41bc91de3ba0950230f4fee41" diff --git a/examples/image/manifest_check.py b/examples/image/manifest_check.py index 4cd53714..f96a097b 100644 --- a/examples/image/manifest_check.py +++ b/examples/image/manifest_check.py @@ -27,7 +27,8 @@ def decode(data: bytes) -> list[dict[str, Any]]: objects = decode(Path(sys.argv[1]).read_bytes()) -root, files = objects[0], objects[1:] +assert objects[-1] == {"mediaType": "application/vnd.uapi.16.trailer"}, "missing trailer" +root, files = objects[0], objects[1:-1] assert root == {"mediaType": "application/vnd.uapi.16.manifest"}, f"bad root object: {root}" assert files, "the manifest lists no files at all" diff --git a/image/BUCK b/image/BUCK index 9e6a2c58..b3a285f6 100644 --- a/image/BUCK +++ b/image/BUCK @@ -57,10 +57,10 @@ python.binary( main = "ddi.py", deps = [ ":finalize", - ":manifest-lib", "//:specs", "//:util", "//rootfs:rootfs", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) @@ -124,8 +124,6 @@ python.test( "finalize_test.py", "layer.py", "layer_test.py", - "manifest.py", - "manifest_test.py", "preload.py", "preload_test.py", "substitute.py", @@ -145,6 +143,7 @@ python.test( "//kmod:kmod", "//package:installer", "//rootfs:rootfs", + "//uapi16:uapi16", ], ) @@ -165,21 +164,11 @@ python.binary( ":finalize", "//:specs", "//:util", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) -# A driver that packages a tree lists it from inside its own action, where the tree is final. -python.library( - name = "manifest-lib", - srcs = ["manifest.py"], - deps = [ - ":finalize", - "//:specs", - "//:util", - ], -) - python.library( name = "finalize", srcs = ["finalize.py"], diff --git a/image/ddi.py b/image/ddi.py index 244ff3ba..df6685c5 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -21,8 +21,8 @@ import util import finalize -import manifest import rootfs +import uapi16 _SEED_NAMESPACE = uuid.UUID("5af2de99-4f9f-4e0b-a04b-bde36b068c4f") @@ -511,7 +511,7 @@ def main(argv: list[str] | None = None) -> None: listings = [] for partition in written: if partition.manifest is not None: - manifest.write(tree, partition.manifest, epoch, roots=partition.definition.sources) + uapi16.write(tree, partition.manifest, epoch, roots=partition.definition.sources) listings.append(partition.manifest) # An imported partition was listed by the invocation that filled it, and this one holds @@ -519,7 +519,7 @@ def main(argv: list[str] | None = None) -> None: # partition this invocation mounts over a directory another one holds is the one on top. if spec["manifest"]: imported = [it.manifest for it in partitions if it.manifest is not None] - manifest.merge(imported + listings, Path(spec["manifest"])) + uapi16.merge(imported + listings, Path(spec["manifest"])) scratch = Path(scratch_dir) repart_definitions = scratch / "repart.d" diff --git a/image/manifest.py b/image/manifest.py index 5dfd8189..d3b69f84 100755 --- a/image/manifest.py +++ b/image/manifest.py @@ -3,298 +3,29 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""List everything a logical image ships, as a UAPI.16 File Manifest. +"""Export the finalized image's UAPI.16 listing.""" -The format is an RFC7464 JSON-SEQ sequence: a root object naming the media type, then one object -per inode, each directory immediately followed by its own contents. Two consumers want this: a -comparison of the paths two images ship (a path in both a system extension and the /usr it merges -onto silently shadows an OS file), and per-image size accounting. Both need every inode type an OS -tree holds, so a type that cannot be listed is an error rather than an omission. - -A driver writing a listing of what it is about to package imports this rather than running it: only -the driver holds the finished tree, and only it knows which paths of that tree its artifact carries. -`roots` is how it says so, and names stay relative to the image, so a listing reads as where each -path lands on a machine rather than where it sits inside one partition. -""" - -import base64 -import hashlib -import json import os -import stat import sys -from collections import Counter -from collections.abc import Iterator, Sequence -from dataclasses import dataclass from pathlib import Path -from typing import cast import specs -from util import fail import finalize +import uapi16 class Spec(finalize.ImageSpec): out: str -MEDIA_TYPE = "application/vnd.uapi.16.manifest" - -# RFC7464 frames every JSON text with a leading record separator and a trailing line feed. -_RS = b"\x1e" -_LF = b"\n" - -# A directory holding a manifest cannot also list it, so the format reserves the name at the top -# level; deeper in the tree it is an ordinary file name. -_RESERVED = "Uapi16Manifest" - -_INODE_TYPES = ( - (stat.S_ISREG, "reg"), - (stat.S_ISDIR, "dir"), - (stat.S_ISLNK, "lnk"), - (stat.S_ISFIFO, "fifo"), - (stat.S_ISCHR, "chr"), - (stat.S_ISBLK, "blk"), - (stat.S_ISSOCK, "sock"), -) - -_NANOSECONDS = 10**9 - - -@dataclass(frozen=True) -class Entry: - """One inode below the image root, under the name the manifest gives it.""" - - name: str - path: Path - st: os.stat_result - - @property - def inode(self) -> tuple[int, int]: - return (self.st.st_dev, self.st.st_ino) - - -def inode_type(mode: int) -> str: - for predicate, name in _INODE_TYPES: - if predicate(mode): - return name - fail(f"manifest: inode type {stat.S_IFMT(mode):#o} has no manifest representation") - - -def check_name(name: str) -> str: - """Reject a path the format cannot encode, rather than listing the image incompletely. - - Nothing normalizes here: scandir yields neither "." nor ".." nor an empty component, so what - is left is the format's own restrictions on the bytes and on the reserved file name. - """ - try: - name.encode("utf-8") - except UnicodeEncodeError: - fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be") - if any(ord(character) < 32 or ord(character) == 127 for character in name): - fail(f"manifest: {name!r} contains a control character") - top = name.split("/", 1)[0] - if top == _RESERVED or top.startswith(_RESERVED + "."): - fail(f"manifest: {name!r} is the reserved manifest file name") - return name - - -def scope(roots: Sequence[str]) -> list[str]: - """The image paths to list, as manifest names: deduplicated, ordered, and non-overlapping. - - A path already covered by another is dropped rather than listed twice, which is what a partition - copying both a directory and something below it would otherwise produce. - """ - names = sorted({root.strip("/") for root in roots}, key=os.fsencode) - if "" in names: # the whole tree, which covers every other root by definition - return [""] - kept: list[str] = [] - for name in names: - if not any(name == covered or name.startswith(covered + "/") for covered in kept): - kept.append(name) - return kept - - -def entries(tree: Path, roots: Sequence[str] = ()) -> Iterator[Entry]: - """Yield every inode below `tree` in manifest order, or only those `roots` reach. - - Pre-order, because the format wants a directory's contents to follow it immediately, and sorted - on the raw name bytes, which is an order neither the locale nor the filesystem can shift. - - A root the image does not have is skipped rather than refused: repart copies what a definition - names if it is there, so the ESP definition can name both /boot and /efi for images that have - one or the other. - """ - for name in scope(roots) if roots else [""]: - if name == "": - yield from _below(tree, "") - continue - path = tree / name - if not path.is_symlink() and not path.exists(): - continue - st = path.lstat() - yield Entry(name=check_name(name), path=path, st=st) - if stat.S_ISDIR(st.st_mode): - yield from _below(path, name + "/") - - -def _below(directory: Path, prefix: str) -> Iterator[Entry]: - with os.scandir(directory) as scan: - children = sorted(scan, key=lambda child: os.fsencode(child.name)) - for child in children: - name = check_name(prefix + child.name) - st = child.stat(follow_symlinks=False) - yield Entry(name=name, path=Path(child.path), st=st) - if stat.S_ISDIR(st.st_mode): - yield from _below(Path(child.path), name + "/") - - -def hardlink_tokens(found: Sequence[Entry]) -> dict[tuple[int, int], int]: - """Number the inodes the manifest reaches by more than one name, in the order it reaches them. - - A link count above one does not make a file hardlinked *here*: the other names may sit outside - the image, and a directory's count is its subdirectories. Only a second name in the manifest - does, so the tokens count reachable names rather than trusting st_nlink. - """ - reachable = Counter( - entry.inode for entry in found if entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode) - ) - tokens: dict[tuple[int, int], int] = {} - for entry in found: - if reachable[entry.inode] > 1 and entry.inode not in tokens: - tokens[entry.inode] = len(tokens) + 1 - return tokens - - -def _digest(path: Path) -> str: - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[str, object]: - """The record for one inode, with its fields in the order the specification lists them.""" - kind = inode_type(entry.st.st_mode) - obj: dict[str, object] = {"name": entry.name, "type": kind} - target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b"" - - if kind == "reg": - obj["size"] = entry.st.st_size - elif kind == "lnk": - obj["size"] = len(target) - elif kind in ("chr", "blk"): - obj["major"] = os.major(entry.st.st_rdev) - obj["minor"] = os.minor(entry.st.st_rdev) - if kind != "lnk": # the format gives a symlink no mode - obj["mode"] = stat.S_IMODE(entry.st.st_mode) - obj["uid"] = entry.st.st_uid - obj["gid"] = entry.st.st_gid - # Clamped exactly as every other reproducible output here clamps: what a package stamped - # survives, what this build stamped collapses onto the epoch. - obj["mTime"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) - if token is not None: - obj["inodeToken"] = token - if kind == "lnk": - # A symlink target has no file of its own to point at, and the format recommends carrying - # it inline; base64 also spares it the UTF-8 rule a name has to satisfy. - obj["contents"] = [{"literal": base64.b64encode(target).decode("ascii")}] - if kind == "reg": - # No `contents`: for a regular file the format then implies the file of that name beside - # the manifest, which is what this manifest describes. - obj["sha256"] = _digest(entry.path) - return obj - - -def _record(obj: dict[str, object]) -> bytes: - return _RS + json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + _LF - - -def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: - """The whole sequence for `tree`: the root object, then one object per inode below it.""" - # The root object carries the media type alone, which is what marks the sequence a manifest. - # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the - # format defaults a nameless first object to a directory anyway. - yield _record({"mediaType": MEDIA_TYPE}) - found = list(entries(tree, roots)) - tokens = hardlink_tokens(found) - for entry in found: - yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode))) - - -def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: - written = 0 - out.parent.mkdir(parents=True, exist_ok=True) - with out.open("wb") as stream: - for record in records(tree, epoch, roots): - stream.write(record) - written += 1 - return written - - -def _objects(source: Path) -> Iterator[dict[str, object]]: - """The named objects of a manifest, the root object it opens with aside.""" - data = source.read_bytes() - head, separator, rest = data.partition(_RS) - if not separator or head: - fail(f"manifest: {source} does not open with a record") - for index, record in enumerate(rest.split(_RS)): - if not record.endswith(_LF): - fail(f"manifest: {source} holds a record that no line feed ends") - obj = cast(dict[str, object], json.loads(record)) - if index == 0: - if obj != {"mediaType": MEDIA_TYPE}: - fail(f"manifest: {source} does not open with a {MEDIA_TYPE} object") - continue - yield obj - - -def order(name: str) -> list[bytes]: - """Sort a name before the contents it holds, which sorting the names themselves does not. - - A manifest is in pre-order, and "dir/x" belongs directly after "dir" even though "dir.txt" - sorts between the two: the components decide, not the bytes of the whole name. - """ - return [os.fsencode(component) for component in name.split("/")] - - -def merge(sources: Sequence[Path], out: Path) -> int: - """Write the listing of a whole from the listings of the parts it is assembled from. - - A disk is filled one partition at a time, so what it carries is what its partitions carry - between them. A name more than one of them lists is taken from the last, which is the partition - mounted over a directory the others merely hold. - """ - objects: dict[str, dict[str, object]] = {} - offset = 0 - for source in sources: - highest = 0 - for obj in _objects(source): - token = obj.get("inodeToken") - if isinstance(token, int): - # A token numbers an inode within one manifest, so each part keeps its own grouping - # by carrying on where the part before it left off. - highest = max(highest, token) - obj["inodeToken"] = token + offset - objects[str(obj["name"])] = obj - offset += highest - - written = 1 - out.parent.mkdir(parents=True, exist_ok=True) - with out.open("wb") as stream: - stream.write(_record({"mediaType": MEDIA_TYPE})) - for name in sorted(objects, key=order): - stream.write(_record(objects[name])) - written += 1 - return written - - def main(argv: list[str] | None = None) -> None: spec = specs.parse(Spec, "manifest", argv) out = Path(spec["out"]) epoch = int(os.environ["SOURCE_DATE_EPOCH"]) with finalize.image(spec, program="manifest") as tree: - written = write(tree, out, epoch) + written = uapi16.write(tree, out, epoch) print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) diff --git a/ruff.toml b/ruff.toml index 5ddc2873..5677281c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -21,6 +21,7 @@ src = [ "package_system/rpm", "remote_cache", "rootfs", + "uapi16", ] lint.select = [ "E", # pycodestyle diff --git a/uapi16/BUCK b/uapi16/BUCK new file mode 100644 index 00000000..98170b32 --- /dev/null +++ b/uapi16/BUCK @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""UAPI.16 manifests and the snapshot stores built on them.""" + +load("//python:defs.bzl", "python") + +python.library( + name = "uapi16", + srcs = ["uapi16.py"], + deps = ["//:util"], + visibility = ["tine//..."], +) + +python.test( + name = "test", + box = "//tools:dev.box", + srcs = ["uapi16_test.py"], + deps = [":uapi16", "//:util"], +) diff --git a/uapi16/pyproject.toml b/uapi16/pyproject.toml new file mode 100644 index 00000000..a0e58bb2 --- /dev/null +++ b/uapi16/pyproject.toml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +# Zed needs a Python manifest here to route these files to their own ty instance. diff --git a/uapi16/uapi16.py b/uapi16/uapi16.py new file mode 100644 index 00000000..6f588465 --- /dev/null +++ b/uapi16/uapi16.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Read, write and combine UAPI.16 filesystem listings.""" + +import base64 +import hashlib +import json +import os +import stat +from collections import Counter +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TypedDict + +from util import fail + +MEDIA_TYPE = "application/vnd.uapi.16.manifest" +TRAILER_TYPE = "application/vnd.uapi.16.trailer" + +# RFC7464 frames every JSON text with a leading record separator and a trailing line feed. +_RS = b"\x1e" +_LF = b"\n" + +# A directory holding a manifest cannot also list it, so the format reserves the name at the top +# level; deeper in the tree it is an ordinary file name. +MANIFEST = "Uapi16Manifest" + +_INODE_TYPES = ( + (stat.S_ISREG, "reg"), + (stat.S_ISDIR, "dir"), + (stat.S_ISLNK, "lnk"), + (stat.S_ISFIFO, "fifo"), + (stat.S_ISCHR, "chr"), + (stat.S_ISBLK, "blk"), + (stat.S_ISSOCK, "sock"), +) + +_NANOSECONDS = 10**9 + + +class Content(TypedDict, total=False): + file: str + literal: str + + +class Xattr(TypedDict, total=False): + text: str + base64: str + + +class Object(TypedDict, total=False): + """One record of a UAPI.16 manifest.""" + + mediaType: str + name: str + type: str + mode: int + uid: int + gid: int + mTimeNSec: int + size: int + major: int + minor: int + sha256: str + contents: list[Content] + inodeToken: int + extendedAttributes: dict[str, dict[str, Xattr]] + + +@dataclass(frozen=True) +class Entry: + """One inode below the image root, under the name the manifest gives it.""" + + name: str + path: Path + st: os.stat_result + + @property + def inode(self) -> tuple[int, int]: + return (self.st.st_dev, self.st.st_ino) + + +def inode_type(mode: int) -> str: + for predicate, name in _INODE_TYPES: + if predicate(mode): + return name + fail(f"manifest: inode type {stat.S_IFMT(mode):#o} has no manifest representation") + + +def check_name(name: str) -> str: + """Reject a path the format cannot encode, rather than listing the image incompletely.""" + try: + name.encode("utf-8") + except UnicodeEncodeError: + fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be") + if any(ord(character) < 32 or ord(character) == 127 for character in name): + fail(f"manifest: {name!r} contains a control character") + if any(part in ("", ".", "..") for part in name.split("/")): + fail(f"manifest: {name!r} must be a normalized relative path") + top = name.split("/", 1)[0] + if top == MANIFEST or top.startswith(MANIFEST + "."): + fail(f"manifest: {name!r} is the reserved manifest file name") + return name + + +def scope(roots: Sequence[str]) -> list[str]: + """The image paths to list, as manifest names: deduplicated, ordered, and non-overlapping. + + A path already covered by another is dropped rather than listed twice, which is what a partition + copying both a directory and something below it would otherwise produce. + """ + names = sorted({root.strip("/") for root in roots}, key=os.fsencode) + if "" in names: # the whole tree, which covers every other root by definition + return [""] + kept: list[str] = [] + for name in names: + if not any(name == covered or name.startswith(covered + "/") for covered in kept): + kept.append(name) + return kept + + +def entries(tree: Path, roots: Sequence[str] = ()) -> Iterator[Entry]: + """Yield every inode below `tree` in manifest order, or only those `roots` reach. + + Pre-order, because the format wants a directory's contents to follow it immediately, and sorted + on the raw name bytes, which is an order neither the locale nor the filesystem can shift. + + A root the image does not have is skipped rather than refused: repart copies what a definition + names if it is there, so the ESP definition can name both /boot and /efi for images that have + one or the other. + """ + for name in scope(roots) if roots else [""]: + if name == "": + yield from _below(tree, "") + continue + path = tree / name + if not path.is_symlink() and not path.exists(): + continue + st = path.lstat() + yield Entry(name=check_name(name), path=path, st=st) + if stat.S_ISDIR(st.st_mode): + yield from _below(path, name + "/") + + +def _below(directory: Path, prefix: str) -> Iterator[Entry]: + with os.scandir(directory) as scan: + children = sorted(scan, key=lambda child: os.fsencode(child.name)) + for child in children: + name = check_name(prefix + child.name) + st = child.stat(follow_symlinks=False) + yield Entry(name=name, path=Path(child.path), st=st) + if stat.S_ISDIR(st.st_mode): + yield from _below(Path(child.path), name + "/") + + +def hardlink_tokens(found: Sequence[Entry]) -> dict[tuple[int, int], int]: + """Number the inodes the manifest reaches by more than one name, in the order it reaches them. + + A link count above one does not make a file hardlinked *here*: the other names may sit outside + the image, and a directory's count is its subdirectories. Only a second name in the manifest + does, so the tokens count reachable names rather than trusting st_nlink. + """ + reachable = Counter( + entry.inode for entry in found if entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode) + ) + tokens: dict[tuple[int, int], int] = {} + for entry in found: + if reachable[entry.inode] > 1 and entry.inode not in tokens: + tokens[entry.inode] = len(tokens) + 1 + return tokens + + +def _digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object: + """Describe one inode using UAPI.16 metadata and content fields.""" + kind = inode_type(entry.st.st_mode) + obj: Object = {"name": entry.name, "type": kind} + target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b"" + + if kind == "reg": + obj["size"] = entry.st.st_size + elif kind == "lnk": + obj["size"] = len(target) + elif kind in ("chr", "blk"): + obj["major"] = os.major(entry.st.st_rdev) + obj["minor"] = os.minor(entry.st.st_rdev) + if kind != "lnk": # the format gives a symlink no mode + obj["mode"] = stat.S_IMODE(entry.st.st_mode) + obj["uid"] = entry.st.st_uid + obj["gid"] = entry.st.st_gid + obj["mTimeNSec"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) + if entry.st.st_mtime_ns < 0: + fail(f"manifest: {entry.name!r} has a timestamp before the UNIX epoch") + if token is not None: + obj["inodeToken"] = token + if kind == "lnk": + # A symlink target has no file of its own to point at, and the format recommends carrying + # it inline; base64 also spares it the UTF-8 rule a name has to satisfy. + obj["contents"] = [{"literal": base64.b64encode(target).decode("ascii")}] + if kind == "reg": + # No `contents`: for a regular file the format then implies the file of that name beside + # the manifest, which is what this manifest describes. + obj["sha256"] = _digest(entry.path) + return obj + + +def _record(obj: Object) -> bytes: + return ( + _RS + + json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + _LF + ) + + +def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: + """The whole sequence for `tree`: the root object, then one object per inode below it.""" + # The root object carries the media type alone, which is what marks the sequence a manifest. + # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the + # format defaults a nameless first object to a directory anyway. + yield _record({"mediaType": MEDIA_TYPE}) + found = list(entries(tree, roots)) + tokens = hardlink_tokens(found) + for entry in found: + yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode))) + yield _record({"mediaType": TRAILER_TYPE}) + + +def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: + written = 0 + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("wb") as stream: + for record in records(tree, epoch, roots): + stream.write(record) + written += 1 + return written + + +def read(source: Path) -> list[Object]: + """Read a manifest that `write` or `write_objects` produced. + + Only a complete manifest ends with the trailer, so a manifest that a failed write cut short fails + here instead of becoming a snapshot. + """ + records = source.read_bytes().split(_RS)[1:] + if not records or records[-1] != _record({"mediaType": TRAILER_TYPE})[1:]: + fail(f"manifest: {source} is incomplete: missing trailer") + return [json.loads(record) for record in records[:-1]] + + +def write_objects(objects: Sequence[Object], out: Path) -> int: + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("wb") as stream: + for obj in objects: + stream.write(_record(obj)) + stream.write(_record({"mediaType": TRAILER_TYPE})) + return len(objects) + 1 + + +def order(name: str) -> list[bytes]: + """Sort a name before the contents it holds, which sorting the names themselves does not. + + A manifest is in pre-order, and "dir/x" belongs directly after "dir" even though "dir.txt" + sorts between the two: the components decide, not the bytes of the whole name. + """ + return [os.fsencode(component) for component in name.split("/")] + + +def merge(sources: Sequence[Path], out: Path) -> int: + """Write the listing of a whole from the listings of the parts it is assembled from. + + A disk is filled one partition at a time, so what it carries is what its partitions carry + between them. A name more than one of them lists is taken from the last, which is the partition + mounted over a directory the others merely hold. + """ + objects: dict[str, Object] = {} + offset = 0 + for source in sources: + highest = 0 + for obj in read(source)[1:]: + token = obj.get("inodeToken") + if isinstance(token, int): + # A token numbers an inode within one manifest, so each part keeps its own grouping + # by carrying on where the part before it left off. + highest = max(highest, token) + obj["inodeToken"] = token + offset + objects[str(obj["name"])] = obj + offset += highest + + tokens: dict[int, int] = {} + counts = Counter(obj.get("inodeToken") for obj in objects.values() if obj.get("inodeToken")) + ordered = [objects[name] for name in sorted(objects, key=order)] + for obj in ordered: + token = obj.pop("inodeToken", None) + if isinstance(token, int) and counts[token] > 1: + obj["inodeToken"] = tokens.setdefault(token, len(tokens) + 1) + return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out) diff --git a/image/manifest_test.py b/uapi16/uapi16_test.py similarity index 81% rename from image/manifest_test.py rename to uapi16/uapi16_test.py index 5375e9ae..06d6ed42 100644 --- a/image/manifest_test.py +++ b/uapi16/uapi16_test.py @@ -3,7 +3,7 @@ """Tests for the UAPI.16 file manifest. -buck test tine//image:test +buck test tine//uapi16:test """ import base64 @@ -17,7 +17,7 @@ from pathlib import Path from typing import override -import manifest +import uapi16 # The epoch every image action runs under (box/runtime.bzl); any value works, but using the real # one keeps the clamping assertions honest about what a build actually produces. @@ -33,16 +33,17 @@ def parse(data: bytes) -> list[dict[str, object]]: for record in rest.split(b"\x1e"): assert record.endswith(b"\n"), "a record is not terminated by a line feed" objects.append(json.loads(record.decode("utf-8"))) - return objects + assert objects[-1] == {"mediaType": uapi16.TRAILER_TYPE} + return objects[:-1] class TreeTest(unittest.TestCase): @override def setUp(self) -> None: - self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest."))) + self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest."))) def manifest(self, tree: Path | None = None, roots: Sequence[str] = ()) -> list[dict[str, object]]: - return parse(b"".join(manifest.records(tree or self.tree, EPOCH, roots))) + return parse(b"".join(uapi16.records(tree or self.tree, EPOCH, roots))) def named(self, tree: Path | None = None) -> dict[str, dict[str, object]]: return {str(obj["name"]): obj for obj in self.manifest(tree)[1:]} @@ -54,18 +55,20 @@ def test_the_first_object_is_the_root_and_names_the_media_type(self) -> None: root = self.manifest()[0] self.assertEqual(root, {"mediaType": "application/vnd.uapi.16.manifest"}) - def test_an_empty_tree_is_the_root_object_alone(self) -> None: + def test_an_empty_tree_has_no_named_objects(self) -> None: + # An empty filesystem still has its root and trailer, with no named file objects. self.assertEqual(len(self.manifest()), 1) def test_each_record_is_framed_by_a_separator_and_a_line_feed(self) -> None: + # A tree with one file needs three JSON-SEQ records: the root, file and trailer. (self.tree / "a").write_text("a") - data = b"".join(manifest.records(self.tree, EPOCH)) - self.assertEqual(data.count(b"\x1e"), 2) + data = b"".join(uapi16.records(self.tree, EPOCH)) + self.assertEqual(data.count(b"\x1e"), 3) self.assertTrue(data.endswith(b"\n")) def test_the_json_carries_no_gratuitous_whitespace(self) -> None: (self.tree / "a").write_text("a") - self.assertNotIn(b" ", b"".join(manifest.records(self.tree, EPOCH))) + self.assertNotIn(b" ", b"".join(uapi16.records(self.tree, EPOCH))) class TestOrder(TreeTest): @@ -92,14 +95,14 @@ def test_the_bytes_do_not_depend_on_the_order_the_tree_was_created_in(self) -> N names = ["gamma", "alpha", "beta", "delta"] outputs = [] for order in (names, list(reversed(names))): - with tempfile.TemporaryDirectory(prefix="manifest.") as scratch: + with tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.") as scratch: tree = Path(scratch) for name in order: (tree / name).mkdir() (tree / name / "file").write_text(name) os.utime(tree / name / "file", (EPOCH - 1, EPOCH - 1)) os.utime(tree / name, (EPOCH - 1, EPOCH - 1)) - outputs.append(b"".join(manifest.records(tree, EPOCH))) + outputs.append(b"".join(uapi16.records(tree, EPOCH))) self.assertEqual(outputs[0], outputs[1]) @@ -193,7 +196,9 @@ def test_a_file_reached_by_one_name_alone_carries_no_token(self) -> None: def test_a_link_count_above_one_is_not_enough_when_the_other_name_is_outside_the_tree( self, ) -> None: - outside = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.outside."))) + outside = Path( + self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.outside.")) + ) (outside / "one").write_text("shared") os.link(outside / "one", self.tree / "inside") self.assertEqual((self.tree / "inside").stat().st_nlink, 2) @@ -229,24 +234,24 @@ def test_a_unix_socket_is_listed_as_one(self) -> None: def test_the_inode_type_names_follow_the_format(self) -> None: self.assertEqual( - [manifest.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], + [uapi16.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], ["blk", "sock", "lnk"], ) def test_an_inode_type_with_no_representation_fails_rather_than_being_guessed(self) -> None: with self.assertRaises(SystemExit): - manifest.inode_type(0) + uapi16.inode_type(0) class TestDeviceNodes(unittest.TestCase): """Device nodes, from a fabricated inode: this sandbox refuses mknod.""" - def device(self, mode: int, rdev: int) -> dict[str, object]: + def device(self, mode: int, rdev: int) -> uapi16.Object: st = os.stat_result( (mode, 0, 0, 1, 0, 0, 0, 0, EPOCH - 1, 0), {"st_rdev": rdev, "st_mtime_ns": (EPOCH - 1) * 10**9}, ) - return manifest.file_object(manifest.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) + return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -> None: self.assertEqual( @@ -259,7 +264,7 @@ def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) - "mode": 0o666, "uid": 0, "gid": 0, - "mTime": (EPOCH - 1) * 10**9, + "mTimeNSec": (EPOCH - 1) * 10**9, }, ) @@ -273,12 +278,12 @@ class TestTimestamps(TreeTest): def test_an_mtime_past_the_epoch_is_clamped_onto_it(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", (EPOCH + 10, EPOCH + 10)) - self.assertEqual(self.named()["file"]["mTime"], EPOCH * 10**9) + self.assertEqual(self.named()["file"]["mTimeNSec"], EPOCH * 10**9) def test_an_mtime_before_the_epoch_survives_in_nanoseconds(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", ns=(0, (EPOCH - 5) * 10**9 + 123)) - self.assertEqual(self.named()["file"]["mTime"], (EPOCH - 5) * 10**9 + 123) + self.assertEqual(self.named()["file"]["mTimeNSec"], (EPOCH - 5) * 10**9 + 123) class TestUnencodableNames(TreeTest): @@ -304,28 +309,32 @@ def test_a_reserved_suffix_at_the_top_level_is_refused_too(self) -> None: class TestWrite(TreeTest): - def test_the_written_file_is_the_sequence_and_the_count_includes_the_root(self) -> None: + def test_the_count_includes_the_root_and_trailer(self) -> None: + # Writing one file must report three records and finish with the mandatory trailer. (self.tree / "file").write_text("x") - out = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.out."))) / "m" - self.assertEqual(manifest.write(self.tree, out, EPOCH), 2) + out = ( + Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.out."))) + / "m" + ) + self.assertEqual(uapi16.write(self.tree, out, EPOCH), 3) self.assertEqual([obj.get("name") for obj in parse(out.read_bytes())], [None, "file"]) class TestScope(unittest.TestCase): def test_a_leading_and_trailing_slash_is_not_part_of_the_name(self) -> None: - self.assertEqual(manifest.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) + self.assertEqual(uapi16.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) def test_the_same_root_named_twice_is_listed_once(self) -> None: - self.assertEqual(manifest.scope(["/usr", "usr/"]), ["usr"]) + self.assertEqual(uapi16.scope(["/usr", "usr/"]), ["usr"]) def test_a_root_below_another_is_dropped_rather_than_listed_twice(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) + self.assertEqual(uapi16.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) def test_a_name_that_merely_shares_a_prefix_is_kept(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) + self.assertEqual(uapi16.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) def test_the_whole_tree_covers_every_other_root(self) -> None: - self.assertEqual(manifest.scope(["/", "/usr"]), [""]) + self.assertEqual(uapi16.scope(["/", "/usr"]), [""]) class TestScopedTree(TreeTest): @@ -361,7 +370,7 @@ def test_no_roots_is_the_whole_tree(self) -> None: class TestMerge(TreeTest): def part(self, name: str, roots: Sequence[str]) -> Path: out = self.tree / name - manifest.write(self.tree / "parts", out, EPOCH, roots) + uapi16.write(self.tree / "parts", out, EPOCH, roots) return out @override @@ -374,7 +383,7 @@ def setUp(self) -> None: def merged(self, *sources: Path) -> list[dict[str, object]]: out = self.tree / "merged" - manifest.merge(sources, out) + uapi16.merge(sources, out) return parse(out.read_bytes()) def test_the_whole_is_every_part_under_one_root_object(self) -> None: @@ -418,5 +427,33 @@ def test_something_that_is_not_a_manifest_is_refused(self) -> None: self.merged(self.tree / "junk") +class TestReader(TreeTest): + def test_requires_trailer(self) -> None: + # A write that stops early leaves a manifest without its trailer, which must fail instead of + # becoming a snapshot. A cut after the root object or inside the trailer must both fail. + out = self.tree / "manifest" + root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n' + trailer = b'\x1e{"mediaType":"application/vnd.uapi.16.trailer"}\n' + for data in (root, root + trailer[:-1]): + out.write_bytes(data) + with self.subTest(data=data), self.assertRaises(SystemExit): + uapi16.read(out) + out.write_bytes(root + trailer) + self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}]) + + def test_accepts_root_metadata(self) -> None: + # A storage manifest includes root metadata, which must survive parsing unchanged. + out = self.tree / "manifest" + objects: list[uapi16.Object] = [{"mediaType": uapi16.MEDIA_TYPE, "mode": 0o750}] + uapi16.write_objects(objects, out) + self.assertEqual(uapi16.read(out), objects) + + def test_rejects_non_relative_names(self) -> None: + # UAPI.16 forbids path traversal, unnormalized names and control characters. + for name in ("", "/a", "a/", "a//b", "a/../b", "./a", "a/.", "a/..", "a\n"): + with self.subTest(name=name), self.assertRaises(SystemExit): + uapi16.check_name(name) + + if __name__ == "__main__": unittest.main()