diff --git a/docs/design/architecture.md b/docs/design/architecture.md index c7a4e71c..fcbae251 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -53,6 +53,8 @@ tine//box/ box bootstrap and sandbox command construction tine//rootfs/ bind/overlay mounting and stored-delta translation tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and drivers, and VM runners +tine//uapi16/ UAPI.16 manifests +tine//archive/ tar writer, cpio reader and writer tine//git/ pinned Git repositories with local checkout overrides tine//cargo/ vendored crate trees and offline Rust source builds tine//go/ go.sum-verified module fetches and offline Go source builds diff --git a/examples/image-secureboot/BUCK b/examples/image-secureboot/BUCK index 17266ad2..a36ea053 100644 --- a/examples/image-secureboot/BUCK +++ b/examples/image-secureboot/BUCK @@ -314,13 +314,15 @@ box.sh_test( name = "disk-manifest-test", labels = ["image"], box = "@tine//tools:dev.box", - test = "@tine//tests:contains.sh", - args = [ - "$(location :image[disk-manifest])", - '"name":"usr/bin/bash"', - '"name":"boot/tine-tmpfiles","type":"dir","mode":448', - '"name":"boot/EFI/Linux', - ], + args = ["$(location :image[disk-manifest])"], + script = """ +# Authored tmpfiles must set the boot directory mode in the disk's filesystem listing. +jq --seq -es ' + any(.[]; .name == "usr/bin/bash") and + any(.[]; .name == "boot/tine-tmpfiles" and .type == "dir" and .mode == 448) and + any(.[]; .name | strings | startswith("boot/EFI/Linux")) +' "$1" +""", ) # The kernel install hook is bypassed, it didn't generate its own initrd. diff --git a/examples/image/expected.json b/examples/image/expected.json index a73b5a37..56c4e769 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -16,13 +16,13 @@ "sha256": "64b78fbe92ff5fcaefb77ef10923c964e402304a54b0a776a8ca11c9b43c85f7" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "c95655cea99dc0f17d5d4c6211595a8ba18728f94c3513a1695220de04c14e89" + "sha256": "b4f4144aed4ede4dec269912df3de2f1b93d8259eb293c8558822f8853e8f10f" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "f81e7f963bbe9cfaf07ac4495a0d8e47392206e9ddb561943304bbb0ae3aa38a" + "sha256": "1894ce1b990d880e6dff777433eeb282359225199b853ee16431485de0321906" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "0c28ac6ca9ee9ded432b4385ed5ce5400ea349324bcbff17323b61fd48a9f841" + "sha256": "b5d0bc2a053559673d2bbc5fd4262d5f2086c44bc1786a72bce76a8b086e6720" }, "boot-demo.fedora[partitions][usr-verity]": { "sha256": "bdff57265eca86d1a93ef53c19d88681e715bed6385ddca2a2f13cfff8766175" @@ -54,13 +54,13 @@ "sha256": "84ab38be46148a1d40243767cdc749b0dffa32bd789814b160339efd2d3d8046" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "7fbf96d7c1c343d54d0e1ef6b8c7fd4209fb6ea8656f72639fd699808ae79780" + "sha256": "eaa180ad12781b3ec5bb94a47595443191c7bcb19842b549e120200ee12a1976" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "549e4e6ccd6e57b8b553dc7d7fabbee1be80b89889f9d65edbb50cdb91c806d3" + "sha256": "9e413eccc8894a4f9c30ac16d690b577156c703b9a87c5b4ca664e912493a2ba" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "f6450f8f8a1e832c420e7409809a7b2ff924be69e0eb0155ed7fe4233d437cc7" + "sha256": "d43c2e1f98613d48a86dbf412ed8c051ed434890ca7fa73d1ae8dd9dbff4aeff" }, "boot-demo.fedora[partitions][usr-verity]": { "sha256": "6636ed1748dfdf35d0692b9c8d4ab0c50e5112e41bc91de3ba0950230f4fee41" diff --git a/examples/image/manifest_check.py b/examples/image/manifest_check.py index 4cd53714..f96a097b 100644 --- a/examples/image/manifest_check.py +++ b/examples/image/manifest_check.py @@ -27,7 +27,8 @@ def decode(data: bytes) -> list[dict[str, Any]]: objects = decode(Path(sys.argv[1]).read_bytes()) -root, files = objects[0], objects[1:] +assert objects[-1] == {"mediaType": "application/vnd.uapi.16.trailer"}, "missing trailer" +root, files = objects[0], objects[1:-1] assert root == {"mediaType": "application/vnd.uapi.16.manifest"}, f"bad root object: {root}" assert files, "the manifest lists no files at all" diff --git a/image/BUCK b/image/BUCK index 9e6a2c58..b3a285f6 100644 --- a/image/BUCK +++ b/image/BUCK @@ -57,10 +57,10 @@ python.binary( main = "ddi.py", deps = [ ":finalize", - ":manifest-lib", "//:specs", "//:util", "//rootfs:rootfs", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) @@ -124,8 +124,6 @@ python.test( "finalize_test.py", "layer.py", "layer_test.py", - "manifest.py", - "manifest_test.py", "preload.py", "preload_test.py", "substitute.py", @@ -145,6 +143,7 @@ python.test( "//kmod:kmod", "//package:installer", "//rootfs:rootfs", + "//uapi16:uapi16", ], ) @@ -165,21 +164,11 @@ python.binary( ":finalize", "//:specs", "//:util", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) -# A driver that packages a tree lists it from inside its own action, where the tree is final. -python.library( - name = "manifest-lib", - srcs = ["manifest.py"], - deps = [ - ":finalize", - "//:specs", - "//:util", - ], -) - python.library( name = "finalize", srcs = ["finalize.py"], diff --git a/image/ddi.py b/image/ddi.py index 244ff3ba..df6685c5 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -21,8 +21,8 @@ import util import finalize -import manifest import rootfs +import uapi16 _SEED_NAMESPACE = uuid.UUID("5af2de99-4f9f-4e0b-a04b-bde36b068c4f") @@ -511,7 +511,7 @@ def main(argv: list[str] | None = None) -> None: listings = [] for partition in written: if partition.manifest is not None: - manifest.write(tree, partition.manifest, epoch, roots=partition.definition.sources) + uapi16.write(tree, partition.manifest, epoch, roots=partition.definition.sources) listings.append(partition.manifest) # An imported partition was listed by the invocation that filled it, and this one holds @@ -519,7 +519,7 @@ def main(argv: list[str] | None = None) -> None: # partition this invocation mounts over a directory another one holds is the one on top. if spec["manifest"]: imported = [it.manifest for it in partitions if it.manifest is not None] - manifest.merge(imported + listings, Path(spec["manifest"])) + uapi16.merge(imported + listings, Path(spec["manifest"])) scratch = Path(scratch_dir) repart_definitions = scratch / "repart.d" diff --git a/image/manifest.py b/image/manifest.py index 5dfd8189..d3b69f84 100755 --- a/image/manifest.py +++ b/image/manifest.py @@ -3,298 +3,29 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""List everything a logical image ships, as a UAPI.16 File Manifest. +"""Export the finalized image's UAPI.16 listing.""" -The format is an RFC7464 JSON-SEQ sequence: a root object naming the media type, then one object -per inode, each directory immediately followed by its own contents. Two consumers want this: a -comparison of the paths two images ship (a path in both a system extension and the /usr it merges -onto silently shadows an OS file), and per-image size accounting. Both need every inode type an OS -tree holds, so a type that cannot be listed is an error rather than an omission. - -A driver writing a listing of what it is about to package imports this rather than running it: only -the driver holds the finished tree, and only it knows which paths of that tree its artifact carries. -`roots` is how it says so, and names stay relative to the image, so a listing reads as where each -path lands on a machine rather than where it sits inside one partition. -""" - -import base64 -import hashlib -import json import os -import stat import sys -from collections import Counter -from collections.abc import Iterator, Sequence -from dataclasses import dataclass from pathlib import Path -from typing import cast import specs -from util import fail import finalize +import uapi16 class Spec(finalize.ImageSpec): out: str -MEDIA_TYPE = "application/vnd.uapi.16.manifest" - -# RFC7464 frames every JSON text with a leading record separator and a trailing line feed. -_RS = b"\x1e" -_LF = b"\n" - -# A directory holding a manifest cannot also list it, so the format reserves the name at the top -# level; deeper in the tree it is an ordinary file name. -_RESERVED = "Uapi16Manifest" - -_INODE_TYPES = ( - (stat.S_ISREG, "reg"), - (stat.S_ISDIR, "dir"), - (stat.S_ISLNK, "lnk"), - (stat.S_ISFIFO, "fifo"), - (stat.S_ISCHR, "chr"), - (stat.S_ISBLK, "blk"), - (stat.S_ISSOCK, "sock"), -) - -_NANOSECONDS = 10**9 - - -@dataclass(frozen=True) -class Entry: - """One inode below the image root, under the name the manifest gives it.""" - - name: str - path: Path - st: os.stat_result - - @property - def inode(self) -> tuple[int, int]: - return (self.st.st_dev, self.st.st_ino) - - -def inode_type(mode: int) -> str: - for predicate, name in _INODE_TYPES: - if predicate(mode): - return name - fail(f"manifest: inode type {stat.S_IFMT(mode):#o} has no manifest representation") - - -def check_name(name: str) -> str: - """Reject a path the format cannot encode, rather than listing the image incompletely. - - Nothing normalizes here: scandir yields neither "." nor ".." nor an empty component, so what - is left is the format's own restrictions on the bytes and on the reserved file name. - """ - try: - name.encode("utf-8") - except UnicodeEncodeError: - fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be") - if any(ord(character) < 32 or ord(character) == 127 for character in name): - fail(f"manifest: {name!r} contains a control character") - top = name.split("/", 1)[0] - if top == _RESERVED or top.startswith(_RESERVED + "."): - fail(f"manifest: {name!r} is the reserved manifest file name") - return name - - -def scope(roots: Sequence[str]) -> list[str]: - """The image paths to list, as manifest names: deduplicated, ordered, and non-overlapping. - - A path already covered by another is dropped rather than listed twice, which is what a partition - copying both a directory and something below it would otherwise produce. - """ - names = sorted({root.strip("/") for root in roots}, key=os.fsencode) - if "" in names: # the whole tree, which covers every other root by definition - return [""] - kept: list[str] = [] - for name in names: - if not any(name == covered or name.startswith(covered + "/") for covered in kept): - kept.append(name) - return kept - - -def entries(tree: Path, roots: Sequence[str] = ()) -> Iterator[Entry]: - """Yield every inode below `tree` in manifest order, or only those `roots` reach. - - Pre-order, because the format wants a directory's contents to follow it immediately, and sorted - on the raw name bytes, which is an order neither the locale nor the filesystem can shift. - - A root the image does not have is skipped rather than refused: repart copies what a definition - names if it is there, so the ESP definition can name both /boot and /efi for images that have - one or the other. - """ - for name in scope(roots) if roots else [""]: - if name == "": - yield from _below(tree, "") - continue - path = tree / name - if not path.is_symlink() and not path.exists(): - continue - st = path.lstat() - yield Entry(name=check_name(name), path=path, st=st) - if stat.S_ISDIR(st.st_mode): - yield from _below(path, name + "/") - - -def _below(directory: Path, prefix: str) -> Iterator[Entry]: - with os.scandir(directory) as scan: - children = sorted(scan, key=lambda child: os.fsencode(child.name)) - for child in children: - name = check_name(prefix + child.name) - st = child.stat(follow_symlinks=False) - yield Entry(name=name, path=Path(child.path), st=st) - if stat.S_ISDIR(st.st_mode): - yield from _below(Path(child.path), name + "/") - - -def hardlink_tokens(found: Sequence[Entry]) -> dict[tuple[int, int], int]: - """Number the inodes the manifest reaches by more than one name, in the order it reaches them. - - A link count above one does not make a file hardlinked *here*: the other names may sit outside - the image, and a directory's count is its subdirectories. Only a second name in the manifest - does, so the tokens count reachable names rather than trusting st_nlink. - """ - reachable = Counter( - entry.inode for entry in found if entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode) - ) - tokens: dict[tuple[int, int], int] = {} - for entry in found: - if reachable[entry.inode] > 1 and entry.inode not in tokens: - tokens[entry.inode] = len(tokens) + 1 - return tokens - - -def _digest(path: Path) -> str: - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[str, object]: - """The record for one inode, with its fields in the order the specification lists them.""" - kind = inode_type(entry.st.st_mode) - obj: dict[str, object] = {"name": entry.name, "type": kind} - target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b"" - - if kind == "reg": - obj["size"] = entry.st.st_size - elif kind == "lnk": - obj["size"] = len(target) - elif kind in ("chr", "blk"): - obj["major"] = os.major(entry.st.st_rdev) - obj["minor"] = os.minor(entry.st.st_rdev) - if kind != "lnk": # the format gives a symlink no mode - obj["mode"] = stat.S_IMODE(entry.st.st_mode) - obj["uid"] = entry.st.st_uid - obj["gid"] = entry.st.st_gid - # Clamped exactly as every other reproducible output here clamps: what a package stamped - # survives, what this build stamped collapses onto the epoch. - obj["mTime"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) - if token is not None: - obj["inodeToken"] = token - if kind == "lnk": - # A symlink target has no file of its own to point at, and the format recommends carrying - # it inline; base64 also spares it the UTF-8 rule a name has to satisfy. - obj["contents"] = [{"literal": base64.b64encode(target).decode("ascii")}] - if kind == "reg": - # No `contents`: for a regular file the format then implies the file of that name beside - # the manifest, which is what this manifest describes. - obj["sha256"] = _digest(entry.path) - return obj - - -def _record(obj: dict[str, object]) -> bytes: - return _RS + json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + _LF - - -def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: - """The whole sequence for `tree`: the root object, then one object per inode below it.""" - # The root object carries the media type alone, which is what marks the sequence a manifest. - # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the - # format defaults a nameless first object to a directory anyway. - yield _record({"mediaType": MEDIA_TYPE}) - found = list(entries(tree, roots)) - tokens = hardlink_tokens(found) - for entry in found: - yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode))) - - -def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: - written = 0 - out.parent.mkdir(parents=True, exist_ok=True) - with out.open("wb") as stream: - for record in records(tree, epoch, roots): - stream.write(record) - written += 1 - return written - - -def _objects(source: Path) -> Iterator[dict[str, object]]: - """The named objects of a manifest, the root object it opens with aside.""" - data = source.read_bytes() - head, separator, rest = data.partition(_RS) - if not separator or head: - fail(f"manifest: {source} does not open with a record") - for index, record in enumerate(rest.split(_RS)): - if not record.endswith(_LF): - fail(f"manifest: {source} holds a record that no line feed ends") - obj = cast(dict[str, object], json.loads(record)) - if index == 0: - if obj != {"mediaType": MEDIA_TYPE}: - fail(f"manifest: {source} does not open with a {MEDIA_TYPE} object") - continue - yield obj - - -def order(name: str) -> list[bytes]: - """Sort a name before the contents it holds, which sorting the names themselves does not. - - A manifest is in pre-order, and "dir/x" belongs directly after "dir" even though "dir.txt" - sorts between the two: the components decide, not the bytes of the whole name. - """ - return [os.fsencode(component) for component in name.split("/")] - - -def merge(sources: Sequence[Path], out: Path) -> int: - """Write the listing of a whole from the listings of the parts it is assembled from. - - A disk is filled one partition at a time, so what it carries is what its partitions carry - between them. A name more than one of them lists is taken from the last, which is the partition - mounted over a directory the others merely hold. - """ - objects: dict[str, dict[str, object]] = {} - offset = 0 - for source in sources: - highest = 0 - for obj in _objects(source): - token = obj.get("inodeToken") - if isinstance(token, int): - # A token numbers an inode within one manifest, so each part keeps its own grouping - # by carrying on where the part before it left off. - highest = max(highest, token) - obj["inodeToken"] = token + offset - objects[str(obj["name"])] = obj - offset += highest - - written = 1 - out.parent.mkdir(parents=True, exist_ok=True) - with out.open("wb") as stream: - stream.write(_record({"mediaType": MEDIA_TYPE})) - for name in sorted(objects, key=order): - stream.write(_record(objects[name])) - written += 1 - return written - - def main(argv: list[str] | None = None) -> None: spec = specs.parse(Spec, "manifest", argv) out = Path(spec["out"]) epoch = int(os.environ["SOURCE_DATE_EPOCH"]) with finalize.image(spec, program="manifest") as tree: - written = write(tree, out, epoch) + written = uapi16.write(tree, out, epoch) print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) diff --git a/ruff.toml b/ruff.toml index 5ddc2873..5677281c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -21,6 +21,7 @@ src = [ "package_system/rpm", "remote_cache", "rootfs", + "uapi16", ] lint.select = [ "E", # pycodestyle diff --git a/uapi16/BUCK b/uapi16/BUCK new file mode 100644 index 00000000..98170b32 --- /dev/null +++ b/uapi16/BUCK @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""UAPI.16 manifests and the snapshot stores built on them.""" + +load("//python:defs.bzl", "python") + +python.library( + name = "uapi16", + srcs = ["uapi16.py"], + deps = ["//:util"], + visibility = ["tine//..."], +) + +python.test( + name = "test", + box = "//tools:dev.box", + srcs = ["uapi16_test.py"], + deps = [":uapi16", "//:util"], +) diff --git a/uapi16/pyproject.toml b/uapi16/pyproject.toml new file mode 100644 index 00000000..a0e58bb2 --- /dev/null +++ b/uapi16/pyproject.toml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +# Zed needs a Python manifest here to route these files to their own ty instance. diff --git a/uapi16/uapi16.py b/uapi16/uapi16.py new file mode 100644 index 00000000..6f588465 --- /dev/null +++ b/uapi16/uapi16.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Read, write and combine UAPI.16 filesystem listings.""" + +import base64 +import hashlib +import json +import os +import stat +from collections import Counter +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TypedDict + +from util import fail + +MEDIA_TYPE = "application/vnd.uapi.16.manifest" +TRAILER_TYPE = "application/vnd.uapi.16.trailer" + +# RFC7464 frames every JSON text with a leading record separator and a trailing line feed. +_RS = b"\x1e" +_LF = b"\n" + +# A directory holding a manifest cannot also list it, so the format reserves the name at the top +# level; deeper in the tree it is an ordinary file name. +MANIFEST = "Uapi16Manifest" + +_INODE_TYPES = ( + (stat.S_ISREG, "reg"), + (stat.S_ISDIR, "dir"), + (stat.S_ISLNK, "lnk"), + (stat.S_ISFIFO, "fifo"), + (stat.S_ISCHR, "chr"), + (stat.S_ISBLK, "blk"), + (stat.S_ISSOCK, "sock"), +) + +_NANOSECONDS = 10**9 + + +class Content(TypedDict, total=False): + file: str + literal: str + + +class Xattr(TypedDict, total=False): + text: str + base64: str + + +class Object(TypedDict, total=False): + """One record of a UAPI.16 manifest.""" + + mediaType: str + name: str + type: str + mode: int + uid: int + gid: int + mTimeNSec: int + size: int + major: int + minor: int + sha256: str + contents: list[Content] + inodeToken: int + extendedAttributes: dict[str, dict[str, Xattr]] + + +@dataclass(frozen=True) +class Entry: + """One inode below the image root, under the name the manifest gives it.""" + + name: str + path: Path + st: os.stat_result + + @property + def inode(self) -> tuple[int, int]: + return (self.st.st_dev, self.st.st_ino) + + +def inode_type(mode: int) -> str: + for predicate, name in _INODE_TYPES: + if predicate(mode): + return name + fail(f"manifest: inode type {stat.S_IFMT(mode):#o} has no manifest representation") + + +def check_name(name: str) -> str: + """Reject a path the format cannot encode, rather than listing the image incompletely.""" + try: + name.encode("utf-8") + except UnicodeEncodeError: + fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be") + if any(ord(character) < 32 or ord(character) == 127 for character in name): + fail(f"manifest: {name!r} contains a control character") + if any(part in ("", ".", "..") for part in name.split("/")): + fail(f"manifest: {name!r} must be a normalized relative path") + top = name.split("/", 1)[0] + if top == MANIFEST or top.startswith(MANIFEST + "."): + fail(f"manifest: {name!r} is the reserved manifest file name") + return name + + +def scope(roots: Sequence[str]) -> list[str]: + """The image paths to list, as manifest names: deduplicated, ordered, and non-overlapping. + + A path already covered by another is dropped rather than listed twice, which is what a partition + copying both a directory and something below it would otherwise produce. + """ + names = sorted({root.strip("/") for root in roots}, key=os.fsencode) + if "" in names: # the whole tree, which covers every other root by definition + return [""] + kept: list[str] = [] + for name in names: + if not any(name == covered or name.startswith(covered + "/") for covered in kept): + kept.append(name) + return kept + + +def entries(tree: Path, roots: Sequence[str] = ()) -> Iterator[Entry]: + """Yield every inode below `tree` in manifest order, or only those `roots` reach. + + Pre-order, because the format wants a directory's contents to follow it immediately, and sorted + on the raw name bytes, which is an order neither the locale nor the filesystem can shift. + + A root the image does not have is skipped rather than refused: repart copies what a definition + names if it is there, so the ESP definition can name both /boot and /efi for images that have + one or the other. + """ + for name in scope(roots) if roots else [""]: + if name == "": + yield from _below(tree, "") + continue + path = tree / name + if not path.is_symlink() and not path.exists(): + continue + st = path.lstat() + yield Entry(name=check_name(name), path=path, st=st) + if stat.S_ISDIR(st.st_mode): + yield from _below(path, name + "/") + + +def _below(directory: Path, prefix: str) -> Iterator[Entry]: + with os.scandir(directory) as scan: + children = sorted(scan, key=lambda child: os.fsencode(child.name)) + for child in children: + name = check_name(prefix + child.name) + st = child.stat(follow_symlinks=False) + yield Entry(name=name, path=Path(child.path), st=st) + if stat.S_ISDIR(st.st_mode): + yield from _below(Path(child.path), name + "/") + + +def hardlink_tokens(found: Sequence[Entry]) -> dict[tuple[int, int], int]: + """Number the inodes the manifest reaches by more than one name, in the order it reaches them. + + A link count above one does not make a file hardlinked *here*: the other names may sit outside + the image, and a directory's count is its subdirectories. Only a second name in the manifest + does, so the tokens count reachable names rather than trusting st_nlink. + """ + reachable = Counter( + entry.inode for entry in found if entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode) + ) + tokens: dict[tuple[int, int], int] = {} + for entry in found: + if reachable[entry.inode] > 1 and entry.inode not in tokens: + tokens[entry.inode] = len(tokens) + 1 + return tokens + + +def _digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object: + """Describe one inode using UAPI.16 metadata and content fields.""" + kind = inode_type(entry.st.st_mode) + obj: Object = {"name": entry.name, "type": kind} + target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b"" + + if kind == "reg": + obj["size"] = entry.st.st_size + elif kind == "lnk": + obj["size"] = len(target) + elif kind in ("chr", "blk"): + obj["major"] = os.major(entry.st.st_rdev) + obj["minor"] = os.minor(entry.st.st_rdev) + if kind != "lnk": # the format gives a symlink no mode + obj["mode"] = stat.S_IMODE(entry.st.st_mode) + obj["uid"] = entry.st.st_uid + obj["gid"] = entry.st.st_gid + obj["mTimeNSec"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) + if entry.st.st_mtime_ns < 0: + fail(f"manifest: {entry.name!r} has a timestamp before the UNIX epoch") + if token is not None: + obj["inodeToken"] = token + if kind == "lnk": + # A symlink target has no file of its own to point at, and the format recommends carrying + # it inline; base64 also spares it the UTF-8 rule a name has to satisfy. + obj["contents"] = [{"literal": base64.b64encode(target).decode("ascii")}] + if kind == "reg": + # No `contents`: for a regular file the format then implies the file of that name beside + # the manifest, which is what this manifest describes. + obj["sha256"] = _digest(entry.path) + return obj + + +def _record(obj: Object) -> bytes: + return ( + _RS + + json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + _LF + ) + + +def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: + """The whole sequence for `tree`: the root object, then one object per inode below it.""" + # The root object carries the media type alone, which is what marks the sequence a manifest. + # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the + # format defaults a nameless first object to a directory anyway. + yield _record({"mediaType": MEDIA_TYPE}) + found = list(entries(tree, roots)) + tokens = hardlink_tokens(found) + for entry in found: + yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode))) + yield _record({"mediaType": TRAILER_TYPE}) + + +def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: + written = 0 + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("wb") as stream: + for record in records(tree, epoch, roots): + stream.write(record) + written += 1 + return written + + +def read(source: Path) -> list[Object]: + """Read a manifest that `write` or `write_objects` produced. + + Only a complete manifest ends with the trailer, so a manifest that a failed write cut short fails + here instead of becoming a snapshot. + """ + records = source.read_bytes().split(_RS)[1:] + if not records or records[-1] != _record({"mediaType": TRAILER_TYPE})[1:]: + fail(f"manifest: {source} is incomplete: missing trailer") + return [json.loads(record) for record in records[:-1]] + + +def write_objects(objects: Sequence[Object], out: Path) -> int: + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("wb") as stream: + for obj in objects: + stream.write(_record(obj)) + stream.write(_record({"mediaType": TRAILER_TYPE})) + return len(objects) + 1 + + +def order(name: str) -> list[bytes]: + """Sort a name before the contents it holds, which sorting the names themselves does not. + + A manifest is in pre-order, and "dir/x" belongs directly after "dir" even though "dir.txt" + sorts between the two: the components decide, not the bytes of the whole name. + """ + return [os.fsencode(component) for component in name.split("/")] + + +def merge(sources: Sequence[Path], out: Path) -> int: + """Write the listing of a whole from the listings of the parts it is assembled from. + + A disk is filled one partition at a time, so what it carries is what its partitions carry + between them. A name more than one of them lists is taken from the last, which is the partition + mounted over a directory the others merely hold. + """ + objects: dict[str, Object] = {} + offset = 0 + for source in sources: + highest = 0 + for obj in read(source)[1:]: + token = obj.get("inodeToken") + if isinstance(token, int): + # A token numbers an inode within one manifest, so each part keeps its own grouping + # by carrying on where the part before it left off. + highest = max(highest, token) + obj["inodeToken"] = token + offset + objects[str(obj["name"])] = obj + offset += highest + + tokens: dict[int, int] = {} + counts = Counter(obj.get("inodeToken") for obj in objects.values() if obj.get("inodeToken")) + ordered = [objects[name] for name in sorted(objects, key=order)] + for obj in ordered: + token = obj.pop("inodeToken", None) + if isinstance(token, int) and counts[token] > 1: + obj["inodeToken"] = tokens.setdefault(token, len(tokens) + 1) + return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out) diff --git a/image/manifest_test.py b/uapi16/uapi16_test.py similarity index 81% rename from image/manifest_test.py rename to uapi16/uapi16_test.py index 5375e9ae..06d6ed42 100644 --- a/image/manifest_test.py +++ b/uapi16/uapi16_test.py @@ -3,7 +3,7 @@ """Tests for the UAPI.16 file manifest. -buck test tine//image:test +buck test tine//uapi16:test """ import base64 @@ -17,7 +17,7 @@ from pathlib import Path from typing import override -import manifest +import uapi16 # The epoch every image action runs under (box/runtime.bzl); any value works, but using the real # one keeps the clamping assertions honest about what a build actually produces. @@ -33,16 +33,17 @@ def parse(data: bytes) -> list[dict[str, object]]: for record in rest.split(b"\x1e"): assert record.endswith(b"\n"), "a record is not terminated by a line feed" objects.append(json.loads(record.decode("utf-8"))) - return objects + assert objects[-1] == {"mediaType": uapi16.TRAILER_TYPE} + return objects[:-1] class TreeTest(unittest.TestCase): @override def setUp(self) -> None: - self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest."))) + self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest."))) def manifest(self, tree: Path | None = None, roots: Sequence[str] = ()) -> list[dict[str, object]]: - return parse(b"".join(manifest.records(tree or self.tree, EPOCH, roots))) + return parse(b"".join(uapi16.records(tree or self.tree, EPOCH, roots))) def named(self, tree: Path | None = None) -> dict[str, dict[str, object]]: return {str(obj["name"]): obj for obj in self.manifest(tree)[1:]} @@ -54,18 +55,20 @@ def test_the_first_object_is_the_root_and_names_the_media_type(self) -> None: root = self.manifest()[0] self.assertEqual(root, {"mediaType": "application/vnd.uapi.16.manifest"}) - def test_an_empty_tree_is_the_root_object_alone(self) -> None: + def test_an_empty_tree_has_no_named_objects(self) -> None: + # An empty filesystem still has its root and trailer, with no named file objects. self.assertEqual(len(self.manifest()), 1) def test_each_record_is_framed_by_a_separator_and_a_line_feed(self) -> None: + # A tree with one file needs three JSON-SEQ records: the root, file and trailer. (self.tree / "a").write_text("a") - data = b"".join(manifest.records(self.tree, EPOCH)) - self.assertEqual(data.count(b"\x1e"), 2) + data = b"".join(uapi16.records(self.tree, EPOCH)) + self.assertEqual(data.count(b"\x1e"), 3) self.assertTrue(data.endswith(b"\n")) def test_the_json_carries_no_gratuitous_whitespace(self) -> None: (self.tree / "a").write_text("a") - self.assertNotIn(b" ", b"".join(manifest.records(self.tree, EPOCH))) + self.assertNotIn(b" ", b"".join(uapi16.records(self.tree, EPOCH))) class TestOrder(TreeTest): @@ -92,14 +95,14 @@ def test_the_bytes_do_not_depend_on_the_order_the_tree_was_created_in(self) -> N names = ["gamma", "alpha", "beta", "delta"] outputs = [] for order in (names, list(reversed(names))): - with tempfile.TemporaryDirectory(prefix="manifest.") as scratch: + with tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.") as scratch: tree = Path(scratch) for name in order: (tree / name).mkdir() (tree / name / "file").write_text(name) os.utime(tree / name / "file", (EPOCH - 1, EPOCH - 1)) os.utime(tree / name, (EPOCH - 1, EPOCH - 1)) - outputs.append(b"".join(manifest.records(tree, EPOCH))) + outputs.append(b"".join(uapi16.records(tree, EPOCH))) self.assertEqual(outputs[0], outputs[1]) @@ -193,7 +196,9 @@ def test_a_file_reached_by_one_name_alone_carries_no_token(self) -> None: def test_a_link_count_above_one_is_not_enough_when_the_other_name_is_outside_the_tree( self, ) -> None: - outside = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.outside."))) + outside = Path( + self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.outside.")) + ) (outside / "one").write_text("shared") os.link(outside / "one", self.tree / "inside") self.assertEqual((self.tree / "inside").stat().st_nlink, 2) @@ -229,24 +234,24 @@ def test_a_unix_socket_is_listed_as_one(self) -> None: def test_the_inode_type_names_follow_the_format(self) -> None: self.assertEqual( - [manifest.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], + [uapi16.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], ["blk", "sock", "lnk"], ) def test_an_inode_type_with_no_representation_fails_rather_than_being_guessed(self) -> None: with self.assertRaises(SystemExit): - manifest.inode_type(0) + uapi16.inode_type(0) class TestDeviceNodes(unittest.TestCase): """Device nodes, from a fabricated inode: this sandbox refuses mknod.""" - def device(self, mode: int, rdev: int) -> dict[str, object]: + def device(self, mode: int, rdev: int) -> uapi16.Object: st = os.stat_result( (mode, 0, 0, 1, 0, 0, 0, 0, EPOCH - 1, 0), {"st_rdev": rdev, "st_mtime_ns": (EPOCH - 1) * 10**9}, ) - return manifest.file_object(manifest.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) + return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -> None: self.assertEqual( @@ -259,7 +264,7 @@ def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) - "mode": 0o666, "uid": 0, "gid": 0, - "mTime": (EPOCH - 1) * 10**9, + "mTimeNSec": (EPOCH - 1) * 10**9, }, ) @@ -273,12 +278,12 @@ class TestTimestamps(TreeTest): def test_an_mtime_past_the_epoch_is_clamped_onto_it(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", (EPOCH + 10, EPOCH + 10)) - self.assertEqual(self.named()["file"]["mTime"], EPOCH * 10**9) + self.assertEqual(self.named()["file"]["mTimeNSec"], EPOCH * 10**9) def test_an_mtime_before_the_epoch_survives_in_nanoseconds(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", ns=(0, (EPOCH - 5) * 10**9 + 123)) - self.assertEqual(self.named()["file"]["mTime"], (EPOCH - 5) * 10**9 + 123) + self.assertEqual(self.named()["file"]["mTimeNSec"], (EPOCH - 5) * 10**9 + 123) class TestUnencodableNames(TreeTest): @@ -304,28 +309,32 @@ def test_a_reserved_suffix_at_the_top_level_is_refused_too(self) -> None: class TestWrite(TreeTest): - def test_the_written_file_is_the_sequence_and_the_count_includes_the_root(self) -> None: + def test_the_count_includes_the_root_and_trailer(self) -> None: + # Writing one file must report three records and finish with the mandatory trailer. (self.tree / "file").write_text("x") - out = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.out."))) / "m" - self.assertEqual(manifest.write(self.tree, out, EPOCH), 2) + out = ( + Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.out."))) + / "m" + ) + self.assertEqual(uapi16.write(self.tree, out, EPOCH), 3) self.assertEqual([obj.get("name") for obj in parse(out.read_bytes())], [None, "file"]) class TestScope(unittest.TestCase): def test_a_leading_and_trailing_slash_is_not_part_of_the_name(self) -> None: - self.assertEqual(manifest.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) + self.assertEqual(uapi16.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) def test_the_same_root_named_twice_is_listed_once(self) -> None: - self.assertEqual(manifest.scope(["/usr", "usr/"]), ["usr"]) + self.assertEqual(uapi16.scope(["/usr", "usr/"]), ["usr"]) def test_a_root_below_another_is_dropped_rather_than_listed_twice(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) + self.assertEqual(uapi16.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) def test_a_name_that_merely_shares_a_prefix_is_kept(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) + self.assertEqual(uapi16.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) def test_the_whole_tree_covers_every_other_root(self) -> None: - self.assertEqual(manifest.scope(["/", "/usr"]), [""]) + self.assertEqual(uapi16.scope(["/", "/usr"]), [""]) class TestScopedTree(TreeTest): @@ -361,7 +370,7 @@ def test_no_roots_is_the_whole_tree(self) -> None: class TestMerge(TreeTest): def part(self, name: str, roots: Sequence[str]) -> Path: out = self.tree / name - manifest.write(self.tree / "parts", out, EPOCH, roots) + uapi16.write(self.tree / "parts", out, EPOCH, roots) return out @override @@ -374,7 +383,7 @@ def setUp(self) -> None: def merged(self, *sources: Path) -> list[dict[str, object]]: out = self.tree / "merged" - manifest.merge(sources, out) + uapi16.merge(sources, out) return parse(out.read_bytes()) def test_the_whole_is_every_part_under_one_root_object(self) -> None: @@ -418,5 +427,33 @@ def test_something_that_is_not_a_manifest_is_refused(self) -> None: self.merged(self.tree / "junk") +class TestReader(TreeTest): + def test_requires_trailer(self) -> None: + # A write that stops early leaves a manifest without its trailer, which must fail instead of + # becoming a snapshot. A cut after the root object or inside the trailer must both fail. + out = self.tree / "manifest" + root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n' + trailer = b'\x1e{"mediaType":"application/vnd.uapi.16.trailer"}\n' + for data in (root, root + trailer[:-1]): + out.write_bytes(data) + with self.subTest(data=data), self.assertRaises(SystemExit): + uapi16.read(out) + out.write_bytes(root + trailer) + self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}]) + + def test_accepts_root_metadata(self) -> None: + # A storage manifest includes root metadata, which must survive parsing unchanged. + out = self.tree / "manifest" + objects: list[uapi16.Object] = [{"mediaType": uapi16.MEDIA_TYPE, "mode": 0o750}] + uapi16.write_objects(objects, out) + self.assertEqual(uapi16.read(out), objects) + + def test_rejects_non_relative_names(self) -> None: + # UAPI.16 forbids path traversal, unnormalized names and control characters. + for name in ("", "/a", "a/", "a//b", "a/../b", "./a", "a/.", "a/..", "a\n"): + with self.subTest(name=name), self.assertRaises(SystemExit): + uapi16.check_name(name) + + if __name__ == "__main__": unittest.main()