diff --git a/docs/design/architecture.md b/docs/design/architecture.md
index c7a4e71c..fcbae251 100644
--- a/docs/design/architecture.md
+++ b/docs/design/architecture.md
@@ -53,6 +53,8 @@ tine//box/ box bootstrap and sandbox command construction
tine//rootfs/ bind/overlay mounting and stored-delta translation
tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and
drivers, and VM runners
+tine//uapi16/ UAPI.16 manifests
+tine//archive/ tar writer, cpio reader and writer
tine//git/ pinned Git repositories with local checkout overrides
tine//cargo/ vendored crate trees and offline Rust source builds
tine//go/ go.sum-verified module fetches and offline Go source builds
diff --git a/examples/image-secureboot/BUCK b/examples/image-secureboot/BUCK
index 17266ad2..a36ea053 100644
--- a/examples/image-secureboot/BUCK
+++ b/examples/image-secureboot/BUCK
@@ -314,13 +314,15 @@ box.sh_test(
name = "disk-manifest-test",
labels = ["image"],
box = "@tine//tools:dev.box",
- test = "@tine//tests:contains.sh",
- args = [
- "$(location :image[disk-manifest])",
- '"name":"usr/bin/bash"',
- '"name":"boot/tine-tmpfiles","type":"dir","mode":448',
- '"name":"boot/EFI/Linux',
- ],
+ args = ["$(location :image[disk-manifest])"],
+ script = """
+# Authored tmpfiles must set the boot directory mode in the disk's filesystem listing.
+jq --seq -es '
+ any(.[]; .name == "usr/bin/bash") and
+ any(.[]; .name == "boot/tine-tmpfiles" and .type == "dir" and .mode == 448) and
+ any(.[]; .name | strings | startswith("boot/EFI/Linux"))
+' "$1"
+""",
)
# The kernel install hook is bypassed, it didn't generate its own initrd.
diff --git a/examples/image/expected.json b/examples/image/expected.json
index a73b5a37..56c4e769 100644
--- a/examples/image/expected.json
+++ b/examples/image/expected.json
@@ -16,13 +16,13 @@
"sha256": "64b78fbe92ff5fcaefb77ef10923c964e402304a54b0a776a8ca11c9b43c85f7"
},
"boot-demo.fedora[disk-manifest]": {
- "sha256": "c95655cea99dc0f17d5d4c6211595a8ba18728f94c3513a1695220de04c14e89"
+ "sha256": "b4f4144aed4ede4dec269912df3de2f1b93d8259eb293c8558822f8853e8f10f"
},
"boot-demo.fedora[manifests][esp]": {
- "sha256": "f81e7f963bbe9cfaf07ac4495a0d8e47392206e9ddb561943304bbb0ae3aa38a"
+ "sha256": "1894ce1b990d880e6dff777433eeb282359225199b853ee16431485de0321906"
},
"boot-demo.fedora[manifests][usr]": {
- "sha256": "0c28ac6ca9ee9ded432b4385ed5ce5400ea349324bcbff17323b61fd48a9f841"
+ "sha256": "b5d0bc2a053559673d2bbc5fd4262d5f2086c44bc1786a72bce76a8b086e6720"
},
"boot-demo.fedora[partitions][usr-verity]": {
"sha256": "bdff57265eca86d1a93ef53c19d88681e715bed6385ddca2a2f13cfff8766175"
@@ -54,13 +54,13 @@
"sha256": "84ab38be46148a1d40243767cdc749b0dffa32bd789814b160339efd2d3d8046"
},
"boot-demo.fedora[disk-manifest]": {
- "sha256": "7fbf96d7c1c343d54d0e1ef6b8c7fd4209fb6ea8656f72639fd699808ae79780"
+ "sha256": "eaa180ad12781b3ec5bb94a47595443191c7bcb19842b549e120200ee12a1976"
},
"boot-demo.fedora[manifests][esp]": {
- "sha256": "549e4e6ccd6e57b8b553dc7d7fabbee1be80b89889f9d65edbb50cdb91c806d3"
+ "sha256": "9e413eccc8894a4f9c30ac16d690b577156c703b9a87c5b4ca664e912493a2ba"
},
"boot-demo.fedora[manifests][usr]": {
- "sha256": "f6450f8f8a1e832c420e7409809a7b2ff924be69e0eb0155ed7fe4233d437cc7"
+ "sha256": "d43c2e1f98613d48a86dbf412ed8c051ed434890ca7fa73d1ae8dd9dbff4aeff"
},
"boot-demo.fedora[partitions][usr-verity]": {
"sha256": "6636ed1748dfdf35d0692b9c8d4ab0c50e5112e41bc91de3ba0950230f4fee41"
diff --git a/examples/image/manifest_check.py b/examples/image/manifest_check.py
index 4cd53714..f96a097b 100644
--- a/examples/image/manifest_check.py
+++ b/examples/image/manifest_check.py
@@ -27,7 +27,8 @@ def decode(data: bytes) -> list[dict[str, Any]]:
objects = decode(Path(sys.argv[1]).read_bytes())
-root, files = objects[0], objects[1:]
+assert objects[-1] == {"mediaType": "application/vnd.uapi.16.trailer"}, "missing trailer"
+root, files = objects[0], objects[1:-1]
assert root == {"mediaType": "application/vnd.uapi.16.manifest"}, f"bad root object: {root}"
assert files, "the manifest lists no files at all"
diff --git a/image/BUCK b/image/BUCK
index 9e6a2c58..b3a285f6 100644
--- a/image/BUCK
+++ b/image/BUCK
@@ -57,10 +57,10 @@ python.binary(
main = "ddi.py",
deps = [
":finalize",
- ":manifest-lib",
"//:specs",
"//:util",
"//rootfs:rootfs",
+ "//uapi16:uapi16",
],
visibility = ["PUBLIC"],
)
@@ -124,8 +124,6 @@ python.test(
"finalize_test.py",
"layer.py",
"layer_test.py",
- "manifest.py",
- "manifest_test.py",
"preload.py",
"preload_test.py",
"substitute.py",
@@ -145,6 +143,7 @@ python.test(
"//kmod:kmod",
"//package:installer",
"//rootfs:rootfs",
+ "//uapi16:uapi16",
],
)
@@ -165,21 +164,11 @@ python.binary(
":finalize",
"//:specs",
"//:util",
+ "//uapi16:uapi16",
],
visibility = ["PUBLIC"],
)
-# A driver that packages a tree lists it from inside its own action, where the tree is final.
-python.library(
- name = "manifest-lib",
- srcs = ["manifest.py"],
- deps = [
- ":finalize",
- "//:specs",
- "//:util",
- ],
-)
-
python.library(
name = "finalize",
srcs = ["finalize.py"],
diff --git a/image/ddi.py b/image/ddi.py
index 244ff3ba..df6685c5 100755
--- a/image/ddi.py
+++ b/image/ddi.py
@@ -21,8 +21,8 @@
import util
import finalize
-import manifest
import rootfs
+import uapi16
_SEED_NAMESPACE = uuid.UUID("5af2de99-4f9f-4e0b-a04b-bde36b068c4f")
@@ -511,7 +511,7 @@ def main(argv: list[str] | None = None) -> None:
listings = []
for partition in written:
if partition.manifest is not None:
- manifest.write(tree, partition.manifest, epoch, roots=partition.definition.sources)
+ uapi16.write(tree, partition.manifest, epoch, roots=partition.definition.sources)
listings.append(partition.manifest)
# An imported partition was listed by the invocation that filled it, and this one holds
@@ -519,7 +519,7 @@ def main(argv: list[str] | None = None) -> None:
# partition this invocation mounts over a directory another one holds is the one on top.
if spec["manifest"]:
imported = [it.manifest for it in partitions if it.manifest is not None]
- manifest.merge(imported + listings, Path(spec["manifest"]))
+ uapi16.merge(imported + listings, Path(spec["manifest"]))
scratch = Path(scratch_dir)
repart_definitions = scratch / "repart.d"
diff --git a/image/manifest.py b/image/manifest.py
index 5dfd8189..d3b69f84 100755
--- a/image/manifest.py
+++ b/image/manifest.py
@@ -3,298 +3,29 @@
# SPDX-FileCopyrightText: Amutable GmbH
# SPDX-License-Identifier: MPL-2.0
-"""List everything a logical image ships, as a UAPI.16 File Manifest.
+"""Export the finalized image's UAPI.16 listing."""
-The format is an RFC7464 JSON-SEQ sequence: a root object naming the media type, then one object
-per inode, each directory immediately followed by its own contents. Two consumers want this: a
-comparison of the paths two images ship (a path in both a system extension and the /usr it merges
-onto silently shadows an OS file), and per-image size accounting. Both need every inode type an OS
-tree holds, so a type that cannot be listed is an error rather than an omission.
-
-A driver writing a listing of what it is about to package imports this rather than running it: only
-the driver holds the finished tree, and only it knows which paths of that tree its artifact carries.
-`roots` is how it says so, and names stay relative to the image, so a listing reads as where each
-path lands on a machine rather than where it sits inside one partition.
-"""
-
-import base64
-import hashlib
-import json
import os
-import stat
import sys
-from collections import Counter
-from collections.abc import Iterator, Sequence
-from dataclasses import dataclass
from pathlib import Path
-from typing import cast
import specs
-from util import fail
import finalize
+import uapi16
class Spec(finalize.ImageSpec):
out: str
-MEDIA_TYPE = "application/vnd.uapi.16.manifest"
-
-# RFC7464 frames every JSON text with a leading record separator and a trailing line feed.
-_RS = b"\x1e"
-_LF = b"\n"
-
-# A directory holding a manifest cannot also list it, so the format reserves the name at the top
-# level; deeper in the tree it is an ordinary file name.
-_RESERVED = "Uapi16Manifest"
-
-_INODE_TYPES = (
- (stat.S_ISREG, "reg"),
- (stat.S_ISDIR, "dir"),
- (stat.S_ISLNK, "lnk"),
- (stat.S_ISFIFO, "fifo"),
- (stat.S_ISCHR, "chr"),
- (stat.S_ISBLK, "blk"),
- (stat.S_ISSOCK, "sock"),
-)
-
-_NANOSECONDS = 10**9
-
-
-@dataclass(frozen=True)
-class Entry:
- """One inode below the image root, under the name the manifest gives it."""
-
- name: str
- path: Path
- st: os.stat_result
-
- @property
- def inode(self) -> tuple[int, int]:
- return (self.st.st_dev, self.st.st_ino)
-
-
-def inode_type(mode: int) -> str:
- for predicate, name in _INODE_TYPES:
- if predicate(mode):
- return name
- fail(f"manifest: inode type {stat.S_IFMT(mode):#o} has no manifest representation")
-
-
-def check_name(name: str) -> str:
- """Reject a path the format cannot encode, rather than listing the image incompletely.
-
- Nothing normalizes here: scandir yields neither "." nor ".." nor an empty component, so what
- is left is the format's own restrictions on the bytes and on the reserved file name.
- """
- try:
- name.encode("utf-8")
- except UnicodeEncodeError:
- fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be")
- if any(ord(character) < 32 or ord(character) == 127 for character in name):
- fail(f"manifest: {name!r} contains a control character")
- top = name.split("/", 1)[0]
- if top == _RESERVED or top.startswith(_RESERVED + "."):
- fail(f"manifest: {name!r} is the reserved manifest file name")
- return name
-
-
-def scope(roots: Sequence[str]) -> list[str]:
- """The image paths to list, as manifest names: deduplicated, ordered, and non-overlapping.
-
- A path already covered by another is dropped rather than listed twice, which is what a partition
- copying both a directory and something below it would otherwise produce.
- """
- names = sorted({root.strip("/") for root in roots}, key=os.fsencode)
- if "" in names: # the whole tree, which covers every other root by definition
- return [""]
- kept: list[str] = []
- for name in names:
- if not any(name == covered or name.startswith(covered + "/") for covered in kept):
- kept.append(name)
- return kept
-
-
-def entries(tree: Path, roots: Sequence[str] = ()) -> Iterator[Entry]:
- """Yield every inode below `tree` in manifest order, or only those `roots` reach.
-
- Pre-order, because the format wants a directory's contents to follow it immediately, and sorted
- on the raw name bytes, which is an order neither the locale nor the filesystem can shift.
-
- A root the image does not have is skipped rather than refused: repart copies what a definition
- names if it is there, so the ESP definition can name both /boot and /efi for images that have
- one or the other.
- """
- for name in scope(roots) if roots else [""]:
- if name == "":
- yield from _below(tree, "")
- continue
- path = tree / name
- if not path.is_symlink() and not path.exists():
- continue
- st = path.lstat()
- yield Entry(name=check_name(name), path=path, st=st)
- if stat.S_ISDIR(st.st_mode):
- yield from _below(path, name + "/")
-
-
-def _below(directory: Path, prefix: str) -> Iterator[Entry]:
- with os.scandir(directory) as scan:
- children = sorted(scan, key=lambda child: os.fsencode(child.name))
- for child in children:
- name = check_name(prefix + child.name)
- st = child.stat(follow_symlinks=False)
- yield Entry(name=name, path=Path(child.path), st=st)
- if stat.S_ISDIR(st.st_mode):
- yield from _below(Path(child.path), name + "/")
-
-
-def hardlink_tokens(found: Sequence[Entry]) -> dict[tuple[int, int], int]:
- """Number the inodes the manifest reaches by more than one name, in the order it reaches them.
-
- A link count above one does not make a file hardlinked *here*: the other names may sit outside
- the image, and a directory's count is its subdirectories. Only a second name in the manifest
- does, so the tokens count reachable names rather than trusting st_nlink.
- """
- reachable = Counter(
- entry.inode for entry in found if entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode)
- )
- tokens: dict[tuple[int, int], int] = {}
- for entry in found:
- if reachable[entry.inode] > 1 and entry.inode not in tokens:
- tokens[entry.inode] = len(tokens) + 1
- return tokens
-
-
-def _digest(path: Path) -> str:
- with path.open("rb") as stream:
- return hashlib.file_digest(stream, "sha256").hexdigest()
-
-
-def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[str, object]:
- """The record for one inode, with its fields in the order the specification lists them."""
- kind = inode_type(entry.st.st_mode)
- obj: dict[str, object] = {"name": entry.name, "type": kind}
- target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b""
-
- if kind == "reg":
- obj["size"] = entry.st.st_size
- elif kind == "lnk":
- obj["size"] = len(target)
- elif kind in ("chr", "blk"):
- obj["major"] = os.major(entry.st.st_rdev)
- obj["minor"] = os.minor(entry.st.st_rdev)
- if kind != "lnk": # the format gives a symlink no mode
- obj["mode"] = stat.S_IMODE(entry.st.st_mode)
- obj["uid"] = entry.st.st_uid
- obj["gid"] = entry.st.st_gid
- # Clamped exactly as every other reproducible output here clamps: what a package stamped
- # survives, what this build stamped collapses onto the epoch.
- obj["mTime"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS)
- if token is not None:
- obj["inodeToken"] = token
- if kind == "lnk":
- # A symlink target has no file of its own to point at, and the format recommends carrying
- # it inline; base64 also spares it the UTF-8 rule a name has to satisfy.
- obj["contents"] = [{"literal": base64.b64encode(target).decode("ascii")}]
- if kind == "reg":
- # No `contents`: for a regular file the format then implies the file of that name beside
- # the manifest, which is what this manifest describes.
- obj["sha256"] = _digest(entry.path)
- return obj
-
-
-def _record(obj: dict[str, object]) -> bytes:
- return _RS + json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + _LF
-
-
-def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]:
- """The whole sequence for `tree`: the root object, then one object per inode below it."""
- # The root object carries the media type alone, which is what marks the sequence a manifest.
- # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the
- # format defaults a nameless first object to a directory anyway.
- yield _record({"mediaType": MEDIA_TYPE})
- found = list(entries(tree, roots))
- tokens = hardlink_tokens(found)
- for entry in found:
- yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode)))
-
-
-def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int:
- written = 0
- out.parent.mkdir(parents=True, exist_ok=True)
- with out.open("wb") as stream:
- for record in records(tree, epoch, roots):
- stream.write(record)
- written += 1
- return written
-
-
-def _objects(source: Path) -> Iterator[dict[str, object]]:
- """The named objects of a manifest, the root object it opens with aside."""
- data = source.read_bytes()
- head, separator, rest = data.partition(_RS)
- if not separator or head:
- fail(f"manifest: {source} does not open with a record")
- for index, record in enumerate(rest.split(_RS)):
- if not record.endswith(_LF):
- fail(f"manifest: {source} holds a record that no line feed ends")
- obj = cast(dict[str, object], json.loads(record))
- if index == 0:
- if obj != {"mediaType": MEDIA_TYPE}:
- fail(f"manifest: {source} does not open with a {MEDIA_TYPE} object")
- continue
- yield obj
-
-
-def order(name: str) -> list[bytes]:
- """Sort a name before the contents it holds, which sorting the names themselves does not.
-
- A manifest is in pre-order, and "dir/x" belongs directly after "dir" even though "dir.txt"
- sorts between the two: the components decide, not the bytes of the whole name.
- """
- return [os.fsencode(component) for component in name.split("/")]
-
-
-def merge(sources: Sequence[Path], out: Path) -> int:
- """Write the listing of a whole from the listings of the parts it is assembled from.
-
- A disk is filled one partition at a time, so what it carries is what its partitions carry
- between them. A name more than one of them lists is taken from the last, which is the partition
- mounted over a directory the others merely hold.
- """
- objects: dict[str, dict[str, object]] = {}
- offset = 0
- for source in sources:
- highest = 0
- for obj in _objects(source):
- token = obj.get("inodeToken")
- if isinstance(token, int):
- # A token numbers an inode within one manifest, so each part keeps its own grouping
- # by carrying on where the part before it left off.
- highest = max(highest, token)
- obj["inodeToken"] = token + offset
- objects[str(obj["name"])] = obj
- offset += highest
-
- written = 1
- out.parent.mkdir(parents=True, exist_ok=True)
- with out.open("wb") as stream:
- stream.write(_record({"mediaType": MEDIA_TYPE}))
- for name in sorted(objects, key=order):
- stream.write(_record(objects[name]))
- written += 1
- return written
-
-
def main(argv: list[str] | None = None) -> None:
spec = specs.parse(Spec, "manifest", argv)
out = Path(spec["out"])
epoch = int(os.environ["SOURCE_DATE_EPOCH"])
with finalize.image(spec, program="manifest") as tree:
- written = write(tree, out, epoch)
+ written = uapi16.write(tree, out, epoch)
print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr)
diff --git a/ruff.toml b/ruff.toml
index 5ddc2873..5677281c 100644
--- a/ruff.toml
+++ b/ruff.toml
@@ -21,6 +21,7 @@ src = [
"package_system/rpm",
"remote_cache",
"rootfs",
+ "uapi16",
]
lint.select = [
"E", # pycodestyle
diff --git a/uapi16/BUCK b/uapi16/BUCK
new file mode 100644
index 00000000..98170b32
--- /dev/null
+++ b/uapi16/BUCK
@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: Amutable GmbH
+# SPDX-License-Identifier: MPL-2.0
+
+"""UAPI.16 manifests and the snapshot stores built on them."""
+
+load("//python:defs.bzl", "python")
+
+python.library(
+ name = "uapi16",
+ srcs = ["uapi16.py"],
+ deps = ["//:util"],
+ visibility = ["tine//..."],
+)
+
+python.test(
+ name = "test",
+ box = "//tools:dev.box",
+ srcs = ["uapi16_test.py"],
+ deps = [":uapi16", "//:util"],
+)
diff --git a/uapi16/pyproject.toml b/uapi16/pyproject.toml
new file mode 100644
index 00000000..a0e58bb2
--- /dev/null
+++ b/uapi16/pyproject.toml
@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Amutable GmbH
+# SPDX-License-Identifier: MPL-2.0
+
+# Zed needs a Python manifest here to route these files to their own ty instance.
diff --git a/uapi16/uapi16.py b/uapi16/uapi16.py
new file mode 100644
index 00000000..6f588465
--- /dev/null
+++ b/uapi16/uapi16.py
@@ -0,0 +1,302 @@
+# SPDX-FileCopyrightText: Amutable GmbH
+# SPDX-License-Identifier: MPL-2.0
+
+"""Read, write and combine UAPI.16 filesystem listings."""
+
+import base64
+import hashlib
+import json
+import os
+import stat
+from collections import Counter
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TypedDict
+
+from util import fail
+
+MEDIA_TYPE = "application/vnd.uapi.16.manifest"
+TRAILER_TYPE = "application/vnd.uapi.16.trailer"
+
+# RFC7464 frames every JSON text with a leading record separator and a trailing line feed.
+_RS = b"\x1e"
+_LF = b"\n"
+
+# A directory holding a manifest cannot also list it, so the format reserves the name at the top
+# level; deeper in the tree it is an ordinary file name.
+MANIFEST = "Uapi16Manifest"
+
+_INODE_TYPES = (
+ (stat.S_ISREG, "reg"),
+ (stat.S_ISDIR, "dir"),
+ (stat.S_ISLNK, "lnk"),
+ (stat.S_ISFIFO, "fifo"),
+ (stat.S_ISCHR, "chr"),
+ (stat.S_ISBLK, "blk"),
+ (stat.S_ISSOCK, "sock"),
+)
+
+_NANOSECONDS = 10**9
+
+
+class Content(TypedDict, total=False):
+ file: str
+ literal: str
+
+
+class Xattr(TypedDict, total=False):
+ text: str
+ base64: str
+
+
+class Object(TypedDict, total=False):
+ """One record of a UAPI.16 manifest."""
+
+ mediaType: str
+ name: str
+ type: str
+ mode: int
+ uid: int
+ gid: int
+ mTimeNSec: int
+ size: int
+ major: int
+ minor: int
+ sha256: str
+ contents: list[Content]
+ inodeToken: int
+ extendedAttributes: dict[str, dict[str, Xattr]]
+
+
+@dataclass(frozen=True)
+class Entry:
+ """One inode below the image root, under the name the manifest gives it."""
+
+ name: str
+ path: Path
+ st: os.stat_result
+
+ @property
+ def inode(self) -> tuple[int, int]:
+ return (self.st.st_dev, self.st.st_ino)
+
+
+def inode_type(mode: int) -> str:
+ for predicate, name in _INODE_TYPES:
+ if predicate(mode):
+ return name
+ fail(f"manifest: inode type {stat.S_IFMT(mode):#o} has no manifest representation")
+
+
+def check_name(name: str) -> str:
+ """Reject a path the format cannot encode, rather than listing the image incompletely."""
+ try:
+ name.encode("utf-8")
+ except UnicodeEncodeError:
+ fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be")
+ if any(ord(character) < 32 or ord(character) == 127 for character in name):
+ fail(f"manifest: {name!r} contains a control character")
+ if any(part in ("", ".", "..") for part in name.split("/")):
+ fail(f"manifest: {name!r} must be a normalized relative path")
+ top = name.split("/", 1)[0]
+ if top == MANIFEST or top.startswith(MANIFEST + "."):
+ fail(f"manifest: {name!r} is the reserved manifest file name")
+ return name
+
+
+def scope(roots: Sequence[str]) -> list[str]:
+ """The image paths to list, as manifest names: deduplicated, ordered, and non-overlapping.
+
+ A path already covered by another is dropped rather than listed twice, which is what a partition
+ copying both a directory and something below it would otherwise produce.
+ """
+ names = sorted({root.strip("/") for root in roots}, key=os.fsencode)
+ if "" in names: # the whole tree, which covers every other root by definition
+ return [""]
+ kept: list[str] = []
+ for name in names:
+ if not any(name == covered or name.startswith(covered + "/") for covered in kept):
+ kept.append(name)
+ return kept
+
+
+def entries(tree: Path, roots: Sequence[str] = ()) -> Iterator[Entry]:
+ """Yield every inode below `tree` in manifest order, or only those `roots` reach.
+
+ Pre-order, because the format wants a directory's contents to follow it immediately, and sorted
+ on the raw name bytes, which is an order neither the locale nor the filesystem can shift.
+
+ A root the image does not have is skipped rather than refused: repart copies what a definition
+ names if it is there, so the ESP definition can name both /boot and /efi for images that have
+ one or the other.
+ """
+ for name in scope(roots) if roots else [""]:
+ if name == "":
+ yield from _below(tree, "")
+ continue
+ path = tree / name
+ if not path.is_symlink() and not path.exists():
+ continue
+ st = path.lstat()
+ yield Entry(name=check_name(name), path=path, st=st)
+ if stat.S_ISDIR(st.st_mode):
+ yield from _below(path, name + "/")
+
+
+def _below(directory: Path, prefix: str) -> Iterator[Entry]:
+ with os.scandir(directory) as scan:
+ children = sorted(scan, key=lambda child: os.fsencode(child.name))
+ for child in children:
+ name = check_name(prefix + child.name)
+ st = child.stat(follow_symlinks=False)
+ yield Entry(name=name, path=Path(child.path), st=st)
+ if stat.S_ISDIR(st.st_mode):
+ yield from _below(Path(child.path), name + "/")
+
+
+def hardlink_tokens(found: Sequence[Entry]) -> dict[tuple[int, int], int]:
+ """Number the inodes the manifest reaches by more than one name, in the order it reaches them.
+
+ A link count above one does not make a file hardlinked *here*: the other names may sit outside
+ the image, and a directory's count is its subdirectories. Only a second name in the manifest
+ does, so the tokens count reachable names rather than trusting st_nlink.
+ """
+ reachable = Counter(
+ entry.inode for entry in found if entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode)
+ )
+ tokens: dict[tuple[int, int], int] = {}
+ for entry in found:
+ if reachable[entry.inode] > 1 and entry.inode not in tokens:
+ tokens[entry.inode] = len(tokens) + 1
+ return tokens
+
+
+def _digest(path: Path) -> str:
+ with path.open("rb") as stream:
+ return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object:
+ """Describe one inode using UAPI.16 metadata and content fields."""
+ kind = inode_type(entry.st.st_mode)
+ obj: Object = {"name": entry.name, "type": kind}
+ target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b""
+
+ if kind == "reg":
+ obj["size"] = entry.st.st_size
+ elif kind == "lnk":
+ obj["size"] = len(target)
+ elif kind in ("chr", "blk"):
+ obj["major"] = os.major(entry.st.st_rdev)
+ obj["minor"] = os.minor(entry.st.st_rdev)
+ if kind != "lnk": # the format gives a symlink no mode
+ obj["mode"] = stat.S_IMODE(entry.st.st_mode)
+ obj["uid"] = entry.st.st_uid
+ obj["gid"] = entry.st.st_gid
+ obj["mTimeNSec"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS)
+ if entry.st.st_mtime_ns < 0:
+ fail(f"manifest: {entry.name!r} has a timestamp before the UNIX epoch")
+ if token is not None:
+ obj["inodeToken"] = token
+ if kind == "lnk":
+ # A symlink target has no file of its own to point at, and the format recommends carrying
+ # it inline; base64 also spares it the UTF-8 rule a name has to satisfy.
+ obj["contents"] = [{"literal": base64.b64encode(target).decode("ascii")}]
+ if kind == "reg":
+ # No `contents`: for a regular file the format then implies the file of that name beside
+ # the manifest, which is what this manifest describes.
+ obj["sha256"] = _digest(entry.path)
+ return obj
+
+
+def _record(obj: Object) -> bytes:
+ return (
+ _RS
+ + json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+ + _LF
+ )
+
+
+def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]:
+ """The whole sequence for `tree`: the root object, then one object per inode below it."""
+ # The root object carries the media type alone, which is what marks the sequence a manifest.
+ # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the
+ # format defaults a nameless first object to a directory anyway.
+ yield _record({"mediaType": MEDIA_TYPE})
+ found = list(entries(tree, roots))
+ tokens = hardlink_tokens(found)
+ for entry in found:
+ yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode)))
+ yield _record({"mediaType": TRAILER_TYPE})
+
+
+def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int:
+ written = 0
+ out.parent.mkdir(parents=True, exist_ok=True)
+ with out.open("wb") as stream:
+ for record in records(tree, epoch, roots):
+ stream.write(record)
+ written += 1
+ return written
+
+
+def read(source: Path) -> list[Object]:
+ """Read a manifest that `write` or `write_objects` produced.
+
+ Only a complete manifest ends with the trailer, so a manifest that a failed write cut short fails
+ here instead of becoming a snapshot.
+ """
+ records = source.read_bytes().split(_RS)[1:]
+ if not records or records[-1] != _record({"mediaType": TRAILER_TYPE})[1:]:
+ fail(f"manifest: {source} is incomplete: missing trailer")
+ return [json.loads(record) for record in records[:-1]]
+
+
+def write_objects(objects: Sequence[Object], out: Path) -> int:
+ out.parent.mkdir(parents=True, exist_ok=True)
+ with out.open("wb") as stream:
+ for obj in objects:
+ stream.write(_record(obj))
+ stream.write(_record({"mediaType": TRAILER_TYPE}))
+ return len(objects) + 1
+
+
+def order(name: str) -> list[bytes]:
+ """Sort a name before the contents it holds, which sorting the names themselves does not.
+
+ A manifest is in pre-order, and "dir/x" belongs directly after "dir" even though "dir.txt"
+ sorts between the two: the components decide, not the bytes of the whole name.
+ """
+ return [os.fsencode(component) for component in name.split("/")]
+
+
+def merge(sources: Sequence[Path], out: Path) -> int:
+ """Write the listing of a whole from the listings of the parts it is assembled from.
+
+ A disk is filled one partition at a time, so what it carries is what its partitions carry
+ between them. A name more than one of them lists is taken from the last, which is the partition
+ mounted over a directory the others merely hold.
+ """
+ objects: dict[str, Object] = {}
+ offset = 0
+ for source in sources:
+ highest = 0
+ for obj in read(source)[1:]:
+ token = obj.get("inodeToken")
+ if isinstance(token, int):
+ # A token numbers an inode within one manifest, so each part keeps its own grouping
+ # by carrying on where the part before it left off.
+ highest = max(highest, token)
+ obj["inodeToken"] = token + offset
+ objects[str(obj["name"])] = obj
+ offset += highest
+
+ tokens: dict[int, int] = {}
+ counts = Counter(obj.get("inodeToken") for obj in objects.values() if obj.get("inodeToken"))
+ ordered = [objects[name] for name in sorted(objects, key=order)]
+ for obj in ordered:
+ token = obj.pop("inodeToken", None)
+ if isinstance(token, int) and counts[token] > 1:
+ obj["inodeToken"] = tokens.setdefault(token, len(tokens) + 1)
+ return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out)
diff --git a/image/manifest_test.py b/uapi16/uapi16_test.py
similarity index 81%
rename from image/manifest_test.py
rename to uapi16/uapi16_test.py
index 5375e9ae..06d6ed42 100644
--- a/image/manifest_test.py
+++ b/uapi16/uapi16_test.py
@@ -3,7 +3,7 @@
"""Tests for the UAPI.16 file manifest.
-buck test tine//image:test
+buck test tine//uapi16:test
"""
import base64
@@ -17,7 +17,7 @@
from pathlib import Path
from typing import override
-import manifest
+import uapi16
# The epoch every image action runs under (box/runtime.bzl); any value works, but using the real
# one keeps the clamping assertions honest about what a build actually produces.
@@ -33,16 +33,17 @@ def parse(data: bytes) -> list[dict[str, object]]:
for record in rest.split(b"\x1e"):
assert record.endswith(b"\n"), "a record is not terminated by a line feed"
objects.append(json.loads(record.decode("utf-8")))
- return objects
+ assert objects[-1] == {"mediaType": uapi16.TRAILER_TYPE}
+ return objects[:-1]
class TreeTest(unittest.TestCase):
@override
def setUp(self) -> None:
- self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.")))
+ self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.")))
def manifest(self, tree: Path | None = None, roots: Sequence[str] = ()) -> list[dict[str, object]]:
- return parse(b"".join(manifest.records(tree or self.tree, EPOCH, roots)))
+ return parse(b"".join(uapi16.records(tree or self.tree, EPOCH, roots)))
def named(self, tree: Path | None = None) -> dict[str, dict[str, object]]:
return {str(obj["name"]): obj for obj in self.manifest(tree)[1:]}
@@ -54,18 +55,20 @@ def test_the_first_object_is_the_root_and_names_the_media_type(self) -> None:
root = self.manifest()[0]
self.assertEqual(root, {"mediaType": "application/vnd.uapi.16.manifest"})
- def test_an_empty_tree_is_the_root_object_alone(self) -> None:
+ def test_an_empty_tree_has_no_named_objects(self) -> None:
+ # An empty filesystem still has its root and trailer, with no named file objects.
self.assertEqual(len(self.manifest()), 1)
def test_each_record_is_framed_by_a_separator_and_a_line_feed(self) -> None:
+ # A tree with one file needs three JSON-SEQ records: the root, file and trailer.
(self.tree / "a").write_text("a")
- data = b"".join(manifest.records(self.tree, EPOCH))
- self.assertEqual(data.count(b"\x1e"), 2)
+ data = b"".join(uapi16.records(self.tree, EPOCH))
+ self.assertEqual(data.count(b"\x1e"), 3)
self.assertTrue(data.endswith(b"\n"))
def test_the_json_carries_no_gratuitous_whitespace(self) -> None:
(self.tree / "a").write_text("a")
- self.assertNotIn(b" ", b"".join(manifest.records(self.tree, EPOCH)))
+ self.assertNotIn(b" ", b"".join(uapi16.records(self.tree, EPOCH)))
class TestOrder(TreeTest):
@@ -92,14 +95,14 @@ def test_the_bytes_do_not_depend_on_the_order_the_tree_was_created_in(self) -> N
names = ["gamma", "alpha", "beta", "delta"]
outputs = []
for order in (names, list(reversed(names))):
- with tempfile.TemporaryDirectory(prefix="manifest.") as scratch:
+ with tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.") as scratch:
tree = Path(scratch)
for name in order:
(tree / name).mkdir()
(tree / name / "file").write_text(name)
os.utime(tree / name / "file", (EPOCH - 1, EPOCH - 1))
os.utime(tree / name, (EPOCH - 1, EPOCH - 1))
- outputs.append(b"".join(manifest.records(tree, EPOCH)))
+ outputs.append(b"".join(uapi16.records(tree, EPOCH)))
self.assertEqual(outputs[0], outputs[1])
@@ -193,7 +196,9 @@ def test_a_file_reached_by_one_name_alone_carries_no_token(self) -> None:
def test_a_link_count_above_one_is_not_enough_when_the_other_name_is_outside_the_tree(
self,
) -> None:
- outside = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.outside.")))
+ outside = Path(
+ self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.outside."))
+ )
(outside / "one").write_text("shared")
os.link(outside / "one", self.tree / "inside")
self.assertEqual((self.tree / "inside").stat().st_nlink, 2)
@@ -229,24 +234,24 @@ def test_a_unix_socket_is_listed_as_one(self) -> None:
def test_the_inode_type_names_follow_the_format(self) -> None:
self.assertEqual(
- [manifest.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)],
+ [uapi16.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)],
["blk", "sock", "lnk"],
)
def test_an_inode_type_with_no_representation_fails_rather_than_being_guessed(self) -> None:
with self.assertRaises(SystemExit):
- manifest.inode_type(0)
+ uapi16.inode_type(0)
class TestDeviceNodes(unittest.TestCase):
"""Device nodes, from a fabricated inode: this sandbox refuses mknod."""
- def device(self, mode: int, rdev: int) -> dict[str, object]:
+ def device(self, mode: int, rdev: int) -> uapi16.Object:
st = os.stat_result(
(mode, 0, 0, 1, 0, 0, 0, 0, EPOCH - 1, 0),
{"st_rdev": rdev, "st_mtime_ns": (EPOCH - 1) * 10**9},
)
- return manifest.file_object(manifest.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH)
+ return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH)
def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -> None:
self.assertEqual(
@@ -259,7 +264,7 @@ def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -
"mode": 0o666,
"uid": 0,
"gid": 0,
- "mTime": (EPOCH - 1) * 10**9,
+ "mTimeNSec": (EPOCH - 1) * 10**9,
},
)
@@ -273,12 +278,12 @@ class TestTimestamps(TreeTest):
def test_an_mtime_past_the_epoch_is_clamped_onto_it(self) -> None:
(self.tree / "file").write_text("x")
os.utime(self.tree / "file", (EPOCH + 10, EPOCH + 10))
- self.assertEqual(self.named()["file"]["mTime"], EPOCH * 10**9)
+ self.assertEqual(self.named()["file"]["mTimeNSec"], EPOCH * 10**9)
def test_an_mtime_before_the_epoch_survives_in_nanoseconds(self) -> None:
(self.tree / "file").write_text("x")
os.utime(self.tree / "file", ns=(0, (EPOCH - 5) * 10**9 + 123))
- self.assertEqual(self.named()["file"]["mTime"], (EPOCH - 5) * 10**9 + 123)
+ self.assertEqual(self.named()["file"]["mTimeNSec"], (EPOCH - 5) * 10**9 + 123)
class TestUnencodableNames(TreeTest):
@@ -304,28 +309,32 @@ def test_a_reserved_suffix_at_the_top_level_is_refused_too(self) -> None:
class TestWrite(TreeTest):
- def test_the_written_file_is_the_sequence_and_the_count_includes_the_root(self) -> None:
+ def test_the_count_includes_the_root_and_trailer(self) -> None:
+ # Writing one file must report three records and finish with the mandatory trailer.
(self.tree / "file").write_text("x")
- out = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.out."))) / "m"
- self.assertEqual(manifest.write(self.tree, out, EPOCH), 2)
+ out = (
+ Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.out.")))
+ / "m"
+ )
+ self.assertEqual(uapi16.write(self.tree, out, EPOCH), 3)
self.assertEqual([obj.get("name") for obj in parse(out.read_bytes())], [None, "file"])
class TestScope(unittest.TestCase):
def test_a_leading_and_trailing_slash_is_not_part_of_the_name(self) -> None:
- self.assertEqual(manifest.scope(["/usr:", "/boot/"]), ["boot", "usr:"])
+ self.assertEqual(uapi16.scope(["/usr:", "/boot/"]), ["boot", "usr:"])
def test_the_same_root_named_twice_is_listed_once(self) -> None:
- self.assertEqual(manifest.scope(["/usr", "usr/"]), ["usr"])
+ self.assertEqual(uapi16.scope(["/usr", "usr/"]), ["usr"])
def test_a_root_below_another_is_dropped_rather_than_listed_twice(self) -> None:
- self.assertEqual(manifest.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"])
+ self.assertEqual(uapi16.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"])
def test_a_name_that_merely_shares_a_prefix_is_kept(self) -> None:
- self.assertEqual(manifest.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"])
+ self.assertEqual(uapi16.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"])
def test_the_whole_tree_covers_every_other_root(self) -> None:
- self.assertEqual(manifest.scope(["/", "/usr"]), [""])
+ self.assertEqual(uapi16.scope(["/", "/usr"]), [""])
class TestScopedTree(TreeTest):
@@ -361,7 +370,7 @@ def test_no_roots_is_the_whole_tree(self) -> None:
class TestMerge(TreeTest):
def part(self, name: str, roots: Sequence[str]) -> Path:
out = self.tree / name
- manifest.write(self.tree / "parts", out, EPOCH, roots)
+ uapi16.write(self.tree / "parts", out, EPOCH, roots)
return out
@override
@@ -374,7 +383,7 @@ def setUp(self) -> None:
def merged(self, *sources: Path) -> list[dict[str, object]]:
out = self.tree / "merged"
- manifest.merge(sources, out)
+ uapi16.merge(sources, out)
return parse(out.read_bytes())
def test_the_whole_is_every_part_under_one_root_object(self) -> None:
@@ -418,5 +427,33 @@ def test_something_that_is_not_a_manifest_is_refused(self) -> None:
self.merged(self.tree / "junk")
+class TestReader(TreeTest):
+ def test_requires_trailer(self) -> None:
+ # A write that stops early leaves a manifest without its trailer, which must fail instead of
+ # becoming a snapshot. A cut after the root object or inside the trailer must both fail.
+ out = self.tree / "manifest"
+ root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n'
+ trailer = b'\x1e{"mediaType":"application/vnd.uapi.16.trailer"}\n'
+ for data in (root, root + trailer[:-1]):
+ out.write_bytes(data)
+ with self.subTest(data=data), self.assertRaises(SystemExit):
+ uapi16.read(out)
+ out.write_bytes(root + trailer)
+ self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}])
+
+ def test_accepts_root_metadata(self) -> None:
+ # A storage manifest includes root metadata, which must survive parsing unchanged.
+ out = self.tree / "manifest"
+ objects: list[uapi16.Object] = [{"mediaType": uapi16.MEDIA_TYPE, "mode": 0o750}]
+ uapi16.write_objects(objects, out)
+ self.assertEqual(uapi16.read(out), objects)
+
+ def test_rejects_non_relative_names(self) -> None:
+ # UAPI.16 forbids path traversal, unnormalized names and control characters.
+ for name in ("", "/a", "a/", "a//b", "a/../b", "./a", "a/.", "a/..", "a\n"):
+ with self.subTest(name=name), self.assertRaises(SystemExit):
+ uapi16.check_name(name)
+
+
if __name__ == "__main__":
unittest.main()