diff --git a/Cargo.lock b/Cargo.lock index 7845d6cd69..32e258fed1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1714,6 +1714,14 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" +[[package]] +name = "mapprobe" +version = "0.1.0" +dependencies = [ + "toyos", + "toyos-abi", +] + [[package]] name = "matchers" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 5ae48789a2..04ca85a94a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ members = [ "tests/guest/osbench/compare", "tests/guest/osbench/lan", "tests/guest/osbench/rules", + "tests/guest/mapprobe", "tests/guest/pagerprobe", "tests/guest/pciholder", "tests/guest/starveling", diff --git a/build/ci.rs b/build/ci.rs index 468a0905d4..591cdb2632 100644 --- a/build/ci.rs +++ b/build/ci.rs @@ -587,6 +587,9 @@ pub(crate) const CONTROLS: &[Control] = &[ red(KERNEL_LOOM, "tlb-tags-resume-relaxed", Some("tlb_tags"), &[ Fails("a_change_passing_an_idle_holder_by_is_flushed_when_its_thread_resumes"), ]), + red(KERNEL_LOOM, "ranges-less-relaxed", Some("paged_ranges"), &[ + Fails("the_last_to_let_go_sees_every_leaf_cleared"), + ]), red(KERNEL_LOOM, "shootdown-served-relaxed", Some("counters_round"), &[ Fails("a_cpu_read_as_answered_is_read_with_its_answer"), ]), diff --git a/build/sourcegate.rs b/build/sourcegate.rs index a705b0d78f..6cf1643777 100644 --- a/build/sourcegate.rs +++ b/build/sourcegate.rs @@ -137,7 +137,7 @@ const AUTO_TRAIT_IMPLS: &[(&str, usize)] = &[ ("kernel/src/drivers/panic_console/mod.rs", 3), ("kernel/src/arch/x86_64/hw.rs", 1), ("kernel/src/mm/mmio.rs", 2), - ("kernel/src/object/paged.rs", 1), + ("kernel/src/object/paged.rs", 3), ("kernel/src/sched/driver.rs", 2), ]; diff --git a/docs/src/architecture/files.md b/docs/src/architecture/files.md index 2e4abe37eb..727f7f89ba 100644 --- a/docs/src/architecture/files.md +++ b/docs/src/architecture/files.md @@ -80,5 +80,11 @@ directory, which the child resolves in its own view. that server's open files, which then answer that they are gone; a fresh open reaches the restarted server. Unsaved changes may be lost; data already confirmed durable stays durable. +- A file server's end revokes every mapping of its files, so a program + executing from, or reading a mapping of, that server's files ends at its + next access to that mapping, a system service mapping a data file included. + A file server's death is the loss of that file system: its parent restarts + it, and programs that depended on it fail clearly rather than run on with + stale memory. - Per-session temporary storage is an ordinary file server holding its contents in RAM. diff --git a/docs/src/architecture/memory.md b/docs/src/architecture/memory.md index df91489e0f..b94a88d8e2 100644 --- a/docs/src/architecture/memory.md +++ b/docs/src/architecture/memory.md @@ -68,6 +68,22 @@ does keeps the server's writes waiting. - Mapped files may be larger than RAM, partially mapped, and coherently shared and written; their length is never committed in RAM. +- A file mapping maps the cache's own pages, so a mapper sees a write as it + lands, and a private mapping shares them until it is written. A mapping + holds its file as an open handle does: a file mapped and then closed or + deleted stays until its last mapping goes. A page leaves the cache only once + no mapping reaches it. +- A wait for a file's page behaves like one for swapped memory: it advances + with its file server's progress, is reported when overdue and never becomes + an error by timeout. The server failing what it cannot read, and its device + waits' own timeouts, bound it. +- A touch of a mapped page its file server failed, or of a file whose server + ended, ends the program with a diagnostic naming the file's page and the + cause; a copying system call answers an I/O error instead. A touch of a + mapped page wholly past the file's end ends the program the same way, as + POSIX's `SIGBUS`, and a copy from one is refused as a bad address. A touched + unwritten page of a file held in RAM becomes resident within its server's + capacity and is refused once that is full. - Clean pages are reclaimed; dirty pages are written back first, without losing concurrent writes. Dirty data growth is bounded, and writeback starts ahead of pressure. diff --git a/docs/src/architecture/services.md b/docs/src/architecture/services.md index 803cc57f08..de9ea83706 100644 --- a/docs/src/architecture/services.md +++ b/docs/src/architecture/services.md @@ -67,6 +67,10 @@ Every server admits its clients the same way, through one shared mechanism: - **Recovery restores durable state, not sessions.** File servers recover what was on disk, not old opens; network sessions may fail; the memory service keeps recoverable swapped contents. +- **A file server's end is the loss of its file system.** Every mapping of its + files is revoked, so a program executing from or reading a mapping of them, + a service included, ends at its next access to that mapping rather than run + on with stale memory. - **Nothing is silently replayed.** An operation with an uncertain outcome is reported, and retried only where safe; a write whose acknowledgement was lost is never replayed behind the client's back. diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index 5c6014790d..09ae7e281f 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -17,8 +17,13 @@ These answer failure in their POSIX form and do nothing LLVM's `getDefaultStackSize` (`llvm/lib/Support/ProgramStack.cpp`) ignores the failure and reads the `rlimit` it did not get. - `gethostname`, `ENOSYS`: no host name is published to a process. -- `msync`, `ENOSYS`: no mapping is of a file, so there is nothing to write back. -- `mmap` of a file, `ENODEV`: the kernel maps no file. +- `msync`, `ENOSYS`: no call answers which pages of a range are mapped, which + POSIX's `ENOMEM` for an unmapped one needs. +- `mmap` of a file other than `PROT_READ`, `ENODEV`: the kernel maps a file's + pages read-only and shared, and a private one shares them until it is written, + which waits for 7b's write faults (step 5); a private copy range of a file + (`PROT_WRITE` or `PROT_EXEC` with `MAP_PRIVATE`, 7b's R4) is not built either. + A file whose server maps no pages answers `ENODEV` too. - `fcntl` (`sdk/libc/src/fdreq.rs`): a record lock, `EINVAL`, POSIX's answer for a file that supports no locking; `F_GETFL`, `F_SETFL`, `F_GETOWN` and `F_SETOWN`, `ENOSYS`. diff --git a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md index 17e02a7382..7704d64d10 100644 --- a/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md +++ b/issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md @@ -25,16 +25,17 @@ names the client that took it (`issues/no-physical-memory-fairness.md`). **What one client leaves charged.** The kernel makes a pager's object slots 256 at a time and keeps every piece for the pager's life (`kernel/pure/slots.rs`). -A freed slot is taken again, but its piece never goes back. A slot is 216 -bytes since #1258: a 192-byte entry, of which the object's own state is 112 -and the pager's record 64, and two 12-byte marks, one for each notice it can -owe. Measured with a host replica of the layouts at #1258's merge of main: -https://github.com/ToyOSOrg/evidence/blob/af88668ec3bf695893aeabf229b7ac3c1bad4a95/r8/slot-sizes.log -At #1273 it was 72: -https://github.com/ToyOSOrg/evidence/tree/f81e4b2ddb7d89a92f82f9eb8da1fdf58587a563/pr-1273/sizes -So the server's job stays charged 216 bytes for each `/tmp` file of the most +A freed slot is taken again, but its piece never goes back. A slot is 272 +bytes: a 248-byte entry and two 12-byte marks, one for each notice it can +owe. Of the entry, the object's own state is 152 bytes, the pager's record 64 +and the list of a mapped object's mappers 16. The walk state mapping needs +adds 40 bytes to the object, and with the list 56 to a slot. Every entry +carries all of it. Measured by the compiler for both kernel targets; main +before mapping, measured the same way on x86-64, has 216: +https://github.com/ToyOSOrg/evidence/tree/94fd400ddab5d9c2ff776ac9df66f1b0bb88b768/r12/sizes +So the server's job stays charged 272 bytes for each `/tmp` file of the most it held at once, until the server ends. A client that makes and removes a -million files leaves about 216 MB. The manual has unused backing return under +million files leaves about 272 MB. The manual has unused backing return under pressure; this never returns. Owner: the file server and the kernel's paged objects, once the owner has diff --git a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md index ab461129bd..5adb4efab7 100644 --- a/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md +++ b/issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md @@ -95,10 +95,13 @@ live split of a user leaf. at its first touch. A program anywhere else is still read whole into memory of the spawner's and copied a page at a time, until 7b's step 3. A shared segment's mapping is never widened, so a program can no longer - make its own read-only segment writable (`mprotect`). Exit for that: 7b's - step 2 converts on protect, so a protect that adds write to such a range - commits its pages and makes it a copy range of the same object; only a - page shared until its first write waits for step 5's write faults. Exit: + make its own read-only segment writable (`mprotect`). Exit for that: a + protect that adds write to such a range commits its pages and makes it a + copy range of the same object (7b's R4, which step 3 builds, since a + program's writable segment mapped from its image is a copy range with a + paged source; step 2 maps a paged object read-only only, and its range + refuses a protect past that); only a page shared until its first write + waits for step 5's write faults. Exit: two processes of one binary hold its text once, wherever it is served from. 6. **`/apps` image sharing.** Decided by the orchestrator under the owner's @@ -118,7 +121,8 @@ live split of a user leaf. **7b lands in steps**, each a pull request: (1) a range of a memory object mapped where the caller says, an immutable one shared and executable, which is stage 5 for `/system`: built; (2) a paged object mapped read-only and -shared: faults that park on the pager, reverse records, revocation; (3) a +shared: faults that park on the pager, reverse records, revocation: built, +but for R4's copy ranges of a paged object, which step 3 builds (item 5); (3) a spawn maps its image from any pager-backed volume (stage 6), sealed against writes in place while it runs; (4) the storage track's attributes and names without a request; (5) writable shared @@ -136,10 +140,46 @@ mappings and write-back; (6) the #1077 retry against main after, four boots a side. Main after must be no slower than before; a spawn no longer copies a `/system` program's text, and whether that makes it faster is not measured on metal. +- **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: the whole + `testcases` boot, whose `mapped_file` member maps a paged object from a + second process (faults that wait, revocation, refusals, kills) and whose + `a_pager_s_end_answers_its_waiting_holders_gone` runs only there and + exercises the rewritten pager end, and the `shared-debug` and + `proctreecase` boots. Main must show every job of each boot exit 0. +- **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: osbench's + `system` (its pipe rows), `files`, `ipc`, `procs` and `mem` groups, main + before #1269 against main after, four boots a side. Main after must show no row + worse than noise: every fault and pinned copy changed, and every process + exit now empties its space through the full clear (futex bucket walks, + shootdowns, table unlinks) where its drop skipped them. +- **Owed on metal:** step 2 (#1269), on the T14 and the AMD bench: osbench's + `map` group on main beside Linux's `mmap` of the same files, recorded with + each row's ratio to Linux's: `touch_16m` and `threads_touch_16m` (a cached + file's pages faulted through a mapping, by one thread and by eight), + `read_4k` (4 KiB read through one) and `truncate_255` (a file 255 + processes map, truncated). Their correctness runs in a guest + (`osbench_map_group_exits_0`); what they measure is unmeasured until then. + +Fault waiters wake on their pager's watch, so a supply, a write's end or a +removal wakes every fault waiter of that pager and not only those its pages +meet: a departure from the design's per-request wakes (§3, S4). No row +prices it, since `threads_touch_16m` faults a RAM file's pages, which are +never absent. Owner: map7b, with step 3, whose spawns fault a DATA image's +pages cold. Exit: a many-threads cold-fault row on a DATA file larger than its +pager's capacity, judged against Linux on metal, and per-request wakes where +it shows the cost. A cold DATA page's miss in stages is the same kind of row +and waits with it. + +Step 2 measures a cached page's fault and a 4 KiB read through a mapping +against Linux (osbench `fs.*.map_touch_16m`, `fs.*.map_read_4k`). Owed, by +map7b, with step 3's metal: a cold DATA page's miss in stages, one object +mapped by 255 children and withdrawn (a revocation's cost grows with its +mappers), and many threads cold-faulting one large file; each exits as an +osbench row judged against Linux. -Until step 2's revocation exists, an object a space maps must keep its frames -for the object's life: ROOT's pages stay non-reclaimable, and ROOT is not -served through a pager, until then. +A memory object a space maps keeps its frames for the object's life, since +only a pager's pages are revoked: ROOT's pages stay non-reclaimable until ROOT +is served through a pager. **Paged objects** (memory design §7, `issues/toyos-builds-itself.md` item 13) are split in two. **7a**, delivered by the storage track's file-server work: diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index e92c4755b7..446ee0c765 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -58,6 +58,9 @@ reap-raise-relaxed = [] # `src/shootdown.rs`'s `serve` reads what it owes `Relaxed`, and # `tlb_shootdown` reds. shootdown-serve-relaxed = [] +# `src/object/paged/ranges.rs`'s `less` goes `Relaxed`, and `paged_ranges` +# reds. +ranges-less-relaxed = [] # `src/shootdown.rs`'s `served` reads `Relaxed`, and `counters_round` reds. shootdown-served-relaxed = [] # `src/shootdown.rs`'s `Asks::serve` takes its asks `Relaxed`, and diff --git a/kernel/loom/Cargo.toml b/kernel/loom/Cargo.toml index 7774945863..46c24d3dea 100644 --- a/kernel/loom/Cargo.toml +++ b/kernel/loom/Cargo.toml @@ -221,6 +221,17 @@ log-ring-loads-swapped = [] tlb-tags-enter-relaxed = [] # The same for `Tags::resume`, which must red the idle holder's model. tlb-tags-resume-relaxed = [] +# The negative control for a mapped object's range count. It makes +# `object/paged/ranges.rs`'s `less` `Relaxed` with no fence, so the last range +# to let go can unlink its node without seeing another's leaf cleared, and +# `paged_ranges.rs` must red: +# +# cargo test --manifest-path kernel/loom/Cargo.toml --features ranges-less-relaxed \ +# --test paged_ranges +# +# Never on by default and never reachable from a kernel build, which declares +# the same name only so `cfg` checking knows it. +ranges-less-relaxed = [] # The negative control for the panic console's published descriptor. It drops # the publisher's `Release` fence between its odd mark and its words, so a diff --git a/kernel/loom/src/lib.rs b/kernel/loom/src/lib.rs index 74955aab90..6bccece3b5 100644 --- a/kernel/loom/src/lib.rs +++ b/kernel/loom/src/lib.rs @@ -158,6 +158,11 @@ pub mod shootdown; #[path = "../../src/tlb_tags.rs"] pub mod tlb_tags; +// A mapped object's count of the ranges holding a space's node: its last +// letting go sees every other's clearing (`tests/paged_ranges.rs`). +#[path = "../../src/object/paged/ranges.rs"] +pub mod paged_ranges; + /// The CPU roster and the release/answer word, driven by `tests/smp_bringup.rs`; /// under `loom` alone, as is the [`arch::cpu`] shim it names. #[cfg(feature = "loom")] diff --git a/kernel/loom/tests/paged_ranges.rs b/kernel/loom/tests/paged_ranges.rs new file mode 100644 index 0000000000..a0f9922d4b --- /dev/null +++ b/kernel/loom/tests/paged_ranges.rs @@ -0,0 +1,66 @@ +//! Loom: a mapped object's node leaves its list only after every range of +//! the space let go of it, each having cleared its leaves first. +//! +//! A space's node of a mapped object is held by each of its ranges of the +//! object and by a fault parked on one of its pages, which holds no space +//! lock. Whoever lets go last unlinks the node, and from then on a walk of +//! the object's mappers does not visit the space: it frees a revoked frame +//! once its pass is done. So the last to let go must see every other's +//! clearing of its leaves, or a frame is freed under a leaf that may still +//! reach it. A leaf here is one cell; `CLEARED` read by the last is the +//! property, and loom reports a read the clearing does not happen before as a +//! causality violation. +//! +//! ```text +//! cargo test --manifest-path kernel/loom/Cargo.toml --features ranges-less-relaxed \ +//! --test paged_ranges +//! ``` +//! +//! relaxes the count's letting go, and this file must red. + +#![cfg(feature = "loom")] + +use kernel_loom::paged_ranges::Ranges; +use loom::cell::UnsafeCell; +use loom::sync::Arc; + +const MAPPED: u64 = 1; +const CLEARED: u64 = 0; + +/// What the last to let go does before the node leaves the list: the walk +/// it stops visiting will free whatever this leaf maps. +fn unlink(leaf: &UnsafeCell) { + // SAFETY: loom checks the access; the model's claim is that it is ordered. + let seen = leaf.with(|leaf| unsafe { *leaf }); + assert_eq!(seen, CLEARED, "the node left its list while a leaf of its space still mapped the page"); +} + +/// An unmap in the space clears its leaf and lets its range go; a fault +/// parked on the range's page lets its hold go as it wakes. Either may be +/// last. +#[test] +fn the_last_to_let_go_sees_every_leaf_cleared() { + loom::model(|| { + let ranges = Arc::new(Ranges::one()); + ranges.more(); + let leaf = Arc::new(UnsafeCell::new(MAPPED)); + + let unmap = { + let (ranges, leaf) = (ranges.clone(), leaf.clone()); + loom::thread::spawn(move || { + // SAFETY: the space's lock is the unmapper's; loom checks it. + leaf.with_mut(|leaf| unsafe { *leaf = CLEARED }); + if ranges.less() { + unlink(&leaf); + } + }) + }; + let parked = loom::thread::spawn(move || { + if ranges.less() { + unlink(&leaf); + } + }); + unmap.join().unwrap(); + parked.join().unwrap(); + }); +} diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 1bd86459e0..7ff5d790ba 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -2,11 +2,21 @@ //! serves, in one table of `capacity` slots made when the pager is, so no //! supply, read, write or write-back allocates. //! -//! **A page is present, failed or absent.** A present page holds a frame and -//! its dirty state; a failed one holds no frame and answers `EIO` until it is -//! supplied again; an absent one has no slot. Unwritten anonymous memory -//! reads zeros without a slot, and its present pages cannot be withdrawn. Nothing here waits: a caller -//! that finds a page absent records its request in its own wait record. +//! **A page is present, failed, revoking or absent.** A present page holds a +//! frame and its dirty state; a failed one holds no frame and answers `EIO` +//! until it is supplied again; an absent one has no slot. Unwritten anonymous +//! memory reads zeros without a slot, and its present pages cannot be +//! withdrawn. Nothing here waits: a caller that finds a page absent records +//! its request in its own wait record. +//! +//! **A removed page keeps its frame until no mapping reaches it.** A +//! withdraw, failure, shrink or the object's end marks the page revoking: it +//! reads as absent, takes no supply or write, and keeps its slot and frame, +//! so no page is ever two frames at once. One walk per object visits each of +//! its mappers ([`Mappers`]) after the marking, and a removal that finds a +//! walk running is taken by that walk's next pass; after the last pass and, +//! for an object a mapper ever linked, a sweep of the futex waits on them, +//! the walk's end gives every revoking page's frame out ([`Pages::walk`]). //! //! **Every write is admitted page by page** ([`Pages::admit`]): refused on a //! gone or sealed object, refused to a holder whose page would become dirty @@ -42,7 +52,7 @@ //! ([`Object::take_dirtied`]) owes the pager a notice; the rest widen the range. //! //! What a removed frame owes (the last pin's free, a zeroed tail) is the -//! caller's: every removal answers its frames. +//! caller's: the walk's end answers every frame. #![forbid(unsafe_code)] @@ -289,6 +299,72 @@ pub enum Supplied { Present, } +/// What a walk's step asks of its caller, or that it ended ([`Pages::walk`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Next { + /// Visit this mapper: clear its leaves of the walk's set. + Visit(N), + /// End every futex wait on a frame of the walk's set: only of an object + /// a mapper ever linked ([`Object::link`]). + Sweep, + /// The walk ended: the set's frames were given out. + Finished, +} + +/// Where a reader of an object's revoking pages has read to +/// ([`Pages::revoking`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Cursor { + Start, + After(u32), +} + +/// What a revoking page becomes once no mapping reaches its frame, by the +/// object as the walk's end finds it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Revoked { + /// Absent inside the object: a reader waiting on it is named again. + Absent, + Failed, + /// Past the size, or the object is gone. + Removed, +} + +/// The mappers of one object, one node per space that maps it, in the order +/// a walk visits them. A node links at the tail, seen at the epoch +/// [`Object::link`] answers, and leaves only once its space maps none of the +/// object's frames. +pub trait Mappers { + type Node: Copy; + fn head(&self) -> Option; + /// The epoch of the walk that last visited `node`, or the one current + /// when it linked. + fn seen(&self, node: Self::Node) -> u64; + /// `node` is visited at `epoch`: it moves to the tail. + fn visited(&mut self, node: Self::Node, epoch: u64); +} + +/// Where an object's walk stands. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +enum Walk { + Idle, + /// A walker visits its mappers; `again` where pages were marked since + /// its pass began, which another pass covers. + Running { again: bool }, +} + +/// What the caller that marked an object's pages does about their mappers. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Revoke { + /// Nothing is revoking. + Done, + /// No walk ran: the caller walks it ([`Pages::walk`]) to its end. + Walk, + /// The running walk takes the pages in a later pass: they are gone once + /// [`Object::finished`] answers true of this epoch. + Queued(u64), +} + #[derive(Clone, Copy)] struct State { dirty: bool, @@ -316,8 +392,12 @@ struct Slot { /// `None` for a failed page. frame: Option, state: State, + /// Removed, its frame kept until no mapping reaches it. + revoking: Option, prev: u32, next: u32, + /// The next of its object's revoking slots, in the order they were marked. + revoke_next: u32, } /// One object's part of the table: its size, whether its pager ended, the @@ -351,6 +431,23 @@ pub struct Object { told: bool, /// A notice is owed and not yet taken. tell: bool, + /// Its slots that are revoking, and the walk's set of them: a list in + /// the order they were marked, which only the walk's end empties. + revoking: usize, + revoke_head: u32, + revoke_tail: u32, + /// The pass of the walk over its mappers, moved at each pass's start. + epoch: u64, + walk: Walk, + /// The epoch whose pass last ended the walk. + finished: u64, + /// The epoch at which a walk was last told to sweep ([`Next::Sweep`]): + /// 0, which no walk's pass is, before the first. + swept: u64, + /// A mapper ever linked. Only a mapping's leaf keys a futex wait on one + /// of its frames, and an unmap clears its leaves without ending such a + /// wait, so this never resets. + linked: bool, } /// What a holder's read began at ([`Object::read_begin`]). @@ -441,6 +538,13 @@ impl Object { self.held.unwrap_or((self.size, self.mtime)) } + /// Where a fault's wait for one of its pages begins: a write or resize + /// since ends it ([`Pages::wait_over`]), and the fault asks afresh + /// ([`Want::anew`]). + pub fn version(&self) -> Version { + Version(self.version) + } + /// Whether no write or resize has begun since the read began at `v`. pub fn unchanged(&self, v: Version) -> bool { self.version == v.0 @@ -468,6 +572,42 @@ impl Object { free } + /// A mapper links: the epoch it is seen at, which the walk running + /// passes by, since nothing it maps from now on is revoking. + pub fn link(&mut self) -> u64 { + self.linked = true; + self.epoch + } + + /// Pages were marked revoking in this hold of the lock: what the caller + /// does about their mappers. + pub fn revoke(&mut self) -> Revoke { + if self.revoking == 0 { + return Revoke::Done; + } + match self.walk { + Walk::Idle => { + self.epoch += 1; + self.walk = Walk::Running { again: false }; + Revoke::Walk + } + Walk::Running { .. } => { + self.walk = Walk::Running { again: true }; + Revoke::Queued(self.epoch + 1) + } + } + } + + /// Whether the walk's pass at `epoch`, or a later one, ended. + pub fn finished(&self, epoch: u64) -> bool { + self.finished >= epoch + } + + /// Whether every page it held is gone, and no walk still owes a frame. + pub fn empty(&self) -> bool { + self.pages == 0 && self.walk == Walk::Idle + } + /// The pager takes the write side, or waits, and asks again, while a /// holder's write holds it: work inside the kernel that ends without /// waiting on anything. No holder's write begins meanwhile. @@ -600,8 +740,9 @@ impl Pages { } /// An object of `size` bytes, at most [`LAST_END`], named `id`: an id no - /// live object of this table has, which a released one's may be, since - /// [`release`](Self::release) takes every page it held. + /// live object of this table has. A released object's may be, once it is + /// [`empty`](Object::empty): [`invalidate`](Self::invalidate) marks every + /// page it held, and its walk's end takes them. pub fn object(&mut self, id: u32, size: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); Object { @@ -621,6 +762,14 @@ impl Pages { dirtied: None, told: false, tell: false, + revoking: 0, + revoke_head: NONE, + revoke_tail: NONE, + swept: 0, + linked: false, + epoch: 0, + walk: Walk::Idle, + finished: 0, } } @@ -670,7 +819,7 @@ impl Pages { if next != NONE { self.slot_mut(next).prev = i; } - self.slots[i as usize] = Some(Slot { object: o.id, page, frame, state, prev: NONE, next }); + self.slots[i as usize] = Some(Slot { object: o.id, page, frame, state, revoking: None, prev: NONE, next, revoke_next: NONE }); o.head = i; o.pages += 1; let mut at = self.hash(o.id, page); @@ -728,6 +877,9 @@ impl Pages { if s.state.counts_dirty() { self.dirty_held -= 1; } + if s.revoking.is_some() { + o.revoking -= 1; + } // Gone before its write-back, it needs none of the room it was paid for. if s.state.credited { o.credit += 1; @@ -744,6 +896,128 @@ impl Pages { !o.unchanged(since) || !matches!(self.lookup(o, page), Lookup::Absent) } + /// Remove slot `i` as `becomes` once no mapping reaches it: at once for a + /// failed page, which no mapping reaches, and as the walk's end for a + /// frame. A page already revoking becomes failed if either removal fails it. + fn revoke(&mut self, o: &mut Object, i: u32, becomes: Revoked) { + let s = self.slot_mut(i); + if s.frame.is_none() { + self.remove(o, i); + return; + } + if let Some(was) = s.revoking { + s.revoking = Some(if was == Revoked::Failed { was } else { becomes }); + return; + } + // Its bytes are no longer the file's: no write-back takes them, and + // it needs none of the room a holder's credit paid for. + let (counted, credited) = (s.state.counts_dirty(), s.state.credited); + s.state = State::CLEAN; + s.revoking = Some(becomes); + s.revoke_next = NONE; + o.revoking += 1; + if counted { + self.dirty_held -= 1; + } + if credited { + o.credit += 1; + } + match o.revoke_tail { + NONE => o.revoke_head = i, + tail => self.slot_mut(tail).revoke_next = i, + } + o.revoke_tail = i; + } + + /// The next step of `o`'s walk, after pages were marked revoking + /// ([`Object::revoke`]): visit a mapper not visited in this pass, a new + /// pass begun where a removal queued more pages meanwhile; once every + /// mapper has been visited, where a mapper ever linked, have every futex + /// wait on the walk's set ended, again whenever a pass has begun since it + /// was last asked, since a page marked during a sweep may be waited on + /// through a leaf an unmap already cleared; and once both are done, end + /// the walk here ([`Next::Finished`]): each revoking page leaves, its + /// frame given out to `out` with its page and what it became. + pub fn walk( + &mut self, + o: &mut Object, + mappers: &mut M, + waiters: &mut impl Waiters, + out: &mut impl FnMut(F, u64, Revoked), + ) -> Next { + loop { + if let Some(node) = mappers.head().filter(|&n| mappers.seen(n) != o.epoch) { + mappers.visited(node, o.epoch); + return Next::Visit(node); + } + match o.walk { + Walk::Running { again: true } => { + o.epoch += 1; + o.walk = Walk::Running { again: false }; + } + Walk::Running { again: false } if o.linked && o.swept != o.epoch => { + o.swept = o.epoch; + return Next::Sweep; + } + Walk::Running { again: false } => { + self.finish(o, waiters, out); + return Next::Finished; + } + Walk::Idle => panic!("paged: a walk of object {} that never began", o.id), + } + } + } + + /// The walk's end, which only [`Self::walk`] reaches: each revoking page + /// leaves, its frame given out with its page and what it became. A page + /// that ends absent was withdrawn, perhaps after a supply for a waiter's + /// notice, without moving the version: every want of `waiters` it meets + /// is named again. + fn finish(&mut self, o: &mut Object, waiters: &mut impl Waiters, out: &mut impl FnMut(F, u64, Revoked)) { + let held = !o.gone; + let mut at = core::mem::replace(&mut o.revoke_head, NONE); + o.revoke_tail = NONE; + while at != NONE { + let s = self.slot(at); + let (next, page, becomes) = (s.revoke_next, s.page, s.revoking.expect("paged: the walk's set holds revoking slots")); + let frame = self.remove(o, at).expect("paged: a revoking page holds its frame"); + let becomes = if held && page < o.page_end() { becomes } else { Revoked::Removed }; + match becomes { + Revoked::Failed => self.insert(o, page, None, State::CLEAN).expect("paged: the slot just freed"), + Revoked::Absent => waiters.each(&mut |want| { + want.withdrawn(page, 1); + }), + Revoked::Removed => {} + } + out(frame, page, becomes); + at = next; + } + debug_assert_eq!(o.revoking, 0); + o.walk = Walk::Idle; + o.finished = o.epoch; + } + + /// `o`'s revoking pages from `at` on, in the order they were marked, each + /// with the frame it keeps until the walk's end: as many as fit in `out`, + /// whose count this answers, `at` moved past them. Only the walk's end + /// takes a slot off the set, so a cursor stays good across the drops of + /// the pager's lock between batches, and sees pages marked since. + pub fn revoking(&self, o: &Object, at: &mut Cursor, out: &mut [(u64, F)]) -> usize { + let mut next = match *at { + Cursor::Start => o.revoke_head, + Cursor::After(i) => self.slot(i).revoke_next, + }; + let mut n = 0; + while n < out.len() && next != NONE { + let s = self.slot(next); + out[n] = (s.page, s.frame.expect("paged: a revoking page holds its frame")); + *at = Cursor::After(next); + next = s.revoke_next; + n += 1; + } + n + } + pub fn lookup(&self, o: &Object, page: u64) -> Lookup { if o.gone { return Lookup::Gone; @@ -751,16 +1025,18 @@ impl Pages { if page >= o.page_end() { return Lookup::PastEnd; } - match self.find(o.id, page) { + match self.find(o.id, page).map(|i| self.slot(i)) { None if o.anonymous => Lookup::Zero, None => Lookup::Absent, - Some(i) => self.slot(i).frame.map_or(Lookup::Failed, Lookup::Present), + Some(s) if s.revoking.is_some() => Lookup::Absent, + Some(s) => s.frame.map_or(Lookup::Failed, Lookup::Present), } } /// Install `frame` at `page`, clean, or dirty with a slot. A dirty page /// extends the size to `end`, where its bytes end past it; a clean one is - /// what the file holds and moves nothing. A failed page is replaced. + /// what the file holds and moves nothing. A failed page is replaced; a + /// revoking one is present until its walk ends. pub fn supply(&mut self, o: &mut Object, page: u64, frame: F, dirty: bool, end: u64) -> Result { if o.gone { return Err(Refused::Gone); @@ -776,7 +1052,7 @@ impl Pages { return Err(Refused::PastEnd); } match self.find(o.id, page) { - Some(i) if self.slot(i).frame.is_some() => return Ok(Supplied::Present), + Some(i) if self.slot(i).frame.is_some() || self.slot(i).revoking.is_some() => return Ok(Supplied::Present), Some(i) => { self.remove(o, i); } @@ -831,7 +1107,7 @@ impl Pages { } let i = self.find(o.id, page).ok_or(Refused::InvalidArgument)?; let s = self.slot(i); - let frame = s.frame.ok_or(Refused::InvalidArgument)?; + let frame = s.frame.filter(|_| s.revoking.is_none()).ok_or(Refused::InvalidArgument)?; let state = s.state; self.admissible(o, Some(state), by)?; let s = self.slot_mut(i); @@ -894,7 +1170,7 @@ impl Pages { match self.find(o.id, page) { Some(i) => { let s = self.slot(i); - if s.frame.is_none() { + if s.frame.is_none() || s.revoking.is_some() { return Err(Refused::Absent); } dirtying += u64::from(!s.state.dirty); @@ -925,9 +1201,9 @@ impl Pages { /// The end of a [`admit`](Self::admit) of `frame`, once its /// bytes are in: the size grows to `end`. A page a shrink or a failure - /// removed meanwhile, even one supplied again since, holds another frame - /// or none; there is nothing left to unpin, and the bytes are not the - /// file's. The caller keeps `frame` pinned until this, so it is not + /// removed meanwhile is revoking, or, supplied again since, holds another + /// frame or none; there is nothing left to unpin, and the bytes are not + /// the file's. The caller keeps `frame` pinned until this, so it is not /// another page's meanwhile. pub fn write_unpin(&mut self, o: &mut Object, page: u64, frame: F, end: u64) where @@ -936,7 +1212,7 @@ impl Pages { assert!(end <= (page + 1) * PAGE && end > page * PAGE, "a write pin's end {end} outside page {page}"); let Some(i) = self.find(o.id, page) else { return }; let s = self.slot_mut(i); - if s.frame != Some(frame) { + if s.frame != Some(frame) || s.revoking.is_some() { return; } s.state.write_pins = s.state.write_pins.checked_sub(1).expect("a write pin released twice"); @@ -952,7 +1228,7 @@ impl Pages { { let Some(i) = self.find(o.id, page) else { return }; let s = self.slot_mut(i); - if s.frame != Some(frame) { + if s.frame != Some(frame) || s.revoking.is_some() { return; } s.state.write_pins = s.state.write_pins.checked_sub(1).expect("a write pin released twice"); @@ -1021,22 +1297,28 @@ impl Pages { } /// Remove the clean pages of the range that no write-back or write pin - /// holds: their frames, for the caller to free at their last pin. How - /// many, and whether a waiter's range they meet is news its pager must - /// hear again: they end absent without moving the version, so they may - /// have been supplied for a notice the waiter has not yet looked at. + /// holds, and the failed ones: how many, and whether a waiter's want is + /// news its pager must hear again. Each ends absent without moving the + /// version, perhaps after a supply for a waiter's notice, so every want of + /// `waiters` it meets is named again: a failed page's now, as it goes at + /// once, a present page's at its walk's end, as it is revoking until then + /// ([`Object::revoke`]). /// A range longer than the object holds pages walks its pages, not the range. - pub fn withdraw(&mut self, o: &mut Object, first: u64, count: u64, waiters: &mut impl Waiters, out: &mut impl FnMut(F)) -> (usize, bool) { + pub fn withdraw(&mut self, o: &mut Object, first: u64, count: u64, waiters: &mut impl Waiters) -> (usize, bool) { if o.anonymous { return (0, false); } let end = first.saturating_add(count); - let mut removed = 0; + let (mut removed, mut news) = (0, false); let mut take = |pages: &mut Self, o: &mut Object, i: u32| { let s = pages.slot(i); - if s.state.counts_dirty() || s.state.write_pins > 0 { + if s.revoking.is_some() || s.state.counts_dirty() || s.state.write_pins > 0 { return; } - if let Some(frame) = pages.remove(o, i) { - out(frame); + if s.frame.is_none() { + let page = s.page; + pages.remove(o, i); + waiters.each(&mut |want| news |= want.withdrawn(page, 1)); + } else { + pages.revoke(o, i, Revoked::Absent); } removed += 1; }; @@ -1057,25 +1339,20 @@ impl Pages { } } } - let mut news = false; - if removed > 0 { - waiters.each(&mut |want| news |= want.withdrawn(first, count)); - } (removed, news) } - /// Mark the range failed: present pages give up their frames, and a read - /// of any page of it answers `Failed` until it is supplied again. - pub fn fail(&mut self, o: &mut Object, first: u64, count: u64, out: &mut impl FnMut(F)) -> Result<(), Refused> { + /// Mark the range failed: a read of any page of it answers `Failed` until + /// it is supplied again, a present page's from its walk's end. + pub fn fail(&mut self, o: &mut Object, first: u64, count: u64) -> Result<(), Refused> { if o.anonymous { return Err(Refused::InvalidArgument); } let end = first.saturating_add(count).min(o.page_end()); for page in first..end { - if let Some(i) = self.find(o.id, page) { - if let Some(frame) = self.remove(o, i) { - out(frame); - } + match self.find(o.id, page) { + Some(i) if self.slot(i).frame.is_some() => self.revoke(o, i, Revoked::Failed), + Some(_) => {} + None => self.insert(o, page, None, State::CLEAN)?, } - self.insert(o, page, None, State::CLEAN)?; } Ok(()) } @@ -1083,7 +1360,7 @@ impl Pages { /// Set the size. A shrink removes every page past it, dirty or not, and /// answers the frame of a partial last page, whose tail past the size the /// caller zeroes. - pub fn resize(&mut self, o: &mut Object, size: u64, out: &mut impl FnMut(F)) -> Result, Refused> { + pub fn resize(&mut self, o: &mut Object, size: u64) -> Result, Refused> { if o.sealed() { return Err(Refused::Sealed); } @@ -1102,9 +1379,7 @@ impl Pages { (s.next, s.page) }; if page >= keep { - if let Some(frame) = self.remove(o, at) { - out(frame); - } + self.revoke(o, at, Revoked::Absent); } at = next; } @@ -1119,12 +1394,14 @@ impl Pages { }) } - /// The pager ended: every page goes, and every later call answers `Gone`. - pub fn invalidate(&mut self, o: &mut Object, out: &mut impl FnMut(F)) { - while o.head != NONE { - if let Some(frame) = self.remove(o, o.head) { - out(frame); - } + /// The pager ended or let the object go: every page goes, and every + /// later call answers `Gone`. + pub fn invalidate(&mut self, o: &mut Object) { + let mut at = o.head; + while at != NONE { + let next = self.slot(at).next; + self.revoke(o, at, Revoked::Absent); + at = next; } o.gone = true; for range in &mut self.open { @@ -1133,11 +1410,6 @@ impl Pages { } } } - - /// The object is let go: every page goes, as at [`invalidate`](Self::invalidate). - pub fn release(&mut self, mut o: Object, out: &mut impl FnMut(F)) { - self.invalidate(&mut o, out); - } } #[cfg(test)] @@ -1147,10 +1419,219 @@ mod tests { use super::*; - fn none(_: u64) {} + /// No space maps anything. + struct Unmapped; + + impl Mappers for Unmapped { + type Node = (); + fn head(&self) -> Option<()> { + None + } + fn seen(&self, (): ()) -> u64 { + 0 + } + fn visited(&mut self, (): (), _: u64) {} + } + + /// The frames of `o`'s revoking pages, once a walk over no mappers ends: + /// none ever linked, so it sweeps nothing. + fn settle(p: &mut Pages, o: &mut Object) -> Vec { + let mut freed = Vec::new(); + match o.revoke() { + Revoke::Done => {} + Revoke::Walk => { + assert_eq!(p.walk(o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Finished); + } + Revoke::Queued(_) => panic!("a walk ran with no walker"), + } + freed + } - fn sink(freed: &mut Vec) -> impl FnMut(u64) + '_ { - |f| freed.push(f) + /// One object's mappers: each node's space and the epoch it was last + /// seen at, in visiting order. + #[derive(Default)] + struct Ring(std::collections::VecDeque<(u32, u64)>); + + impl Ring { + fn link(&mut self, space: u32, o: &mut Object) { + self.0.push_back((space, o.link())); + } + } + + impl Mappers for Ring { + type Node = u32; + fn head(&self) -> Option { + self.0.front().map(|&(n, _)| n) + } + fn seen(&self, node: u32) -> u64 { + self.0.iter().find(|&&(n, _)| n == node).expect("linked").1 + } + fn visited(&mut self, node: u32, epoch: u64) { + let at = self.0.iter().position(|&(n, _)| n == node).expect("linked"); + self.0.remove(at); + self.0.push_back((node, epoch)); + } + } + + /// A page marked while the sweep has its lock dropped is swept before the + /// walk gives its frame back, though the pass its marking begins finds no + /// mapper: an unmap may have cleared the leaf a waiter keyed on it went + /// through, and only the sweep ends that wait. + #[test] + fn a_page_marked_during_the_sweep_is_swept_before_its_frame_goes_back() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + // A mapper linked, and its unmap since cleared its leaves. + o.link(); + p.supply(&mut o, 0, 100, false, PAGE).unwrap(); + p.supply(&mut o, 1, 101, false, 2 * PAGE).unwrap(); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); + assert_eq!(o.revoke(), Revoke::Walk); + let mut swept = Vec::new(); + let mut sweep = |p: &Pages, o: &Object| { + let mut batch = [(0, 0); 4]; + let n = p.revoking(o, &mut Cursor::Start, &mut batch); + swept.extend(batch[..n].iter().map(|&(_, frame)| frame)); + }; + let mut freed = Vec::new(); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |frame, _, _| freed.push(frame)), Next::Sweep); + sweep(&p, &o); + // Between the sweep's last read and its relock, another removal. + assert_eq!(p.withdraw(&mut o, 1, 1, &mut Nobody).0, 1); + assert!(matches!(o.revoke(), Revoke::Queued(_))); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |frame, _, _| freed.push(frame)), Next::Sweep, "a pass began since the sweep"); + sweep(&p, &o); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |frame, _, _| freed.push(frame)), Next::Finished); + assert_eq!(freed, [100, 101]); + assert!(freed.iter().all(|frame| swept.contains(frame)), "a frame went back unswept: swept {swept:?}"); + } + + /// The walk's set holds each revoking page once, in the order they were + /// marked; a cursor reads it a batch at a time and sees pages marked + /// between batches; the walk's end empties it. + #[test] + fn the_walks_set_is_read_a_batch_at_a_time_in_marking_order() { + let mut p = Pages::new(16, 4).unwrap(); + let mut o = p.object(1, 16 * PAGE); + for page in [9, 2, 14, 5, 0, 11, 7] { + p.supply(&mut o, page, 100 + page, false, (page + 1) * PAGE).unwrap(); + } + for page in [7, 2, 11] { + assert_eq!(p.withdraw(&mut o, page, 1, &mut Nobody).0, 1); + } + let mut at = Cursor::Start; + let mut batch = [(0, 0); 2]; + assert_eq!(p.revoking(&o, &mut at, &mut batch), 2); + assert_eq!(batch, [(7, 107), (2, 102)]); + assert_eq!(p.withdraw(&mut o, 2, 1, &mut Nobody).0, 0, "a revoking page is marked once"); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); + assert_eq!(p.revoking(&o, &mut at, &mut batch), 2, "a page marked between batches is read"); + assert_eq!(batch, [(11, 111), (0, 100)]); + assert_eq!(p.revoking(&o, &mut at, &mut batch), 0); + assert_eq!(o.revoke(), Revoke::Walk); + let mut freed = Vec::new(); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, _, _| freed.push(f)), Next::Finished); + assert_eq!(freed, [107, 102, 111, 100]); + assert_eq!(p.revoking(&o, &mut Cursor::Start, &mut batch), 0, "the walk's end empties its set"); + } + + /// A removed page reads as absent, takes no second frame and no write, + /// and keeps its frame until every mapper of its object was visited; + /// then its frame goes out once and the page is supplied afresh. + #[test] + fn a_revoking_page_keeps_its_one_frame_until_every_mapper_is_visited() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + let mut ring = Ring::default(); + ring.link(10, &mut o); + ring.link(11, &mut o); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 1); + assert_eq!(p.lookup(&o, 0), Lookup::Absent); + assert_eq!(p.supply(&mut o, 0, 8, false, PAGE), Ok(Supplied::Present), "a second frame for the page"); + assert_eq!(p.admit(&mut o, 0, Writer::Pager), Err(Refused::InvalidArgument)); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 0, "withdrawn once"); + assert_eq!(o.revoke(), Revoke::Walk); + let mut freed = Vec::new(); + let mut out = |f, page, becomes| freed.push((f, page, becomes)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(11)); + assert!(!o.empty()); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Sweep); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished); + assert_eq!(freed, [(7, 0, Revoked::Absent)]); + assert!(o.empty()); + assert_eq!(p.supply(&mut o, 0, 8, false, PAGE), Ok(Supplied::Installed)); + assert_eq!(o.revoke(), Revoke::Done, "nothing revoking"); + } + + /// A removal while a walk runs is taken by a second pass over every + /// mapper, those the first visited and one linked during it, which may + /// map the newly removed page; its caller knows the pass by its epoch. + #[test] + fn a_removal_during_a_walk_is_taken_by_a_second_pass() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + let mut ring = Ring::default(); + ring.link(10, &mut o); + ring.link(11, &mut o); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + p.supply(&mut o, 1, 8, false, 2 * PAGE).unwrap(); + p.withdraw(&mut o, 0, 1, &mut Nobody); + assert_eq!(o.revoke(), Revoke::Walk); + let mut freed = Vec::new(); + let mut out = |f, page, becomes| freed.push((f, page, becomes)); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); + ring.link(12, &mut o); + p.fail(&mut o, 1, 1).unwrap(); + let Revoke::Queued(second) = o.revoke() else { panic!("a walk runs") }; + assert!(!o.finished(second)); + let visits: Vec<_> = core::iter::from_fn(|| match p.walk(&mut o, &mut ring, &mut Nobody, &mut out) { + Next::Visit(node) => Some(node), + Next::Sweep | Next::Finished => None, + }) + .collect(); + assert_eq!(visits, [11, 10, 12, 11], "the first pass ends at 11, the second visits all three"); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished, "the sweep, then the end"); + assert!(o.finished(second)); + assert_eq!(freed, [(7, 0, Revoked::Absent), (8, 1, Revoked::Failed)], "in marking order"); + assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Absent, Lookup::Failed)); + } + + /// A mapper linked during a walk that nothing else joins maps nothing + /// revoking, so the walk passes it by. + #[test] + fn a_mapper_linked_during_a_walk_is_passed_by() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, PAGE); + let mut ring = Ring::default(); + ring.link(10, &mut o); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + p.withdraw(&mut o, 0, 1, &mut Nobody); + assert_eq!(o.revoke(), Revoke::Walk); + let mut out = |_, _, _| {}; + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Visit(10)); + ring.link(11, &mut o); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Sweep); + assert_eq!(p.walk(&mut o, &mut ring, &mut Nobody, &mut out), Next::Finished); + } + + /// A page failed and then shrunk away while revoking ends with no slot, + /// and one failed and then withdrawn still ends failed. + #[test] + fn a_revoking_page_ends_failed_only_where_the_object_still_holds_it() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + p.supply(&mut o, 0, 7, false, PAGE).unwrap(); + p.supply(&mut o, 1, 8, false, 2 * PAGE).unwrap(); + p.fail(&mut o, 0, 2).unwrap(); + assert_eq!(p.withdraw(&mut o, 0, 2, &mut Nobody).0, 0, "nothing left to withdraw"); + p.resize(&mut o, PAGE).unwrap(); + assert_eq!(o.revoke(), Revoke::Walk); + let mut ended = Vec::new(); + assert_eq!(p.walk(&mut o, &mut Unmapped, &mut Nobody, &mut |f, page, becomes| ended.push((f, page, becomes))), Next::Finished); + assert_eq!(ended, [(7, 0, Revoked::Failed), (8, 1, Revoked::Removed)]); + assert_eq!((p.lookup(&o, 0), p.used()), (Lookup::Failed, 1)); } /// No reader waits. @@ -1194,7 +1675,7 @@ mod tests { assert!(!o.unchanged(before), "and still does once it ended"); let after = o.read_begin().expect("the write ended"); assert!(o.unchanged(after)); - p.resize(&mut o, 0, &mut none).unwrap(); + p.resize(&mut o, 0).unwrap(); assert!(!o.unchanged(after), "a shrink is a change a read meets"); } @@ -1231,9 +1712,9 @@ mod tests { o.stamp(9); assert_eq!((o.size(), o.mtime()), (2 * PAGE, 9), "the pager sees its own"); assert_eq!(o.stat(), (100, 5), "a holder sees the moment the hold began"); - p.resize(&mut o, 0, &mut none).unwrap(); + p.resize(&mut o, 0).unwrap(); assert_eq!(o.stat(), (100, 5), "and a truncation under it"); - p.resize(&mut o, 3 * PAGE, &mut none).unwrap(); + p.resize(&mut o, 3 * PAGE).unwrap(); o.give(Side::Pager); assert_eq!(o.stat(), (3 * PAGE, 9), "the unhold shows what the writes left"); assert!(o.take_holder()); @@ -1284,7 +1765,7 @@ mod tests { p.admit(&mut o, 0, Writer::Pager).unwrap(); p.write_unpin(&mut o, 0, 0, PAGE); assert_eq!(o.credit(), 1, "the pager's write of page 0 brought its own room"); - p.resize(&mut o, 2 * PAGE, &mut none).unwrap(); + p.resize(&mut o, 2 * PAGE).unwrap(); assert_eq!(o.credit(), 2, "page 2 went unwritten"); p.writeback_begin(&mut o, 1, 1).unwrap(); p.writeback_end(&mut o, 1, 1, false).unwrap(); @@ -1310,20 +1791,20 @@ mod tests { assert_eq!(pages.dirty_held(), 0); assert_eq!(pages.admit(&mut object, 0, Writer::Pager), Ok(42)); pages.write_unpin(&mut object, 0, 42, PAGE); - assert_eq!(pages.withdraw(&mut object, 0, 1, &mut Nobody, &mut none).0, 0); + assert_eq!(pages.withdraw(&mut object, 0, 1, &mut Nobody).0, 0); assert_eq!(pages.writeback_begin(&mut object, 0, 1), Err(Refused::InvalidArgument)); - assert_eq!(pages.fail(&mut object, 0, 1, &mut none), Err(Refused::InvalidArgument)); + assert_eq!(pages.fail(&mut object, 0, 1), Err(Refused::InvalidArgument)); assert_eq!(pages.lookup(&object, 0), Lookup::Present(42)); assert_eq!(pages.supply(&mut object, 1, 43, true, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(pages.supply(&mut object, 2, 44, true, 3 * PAGE), Err(Refused::NoRoom)); - pages.resize(&mut object, 4 * PAGE, &mut none).unwrap(); + pages.resize(&mut object, 4 * PAGE).unwrap(); assert_eq!(pages.lookup(&object, 3), Lookup::Zero, "a hole reads without any free slot"); - let mut freed = Vec::new(); - pages.resize(&mut object, PAGE, &mut sink(&mut freed)).unwrap(); - assert_eq!(freed, [43]); - pages.release(object, &mut sink(&mut freed)); - assert_eq!(freed, [43, 42]); + pages.resize(&mut object, PAGE).unwrap(); + assert_eq!(settle(&mut pages, &mut object), [43]); + pages.invalidate(&mut object); + assert_eq!(settle(&mut pages, &mut object), [42]); assert_eq!(pages.used(), 0); + assert!(object.empty()); } #[test] @@ -1384,11 +1865,12 @@ mod tests { let mut o = p.object(5, PAGE); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); p.admit(&mut o, 0, Writer::Pager).unwrap(); - p.resize(&mut o, 0, &mut none).unwrap(); + p.resize(&mut o, 0).unwrap(); + settle(&mut p, &mut o); p.supply(&mut o, 0, 8, true, 10).unwrap(); p.write_unpin(&mut o, 0, 7, PAGE); assert_eq!(o.size(), 10, "the shrink's size, and the new page's, stand over a copy they overtook"); - assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody, &mut none).0, 0, "the new page is dirty"); + assert_eq!(p.withdraw(&mut o, 0, 1, &mut Nobody).0, 0, "the new page is dirty"); p.writeback_begin(&mut o, 0, 1).unwrap(); assert_eq!(p.writeback_end(&mut o, 0, 1, true).unwrap()[0], 0, "and holds no pin of the old one"); } @@ -1420,9 +1902,10 @@ mod tests { p.write_unpin(&mut o, 0, 0, PAGE); p.admit(&mut o, 1, Writer::Pager).unwrap(); // dirtied inside p.write_unpin(&mut o, 1, 1, 2 * PAGE); - let mut freed = Vec::new(); - p.resize(&mut o, 5 * PAGE, &mut sink(&mut freed)).unwrap(); // page 5 removed inside - p.fail(&mut o, 4, 1, &mut sink(&mut freed)).unwrap(); // page 4 failed inside + p.resize(&mut o, 5 * PAGE).unwrap(); // page 5 removed inside + p.fail(&mut o, 4, 1).unwrap(); // page 4 failed inside + assert_eq!(p.dirty_held(), 4, "a revoking page's bytes are no longer written back"); + assert_eq!(settle(&mut p, &mut o), [5, 4]); let kept = p.writeback_end(&mut o, 0, 6, true).unwrap()[0]; assert_eq!(kept, 0b11, "pinned since the begin and dirtied since it; the removed are clear"); assert_eq!(p.dirty_held(), 2); @@ -1465,9 +1948,9 @@ mod tests { p.supply(&mut o, 3, 13, false, 4 * PAGE).unwrap(); p.writeback_begin(&mut o, 2, 1).unwrap(); // clean but under write-back p.admit(&mut o, 3, Writer::Pager).unwrap(); // write-pinned - let mut freed = Vec::new(); - assert_eq!(p.withdraw(&mut o, 0, 4, &mut Nobody, &mut sink(&mut freed)).0, 1); - assert_eq!(freed, [10]); + assert_eq!(p.withdraw(&mut o, 0, 4, &mut Nobody).0, 1); + assert_eq!(p.lookup(&o, 0), Lookup::Absent, "revoking reads as absent"); + assert_eq!(settle(&mut p, &mut o), [10]); assert_eq!(p.lookup(&o, 0), Lookup::Absent); } @@ -1519,13 +2002,13 @@ mod tests { let mut o = p.object(1, LAST_END); p.supply(&mut o, 7, 1, false, 8 * PAGE).unwrap(); p.supply(&mut o, 1 << 40, 2, false, ((1 << 40) + 1) * PAGE).unwrap(); - let mut freed = Vec::new(); p.probes.set(0); - assert_eq!(p.withdraw(&mut o, 8, 1 << 24, &mut Nobody, &mut sink(&mut freed)).0, 0, "neither page lies in 8..8+2^24"); + assert_eq!(p.withdraw(&mut o, 8, 1 << 24, &mut Nobody).0, 0, "neither page lies in 8..8+2^24"); assert!(p.probes.get() < 8, "{} probes for a range of 2^24 pages", p.probes.get()); - assert_eq!(p.withdraw(&mut o, 8, u64::MAX, &mut Nobody, &mut sink(&mut freed)).0, 1); - assert_eq!(freed, [2], "only the page inside the range"); - assert_eq!(p.withdraw(&mut o, 0, u64::MAX, &mut Nobody, &mut sink(&mut freed)).0, 1); + assert_eq!(p.withdraw(&mut o, 8, u64::MAX, &mut Nobody).0, 1); + assert_eq!(settle(&mut p, &mut o), [2], "only the page inside the range"); + assert_eq!(p.withdraw(&mut o, 0, u64::MAX, &mut Nobody).0, 1); + settle(&mut p, &mut o); assert_eq!(o.pages(), 0); } @@ -1551,12 +2034,11 @@ mod tests { for page in 0..3 { p.supply(&mut o, page, page + 100, true, (page + 1) * PAGE).unwrap(); } - let mut freed = Vec::new(); - assert_eq!(p.resize(&mut o, PAGE + 5, &mut sink(&mut freed)), Ok(Some(101))); - assert_eq!(freed, [102]); + assert_eq!(p.resize(&mut o, PAGE + 5), Ok(Some(101))); assert_eq!(p.dirty_held(), 2); assert_eq!(p.lookup(&o, 2), Lookup::PastEnd); - assert_eq!(p.resize(&mut o, 4 * PAGE, &mut none), Ok(None), "growth removes nothing"); + assert_eq!(settle(&mut p, &mut o), [102]); + assert_eq!(p.resize(&mut o, 4 * PAGE), Ok(None), "growth removes nothing"); assert_eq!(p.lookup(&o, 2), Lookup::Absent, "and leaves the new range absent"); } @@ -1565,9 +2047,9 @@ mod tests { let mut p = Pages::new(8, 8).unwrap(); let mut o = p.object(12, 2 * PAGE); p.supply(&mut o, 0, 5, false, PAGE).unwrap(); - let mut freed = Vec::new(); - p.fail(&mut o, 0, 2, &mut sink(&mut freed)).unwrap(); - assert_eq!(freed, [5]); + p.fail(&mut o, 0, 2).unwrap(); + assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Absent, Lookup::Failed), "page 0 revoking"); + assert_eq!(settle(&mut p, &mut o), [5]); assert_eq!((p.lookup(&o, 0), p.lookup(&o, 1)), (Lookup::Failed, Lookup::Failed)); assert_eq!(p.supply(&mut o, 1, 6, false, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(p.lookup(&o, 1), Lookup::Present(6)); @@ -1579,8 +2061,9 @@ mod tests { let mut o = p.object(13, 4 * PAGE); p.supply(&mut o, 0, 1, true, PAGE).unwrap(); p.supply(&mut o, 2, 3, false, 3 * PAGE).unwrap(); - let mut freed = Vec::new(); - p.invalidate(&mut o, &mut sink(&mut freed)); + p.invalidate(&mut o); + assert_eq!(p.lookup(&o, 0), Lookup::Gone); + let mut freed = settle(&mut p, &mut o); freed.sort(); assert_eq!(freed, [1, 3]); assert_eq!((p.used(), p.dirty_held()), (0, 0)); @@ -1600,9 +2083,8 @@ mod tests { let page = (x >> 40) % 97; let o = &mut objects[which]; if (x >> 20).is_multiple_of(3) { - let mut freed = Vec::new(); - p.withdraw(o, page, 1, &mut Nobody, &mut sink(&mut freed)); - if !freed.is_empty() { + p.withdraw(o, page, 1, &mut Nobody); + if !settle(&mut p, o).is_empty() { assert!(model.remove(&(which, page))); } } else { @@ -1692,8 +2174,8 @@ mod tests { assert_eq!(p.admit(&mut o, 0, Writer::Pager), Err(Refused::Sealed)); assert_eq!(p.admit_fresh(&mut o, 1, 2, Writer::Holder), Err(Refused::Sealed)); assert_eq!(p.supply(&mut o, 1, 2, true, 2 * PAGE), Err(Refused::Sealed)); - assert_eq!(p.resize(&mut o, 0, &mut none), Err(Refused::Sealed)); - assert_eq!(p.resize(&mut o, 4 * PAGE, &mut none), Err(Refused::Sealed)); + assert_eq!(p.resize(&mut o, 0), Err(Refused::Sealed)); + assert_eq!(p.resize(&mut o, 4 * PAGE), Err(Refused::Sealed)); assert_eq!((o.size(), o.credit(), p.dirty_held()), (PAGE, 4, 0), "nothing changed"); assert_eq!(p.lookup(&o, 0), Lookup::Present(1)); o.seals = 0; @@ -1721,7 +2203,7 @@ mod tests { assert_eq!(p.holder_fits(&o, 2, 3), Err(Refused::DirtyFull), "1 dirty and 3 more pass a ceiling of 3"); assert_eq!(p.holder_fits(&o, 2, 2), Ok(1), "page 3 is fresh"); assert_eq!((o.credit(), p.dirty_held()), (3, 1), "a check changes nothing"); - p.fail(&mut o, 0, 1, &mut none).unwrap(); + p.fail(&mut o, 0, 1).unwrap(); assert_eq!(p.holder_fits(&o, 0, 1), Err(Refused::Absent), "a failed page is the pager's to supply again"); o.seals = 1; assert_eq!(p.holder_fits(&o, 1, 1), Err(Refused::Sealed)); @@ -1729,11 +2211,35 @@ mod tests { let mut m = small.anonymous(2, 0); assert_eq!(small.holder_fits(&m, 0, 1), Ok(1), "anonymous memory needs no credit"); assert_eq!(small.holder_fits(&m, 0, 2), Err(Refused::NoRoom), "but a slot for each fresh page"); - small.resize(&mut m, 2 * PAGE, &mut none).unwrap(); + small.resize(&mut m, 2 * PAGE).unwrap(); assert_eq!(small.admit_fresh(&mut m, 1, 7, Writer::Holder), Ok(()), "a hole inside the size reads as zeros"); assert_eq!(small.holder_fits(&m, 0, 2), Err(Refused::NoRoom), "page 1 holds the only slot"); } + /// A revoking page takes no write: a holder's check finds it absent, and + /// neither an admission nor a fresh page lands on it. The room a holder's + /// credit paid for it is credit again as it is marked, as for a page gone + /// before its write-back, and its walk's end gives back nothing twice. + #[test] + fn a_revoking_page_takes_no_write_and_gives_its_credit_back_once() { + let mut p = Pages::new(8, 8).unwrap(); + let mut o = p.object(1, 2 * PAGE); + p.supply(&mut o, 0, 10, false, PAGE).unwrap(); + p.supply(&mut o, 1, 11, false, 2 * PAGE).unwrap(); + o.grant(1); + p.admit(&mut o, 1, Writer::Holder).unwrap(); + p.write_unpin(&mut o, 1, 11, 2 * PAGE); + assert_eq!(o.credit(), 0); + p.resize(&mut o, PAGE).unwrap(); + assert_eq!(o.credit(), 1, "marked revoking, the page needs none of its room"); + assert_eq!(p.holder_fits(&o, 1, 1), Err(Refused::Absent)); + assert_eq!(p.admit(&mut o, 1, Writer::Pager), Err(Refused::InvalidArgument)); + assert_eq!(p.admit_fresh(&mut o, 1, 12, Writer::Holder), Err(Refused::InvalidArgument)); + assert_eq!(settle(&mut p, &mut o), [11]); + assert_eq!(o.credit(), 1, "the walk's end gives nothing twice"); + assert_eq!(p.holder_fits(&o, 1, 1), Ok(1), "past the size, a fresh page once the walk ended"); + } + /// The first page a holder dirties owes the pager one notice; later ones /// widen the range without another, until the pager takes it. A shrink /// keeps only what lies inside the size. @@ -1760,7 +2266,7 @@ mod tests { p.admit_fresh(&mut o, 5, 15, Writer::Holder).unwrap(); p.write_unpin(&mut o, 5, 15, 5 * PAGE + 1); assert!(o.take_tell(), "owed again once taken"); - p.resize(&mut o, 2 * PAGE, &mut none).unwrap(); + p.resize(&mut o, 2 * PAGE).unwrap(); assert_eq!(o.take_dirtied(), None, "a shrink takes what lay past it"); } @@ -1976,14 +2482,15 @@ mod tests { assert!(!p.wait_over(&o, 0, since), "absent"); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); assert!(p.wait_over(&o, 0, since), "supplied"); - p.resize(&mut o, 0, &mut none).unwrap(); - p.resize(&mut o, PAGE, &mut none).unwrap(); + p.resize(&mut o, 0).unwrap(); + settle(&mut p, &mut o); + p.resize(&mut o, PAGE).unwrap(); assert_eq!(p.lookup(&o, 0), Lookup::Absent); assert!(p.wait_over(&o, 0, since), "shrunk and grown back before the reader looked"); let mut failed = p.object(2, PAGE); let since = failed.read_begin().unwrap(); - p.fail(&mut failed, 0, 1, &mut none).unwrap(); + p.fail(&mut failed, 0, 1).unwrap(); assert!(failed.take_pager()); assert_eq!(p.supply(&mut failed, 0, 8, true, PAGE), Err(Refused::DirtyFull)); failed.give(Side::Pager); @@ -1996,10 +2503,12 @@ mod tests { let mut want = Want::NONE; want.ask(0, 1); assert_eq!(want.report(), Some((0, 1))); - let mut waiting = One(&mut want); - assert_eq!(p.withdraw(&mut taken, 0, 1, &mut waiting, &mut none), (1, true)); + assert_eq!(p.withdraw(&mut taken, 0, 1, &mut Nobody).0, 1); + assert_eq!(taken.revoke(), Revoke::Walk); + assert!(!want.unreported(), "named only once no mapping reaches the page"); + assert_eq!(p.walk(&mut taken, &mut Unmapped, &mut One(&mut want), &mut |_, _, _| ()), Next::Finished); assert!(!p.wait_over(&taken, 0, since), "a withdraw leaves the read waiting"); - assert!(want.unreported(), "and names its range again"); + assert!(want.unreported(), "and its walk's end names its range again"); } /// One reader's want. @@ -2065,17 +2574,20 @@ mod tests { let next = self.next + 1; match step { Step::Withdraw => { - let mut waiting = One(&mut self.want); - self.p.withdraw(&mut self.o, 0, 1, &mut waiting, &mut |_| ()); + self.p.withdraw(&mut self.o, 0, 1, &mut One(&mut self.want)); + self.settle(); } Step::Fail => { - self.p.fail(&mut self.o, 0, 1, &mut |_| ()).unwrap(); + self.p.fail(&mut self.o, 0, 1).unwrap(); + self.settle(); } Step::Shrink if self.o.size() == PAGE => { - self.p.resize(&mut self.o, 0, &mut |_| ()).unwrap(); + self.p.resize(&mut self.o, 0).unwrap(); + self.settle(); } Step::Grow if self.o.size() == 0 => { - self.p.resize(&mut self.o, PAGE, &mut |_| ()).unwrap(); + self.p.resize(&mut self.o, PAGE).unwrap(); + self.settle(); } Step::Refused => { assert!(self.o.take_pager()); @@ -2091,7 +2603,7 @@ mod tests { // The file server fails a requested range past the // end, which inserts nothing there. if self.p.supply(&mut self.o, 0, next, false, PAGE) == Err(Refused::PastEnd) { - self.p.fail(&mut self.o, 0, 1, &mut |_| ()).unwrap(); + self.p.fail(&mut self.o, 0, 1).unwrap(); } } Step::Look if self.waits && self.p.wait_over(&self.o, 0, self.since) => { @@ -2120,6 +2632,13 @@ mod tests { true } + /// A removal's walk, over no mapper ever linked, to its end. + fn settle(&mut self) { + if self.o.revoke() == Revoke::Walk { + assert_eq!(self.p.walk(&mut self.o, &mut super::Unmapped, &mut One(&mut self.want), &mut |_, _, _| ()), Next::Finished); + } + } + /// The pager answers what it hears and the reader looks whenever /// its wait is over: the read ends. pub(super) fn drain(&mut self, steps: &[Step]) { @@ -2158,4 +2677,491 @@ mod tests { } assert!(played > 10_000, "only {played} sequences played"); } + + /// Two spaces mapping one object of two pages, as the kernel drives the + /// index: a fault writes a leaf only for a present page, a visit clears + /// every leaf the index no longer holds present as that frame, and an + /// unmap clears its leaves before its node leaves the ring. + mod mapped { + use super::*; + + #[derive(Clone, Copy, Debug)] + pub(super) enum Step { + Map(u32), + Unmap(u32), + Fault(u32, u64), + Supply(u64), + Withdraw(u64), + Fail(u64), + Shrink, + Grow, + End, + /// The walker's step under the pager's lock. + Walk, + /// The walker's visit of the node it took, under that space's lock. + Visit, + /// The walker's sweep of the futex waits on the walk's set, a + /// frame at a time. + Sweep, + /// The server reads its notices: each request not yet named. + Report, + /// The server answers the oldest request it read, by a supply. + Answer, + /// Every parked fault looks at its page again. + Retry, + /// A thread of space `s` begins a futex wait on a word in `page`: + /// it translates the word to the frame its leaf maps, under the + /// space's lock, and notes the space's count of changes. + Translate(u32, u64), + /// That thread arms its wait, keyed on the frame: where the count + /// has not moved it trusts the translation, and otherwise looks + /// at the leaf again. Only a sweep ends the wait, since a visit or + /// an unmap clears a leaf and revokes no frame it does not own. + Arm, + } + + pub(super) const STEPS: [Step; 25] = [ + Step::Map(0), + Step::Map(1), + Step::Unmap(0), + Step::Unmap(1), + Step::Fault(0, 0), + Step::Fault(0, 1), + Step::Fault(1, 0), + Step::Fault(1, 1), + Step::Supply(0), + Step::Supply(1), + Step::Withdraw(0), + Step::Withdraw(1), + Step::Fail(0), + Step::Shrink, + Step::Grow, + Step::End, + Step::Walk, + Step::Visit, + Step::Sweep, + Step::Report, + Step::Answer, + Step::Retry, + Step::Translate(0, 0), + Step::Translate(1, 1), + Step::Arm, + ]; + + pub(super) struct World { + p: Pages, + o: Object, + ring: Ring, + mapped: [bool; 2], + leaves: [[Option; 2]; 2], + next: u64, + freed: BTreeSet, + walking: bool, + /// The space a walk visits, and where its visit has read the + /// object's revoking pages to. + visiting: Option<(u32, Cursor)>, + /// Where the walker's sweep has read the walk's set to. + sweeping: Option, + /// The frames a futex waiter is keyed on. + futex: BTreeSet, + /// Each space's count of the leaf changes it shot down. + counted: [u64; 2], + /// A futex wait translated and not yet armed: its space, page, + /// frame and the count it read. + translated: Option<(u32, u64, u64, u64)>, + /// Faults parked on an absent page: the space, the page, and the + /// request as the pager hears of it. + parked: std::vec::Vec<(u32, u64, Want, Version)>, + /// Pages the server read requests for and has yet to supply. + asked: std::collections::VecDeque, + } + + /// The parked faults' wants, which removals name again. + struct Parked<'a>(&'a mut std::vec::Vec<(u32, u64, Want, Version)>); + + impl Waiters for Parked<'_> { + fn each(&mut self, f: &mut dyn FnMut(&mut Want)) { + for (_, _, want, _) in self.0.iter_mut() { + f(want); + } + } + } + + impl World { + pub(super) fn new() -> Self { + let mut p = Pages::new(4, 4).unwrap(); + let o = p.object(1, 2 * PAGE); + World { + p, + o, + ring: Ring::default(), + mapped: [false; 2], + leaves: [[None; 2]; 2], + next: 100, + freed: BTreeSet::new(), + walking: false, + visiting: None, + sweeping: None, + futex: BTreeSet::new(), + counted: [0; 2], + translated: None, + parked: std::vec::Vec::new(), + asked: std::collections::VecDeque::new(), + } + } + + /// Whether `step` can be taken now; takes it if so. + pub(super) fn step(&mut self, step: Step) -> bool { + match step { + Step::Map(s) if !self.mapped[s as usize] => { + self.ring.link(s, &mut self.o); + self.mapped[s as usize] = true; + } + Step::Unmap(s) if self.mapped[s as usize] => { + self.leaves[s as usize] = [None; 2]; + self.counted[s as usize] += 1; + let at = self.ring.0.iter().position(|&(n, _)| n == s).expect("linked while mapped"); + self.ring.0.remove(at); + self.mapped[s as usize] = false; + } + Step::Fault(s, page) if self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_none() => { + if !self.touch(s, page) { + if self.p.lookup(&self.o, page) != Lookup::Absent || self.parked.iter().any(|&(t, q, _, _)| (t, q) == (s, page)) { + return false; + } + let mut want = Want::NONE; + want.ask(page, 1); + self.parked.push((s, page, want, self.o.version())); + } + } + Step::Supply(page) if !self.o.gone() => { + self.next += 1; + let _ = self.p.supply(&mut self.o, page, self.next, false, (page + 1) * PAGE); + } + Step::Withdraw(page) if !self.o.gone() => { + self.p.withdraw(&mut self.o, page, 1, &mut Parked(&mut self.parked)); + self.revoke(); + } + Step::Fail(page) if !self.o.gone() => { + let _ = self.p.fail(&mut self.o, page, 1); + self.revoke(); + } + Step::Shrink if !self.o.gone() && self.o.size() == 2 * PAGE => { + self.p.resize(&mut self.o, PAGE).unwrap(); + self.revoke(); + } + Step::Grow if !self.o.gone() && self.o.size() == PAGE => { + self.p.resize(&mut self.o, 2 * PAGE).unwrap(); + self.revoke(); + } + Step::End if !self.o.gone() => { + self.p.invalidate(&mut self.o); + self.revoke(); + } + Step::Walk if self.walking && self.visiting.is_none() && self.sweeping.is_none() => { + let (freed, futex) = (&mut self.freed, &self.futex); + let walked = self.p.walk(&mut self.o, &mut self.ring, &mut Parked(&mut self.parked), &mut |f, _, _| { + assert!(!futex.contains(&f), "frame {f} given back with a futex waiter keyed on it"); + assert!(freed.insert(f), "frame {f} given back twice"); + }); + match walked { + Next::Visit(node) => self.visiting = Some((node, Cursor::Start)), + Next::Sweep => self.sweeping = Some(Cursor::Start), + Next::Finished => self.walking = false, + } + } + // The sweep, a frame at a time: anything may run between. + Step::Sweep if self.sweeping.is_some() => { + let mut at = self.sweeping.take().unwrap(); + let mut batch = [(0, 0)]; + if self.p.revoking(&self.o, &mut at, &mut batch) == 1 { + self.futex.remove(&batch[0].1); + self.sweeping = Some(at); + } + } + Step::Translate(s, page) if self.translated.is_none() && self.mapped[s as usize] && self.leaves[s as usize][page as usize].is_some() => { + let f = self.leaves[s as usize][page as usize].unwrap(); + self.translated = Some((s, page, f, self.counted[s as usize])); + } + Step::Arm if self.translated.is_some() => { + let (s, page, f, at) = self.translated.take().unwrap(); + let holds = self.counted[s as usize] == at || self.leaves[s as usize][page as usize] == Some(f); + if holds { + self.futex.insert(f); + } + } + Step::Report => { + let before = self.asked.len(); + for (_, _, want, _) in &mut self.parked { + if let Some((first, _)) = want.report() { + self.asked.push_back(first); + } + } + if self.asked.len() == before { + return false; + } + } + Step::Answer => { + let Some(page) = self.asked.pop_front() else { return false }; + if !self.o.gone() { + self.next += 1; + let _ = self.p.supply(&mut self.o, page, self.next, false, (page + 1) * PAGE); + } + } + Step::Retry if !self.parked.is_empty() => { + let parked = core::mem::take(&mut self.parked); + // A wake may be stale: each looks whether or not its + // wait is over, as a thread woken for another's page. + for (s, page, mut want, since) in parked { + if !self.mapped[s as usize] || self.touch(s, page) { + continue; + } + match self.p.lookup(&self.o, page) { + // A write or resize ended the wait: ask afresh. + Lookup::Absent if !self.o.unchanged(since) => { + want.anew(); + want.ask(page, 1); + self.parked.push((s, page, want, self.o.version())); + } + Lookup::Absent => { + want.ask(page, 1); + self.parked.push((s, page, want, since)); + } + // Refused: failed, past the end, or gone. + _ => {} + } + } + } + // The kernel's visit, a batch of one: the next revoking + // page read, and the space's leaf of it cleared where it + // maps that frame; anything may run between batches. + Step::Visit if self.visiting.is_some() => { + let (s, mut at) = self.visiting.take().unwrap(); + let mut batch = [(0, 0)]; + if self.p.revoking(&self.o, &mut at, &mut batch) == 1 { + let (page, f) = batch[0]; + if self.leaves[s as usize][page as usize] == Some(f) { + self.leaves[s as usize][page as usize] = None; + // The visit's shootdown is counted, as an + // unmap's is. + self.counted[s as usize] += 1; + } + self.visiting = Some((s, at)); + } + } + _ => return false, + } + self.check(step); + true + } + + /// A fault's hit: `page` and its neighbour written where present. + fn touch(&mut self, s: u32, page: u64) -> bool { + let Lookup::Present(_) = self.p.lookup(&self.o, page) else { return false }; + for at in 0..2 { + if let (None, Lookup::Present(f)) = (self.leaves[s as usize][at], self.p.lookup(&self.o, at as u64)) { + self.leaves[s as usize][at] = Some(f); + } + } + true + } + + /// Once nothing is removed any more, the walk ends, the server + /// answers every request it hears of, and every parked fault + /// retries: none is left parked. + pub(super) fn drain(&mut self, steps: &[Step]) { + let fair = [Step::Walk, Step::Visit, Step::Sweep, Step::Report, Step::Answer, Step::Retry]; + for _ in 0..64 { + if !fair.iter().fold(false, |moved, &step| self.step(step) | moved) { + break; + } + } + assert!(self.parked.is_empty(), "after {steps:?}: faults left parked {:?}", self.parked); + } + + fn revoke(&mut self) { + match self.o.revoke() { + Revoke::Walk => { + assert!(!self.walking, "a second walker"); + self.walking = true; + } + Revoke::Queued(_) => assert!(self.walking, "queued behind no walk"), + Revoke::Done => {} + } + } + + /// Every leaf maps the frame the index holds for its page, present + /// or revoking, and never one given back. + fn check(&self, step: Step) { + for (s, leaves) in self.leaves.iter().enumerate() { + for (page, leaf) in leaves.iter().enumerate() { + let Some(f) = *leaf else { continue }; + assert!(!self.freed.contains(&f), "after {step:?}: space {s} page {page} maps given-back frame {f}"); + let held = self.p.find(self.o.id, page as u64).map(|i| self.p.slot(i).frame); + assert_eq!(held, Some(Some(f)), "after {step:?}: space {s} page {page} maps {f}, the index holds {held:?}"); + } + } + assert!(self.walking || self.o.revoking == 0, "after {step:?}: pages revoking with no walk"); + assert!(self.futex.is_disjoint(&self.freed), "after {step:?}: a futex waiter keyed on a given-back frame"); + } + } + } + + fn play(steps: &[mapped::Step]) -> Option { + let mut w = mapped::World::new(); + steps.iter().all(|&s| w.step(s)).then_some(w) + } + + /// Every sequence of `depth` steps after `prefix`, each then drained: how + /// many were played. + fn explore(prefix: &[mapped::Step], depth: usize) -> u64 { + let mut stack = std::vec![prefix.to_vec()]; + let mut played = 0u64; + while let Some(seq) = stack.pop() { + played += 1; + play(&seq).expect("a sequence played before").drain(&seq); + if seq.len() == prefix.len() + depth { + continue; + } + for &step in &mapped::STEPS { + let mut next = seq.clone(); + next.push(step); + if play(&next).is_some() { + stack.push(next); + } + } + } + played + } + + /// From both spaces mapping both pages, every sequence of five steps of + /// faults, supplies, removals, mappings and the walk keeps each leaf on + /// its page's one frame, and leaves no fault parked once drained. + #[test] + fn every_interleaving_keeps_each_leaf_on_its_pages_one_frame() { + use mapped::Step; + let played = explore(&[Step::Map(0), Step::Map(1), Step::Supply(0), Step::Supply(1), Step::Fault(0, 0), Step::Fault(1, 1)], 5); + assert!(played > 20_000, "only {played} sequences played"); + } + + /// A futex wait that translated its word before a walk's visit cleared + /// the leaf, and arms only after the walk's sweep, does not arm on the + /// frame the walk then gives back: the visit's shootdown is counted, so + /// the arm looks at the leaf again. + #[test] + fn a_wait_translated_before_a_visit_does_not_arm_on_a_frame_given_back() { + use mapped::Step; + let steps = [ + Step::Map(0), + Step::Supply(0), + Step::Supply(1), + Step::Fault(0, 0), + Step::Translate(0, 0), + Step::Withdraw(0), + Step::Walk, + Step::Visit, + Step::Visit, + Step::Walk, + Step::Sweep, + Step::Sweep, + Step::Arm, + Step::Walk, + ]; + play(&steps).expect("every step is taken").drain(&steps); + } + + /// From a futex wait translated before its page was withdrawn and a walk + /// begun, every sequence of six steps arms no waiter on a frame the walk + /// gives back, whenever it arms: a visit's clear is counted, so the arm + /// looks at the leaf again. + #[test] + fn a_wait_translated_before_a_removal_never_arms_on_a_frame_given_back() { + use mapped::Step; + let prefix = [ + Step::Map(0), + Step::Supply(0), + Step::Supply(1), + Step::Fault(0, 0), + Step::Translate(0, 0), + Step::Withdraw(0), + Step::Walk, + ]; + let played = explore(&prefix, 6); + assert!(played > 100_000, "only {played} sequences played"); + } + + /// From a futex waiter an unmap left keyed on a frame and a walk begun + /// with no mapper left, every sequence of five steps gives back no frame + /// a waiter is keyed on: a page marked while the sweep has its lock + /// dropped is swept before the walk ends. + #[test] + fn a_waiter_an_unmap_left_is_never_on_a_frame_given_back() { + use mapped::Step; + let prefix = [ + Step::Map(0), + Step::Supply(0), + Step::Supply(1), + Step::Fault(0, 0), + Step::Translate(0, 0), + Step::Arm, + Step::Unmap(0), + Step::Withdraw(1), + Step::Walk, + ]; + let played = explore(&prefix, 5); + assert!(played > 1_000, "only {played} sequences played"); + } + + /// From a walk begun on an object no mapper has linked, which ends with + /// no sweep, every sequence of five steps gives back no frame a futex + /// waiter is keyed on: a mapper linked during the walk maps nothing + /// revoking, and the walk sweeps the pages removed once it has linked. + #[test] + fn a_walk_begun_before_any_mapping_gives_back_no_frame_a_waiter_is_keyed_on() { + use mapped::Step; + let played = explore(&[Step::Supply(0), Step::Supply(1), Step::Withdraw(0)], 5); + assert!(played > 10_000, "only {played} sequences played"); + } + + /// A fault whose request the pager read and answered, before the fault + /// looked again, meets a shrink and a grow: the version moved, so it + /// asks afresh and is heard. + #[test] + fn a_fault_answered_then_shrunk_and_grown_asks_afresh() { + use mapped::Step; + let steps = [ + Step::Map(0), + Step::Fault(0, 1), + Step::Report, + Step::Answer, + Step::Shrink, + Step::Walk, + // A batch of one: the revoked page, then the read that finds none. + Step::Visit, + Step::Visit, + Step::Walk, + // The sweep: the revoked page's frame, then the read that finds none. + Step::Sweep, + Step::Sweep, + Step::Walk, + Step::Grow, + Step::Retry, + ]; + play(&steps).expect("every step is taken").drain(&steps); + } + + /// The same over long sequences, seeded: removals queued behind running + /// walks, ends meeting walks, unmaps while their node is visited. + #[test] + fn long_runs_keep_each_leaf_on_its_pages_one_frame() { + let mut x = 7u64; + for _ in 0..2_000 { + let mut w = mapped::World::new(); + for _ in 0..200 { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + w.step(mapped::STEPS[(x >> 33) as usize % mapped::STEPS.len()]); + } + w.drain(&[]); + } + } } diff --git a/kernel/pure/vm.rs b/kernel/pure/vm.rs index a462a7ca04..091d7d2d83 100644 --- a/kernel/pure/vm.rs +++ b/kernel/pure/vm.rs @@ -21,6 +21,14 @@ pub const PAGE: u64 = 4096; /// run of 64 KiB. pub const AROUND: u64 = 16 * PAGE; +/// Where a range at `[start, end)` that maps its object from byte `offset` +/// maps the object's `page`, if it does. A pager names the page, so every +/// step is checked and none overflows. +pub fn page_at(start: u64, end: u64, offset: u64, page: u64) -> Option { + let into = page.checked_mul(PAGE)?.checked_sub(offset)?; + (into < end - start).then(|| start + into) +} + /// The pages `[from, to)` a touch of the empty `page` of the range `[start, /// end)` populates: its own, and where a page beside it in the range is /// `populated`, every page of its aligned run of [`AROUND`] in the range. So a @@ -393,6 +401,20 @@ mod tests { use std::vec; use std::vec::Vec; + /// A page a range maps lands at its place; one before the range's + /// offset, at or past its end, or so high its byte offset overflows is + /// no place, never an overflow: a pager names the page. + #[test] + fn a_page_is_placed_only_inside_its_range() { + let (start, end) = (0x10_0000, 0x10_0000 + 4 * PAGE); + assert_eq!(page_at(start, end, 2 * PAGE, 2), Some(start)); + assert_eq!(page_at(start, end, 2 * PAGE, 5), Some(start + 3 * PAGE)); + assert_eq!(page_at(start, end, 2 * PAGE, 6), None, "the range's end"); + assert_eq!(page_at(start, end, 2 * PAGE, 1), None, "before the range's offset"); + assert_eq!(page_at(start, end, 0, (1 << 52) - 2), None, "a page near the last a pager may name"); + assert_eq!(page_at(start, end, 0, u64::MAX), None, "a byte offset past u64"); + } + /// Shared memory is executed only where nobody can write it and /// written only where somebody can; a range is whole pages inside the /// object, and an aperture's is the whole of it. diff --git a/kernel/src/arch/aarch64/trap.rs b/kernel/src/arch/aarch64/trap.rs index 546bb2eb2a..cd7aef57af 100644 --- a/kernel/src/arch/aarch64/trap.rs +++ b/kernel/src/arch/aarch64/trap.rs @@ -151,6 +151,7 @@ fn user_copy_abort(frame: &mut Frame) -> bool { if frame.esr >> 26 != EC_DABT_SAME || !far_valid || !toyos_userbound::is_user_addr(frame.far) { return false; } + let mut faulted = crate::process::Faulted::Unreachable; if is_translation_fault(frame.esr) { let prev = percpu::swap_fault_state(CpuFaultState::PageFault); if prev != CpuFaultState::Normal { @@ -161,13 +162,14 @@ fn user_copy_abort(frame: &mut Frame) -> bool { cpu::enable_interrupts(); // `WnR`, bit 6: the access was a write. let access = if frame.esr & (1 << 6) != 0 { kernel::vm::Access::Write } else { kernel::vm::Access::Read }; - let served = crate::process::handle_page_fault(frame.far, access); + faulted = crate::process::handle_page_fault(frame.far, access); cpu::disable_interrupts(); percpu::set_fault_state(CpuFaultState::Normal); - if served { + if faulted == crate::process::Faulted::Served { return true; } } + crate::user_ptr::stopped(faulted.stop()); frame.elr = fixup; true } @@ -354,12 +356,21 @@ fn user_abort(frame: &mut Frame) { (false, true) => kernel::vm::Access::Write, (false, false) => kernel::vm::Access::Read, }; - let served = crate::process::handle_page_fault(frame.far, access); + let faulted = crate::process::handle_page_fault(frame.far, access); cpu::disable_interrupts(); - if served { - percpu::set_fault_state(CpuFaultState::Normal); - percpu::preempt_count_down(); - return; + match faulted { + crate::process::Faulted::Unreachable => {} + crate::process::Faulted::Lost(lost) | crate::process::Faulted::PastEnd(lost) => { + percpu::set_fault_state(CpuFaultState::Normal); + crate::process::lost(frame.far, lost); + } + crate::process::Faulted::Absent => unreachable!("an EL0 fault while its thread's copies hold pins"), + // A kill that ended a wait is acted on as the thread returns to EL0. + crate::process::Faulted::Served | crate::process::Faulted::Killed => { + percpu::set_fault_state(CpuFaultState::Normal); + percpu::preempt_count_down(); + return; + } } log!( "#PF UNHANDLED: far={:#x} pc={:#x} esr={:#x} tid={:?}", diff --git a/kernel/src/arch/aarch64/uaccess.rs b/kernel/src/arch/aarch64/uaccess.rs index 38580fd699..3646cd71b7 100644 --- a/kernel/src/arch/aarch64/uaccess.rs +++ b/kernel/src/arch/aarch64/uaccess.rs @@ -23,7 +23,7 @@ use toyos_userbound::USER_TOP; -use crate::user_ptr::Faulted; +use crate::user_ptr::Stopped; core::arch::global_asm!( ".pushsection .text.toyos_uaccess, \"ax\"", @@ -165,8 +165,8 @@ unsafe extern "C" { } /// A routine's answer: 0 where it copied every byte. -fn whole(left: usize) -> Result<(), Faulted> { - if left == 0 { Ok(()) } else { Err(Faulted) } +fn whole(left: usize) -> Result<(), Stopped> { + if left == 0 { Ok(()) } else { Err(crate::user_ptr::last_stop()) } } /// Copy `len` bytes from user address `user` to `dst`. On `Err` some or none @@ -175,7 +175,7 @@ fn whole(left: usize) -> Result<(), Faulted> { /// # Safety /// `dst` is writable kernel memory for `len` bytes; `[user, user + len)` lies /// below [`USER_TOP`]; no lock is held (`crate::user_ptr`). -pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_copy_from_user(dst, user, len) }) } @@ -184,7 +184,7 @@ pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`], with `src` readable. -pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_copy_to_user(user, src, len) }) } @@ -193,7 +193,7 @@ pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`]. -pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_clear_user(user, len) }) } diff --git a/kernel/src/arch/x86_64/idt/exceptions.rs b/kernel/src/arch/x86_64/idt/exceptions.rs index a67e742e6a..423117689b 100644 --- a/kernel/src/arch/x86_64/idt/exceptions.rs +++ b/kernel/src/arch/x86_64/idt/exceptions.rs @@ -532,9 +532,21 @@ pub(super) fn page_fault_handler(frame: &mut TrapFrame) { } else { kernel::vm::Access::Read }; - if is_user && process::handle_page_fault(fault_addr, access) { - percpu::set_fault_state(percpu::CpuFaultState::Normal); - return; + if is_user { + match process::handle_page_fault(fault_addr, access) { + process::Faulted::Unreachable => {} + process::Faulted::Lost(lost) | process::Faulted::PastEnd(lost) => { + percpu::set_fault_state(percpu::CpuFaultState::Normal); + process::lost(fault_addr, lost); + } + process::Faulted::Absent => unreachable!("a Ring 3 fault while its thread's copies hold pins"), + // A kill that ended a wait is acted on as the thread returns + // to Ring 3. + process::Faulted::Served | process::Faulted::Killed => { + percpu::set_fault_state(percpu::CpuFaultState::Normal); + return; + } + } } log!("#PF UNHANDLED: cr2={:#x} rip={:#x} err={:#x} user={} tid={:?}", fault_addr, frame.rip, frame.error_code, is_user, percpu::current_tid()); @@ -559,14 +571,18 @@ fn user_copy_fault(frame: &mut TrapFrame, addr: u64, fixup: u64) { const AC: u64 = 1 << 18; assert!(frame.rflags & AC != 0, "uaccess: a copy faulted at {:#x} with RFLAGS.AC clear", frame.rip); let access = if frame.error_code & PF_WRITE != 0 { kernel::vm::Access::Write } else { kernel::vm::Access::Read }; - if frame.error_code & PF_PRESENT == 0 { + let faulted = if frame.error_code & PF_PRESENT == 0 { percpu::preempt_count_down(); - let served = process::handle_page_fault(addr, access); + let faulted = process::handle_page_fault(addr, access); percpu::preempt_count_up(); - if served { - return; - } + faulted + } else { + process::Faulted::Unreachable + }; + if faulted == process::Faulted::Served { + return; } + crate::user_ptr::stopped(faulted.stop()); frame.rip = fixup; } diff --git a/kernel/src/arch/x86_64/uaccess.rs b/kernel/src/arch/x86_64/uaccess.rs index 5d85738367..62009a7077 100644 --- a/kernel/src/arch/x86_64/uaccess.rs +++ b/kernel/src/arch/x86_64/uaccess.rs @@ -23,7 +23,7 @@ use toyos_userbound::USER_TOP; use super::copy::{self, copy_down_asm}; -use crate::user_ptr::Faulted; +use crate::user_ptr::Stopped; core::arch::global_asm!( ".pushsection .text.toyos_uaccess, \"ax\"", @@ -139,8 +139,8 @@ unsafe extern "C" { const REP_STRING_LEN: u64 = 2; /// A routine's answer: 0 where it copied every byte. -fn whole(left: usize) -> Result<(), Faulted> { - if left == 0 { Ok(()) } else { Err(Faulted) } +fn whole(left: usize) -> Result<(), Stopped> { + if left == 0 { Ok(()) } else { Err(crate::user_ptr::last_stop()) } } /// Copy `len` bytes from user address `user` to `dst`. On `Err` some or none @@ -149,7 +149,7 @@ fn whole(left: usize) -> Result<(), Faulted> { /// # Safety /// `dst` is writable kernel memory for `len` bytes; `[user, user + len)` lies /// below [`USER_TOP`]; no lock is held (`crate::user_ptr`). -pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; each routine touches nothing else, and // the rule sends down only a copy of at least 8 bytes. whole(unsafe { @@ -164,7 +164,7 @@ pub unsafe fn copy_from_user(dst: *mut u8, user: u64, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`], with `src` readable. -pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Faulted> { +pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), Stopped> { // SAFETY: as in `copy_from_user`. whole(unsafe { match copy::rule().goes_down(user as usize, src as usize, len) { @@ -178,7 +178,7 @@ pub unsafe fn copy_to_user(user: u64, src: *const u8, len: usize) -> Result<(), /// /// # Safety /// As [`copy_from_user`]. -pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Faulted> { +pub unsafe fn clear_user(user: u64, len: usize) -> Result<(), Stopped> { // SAFETY: the caller's contract; the routine touches nothing else. whole(unsafe { toyos_clear_user(user, len) }) } diff --git a/kernel/src/mm/space.rs b/kernel/src/mm/space.rs index 16ba4c85db..93a607745a 100644 --- a/kernel/src/mm/space.rs +++ b/kernel/src/mm/space.rs @@ -33,13 +33,32 @@ use super::charge::{self, Charge}; use super::paging::{Leaf, Mapping, Root, Unlinked, UserTables}; use super::policy::{self, CachePolicy}; use super::{pmm, PAGE_2M, PAGE_SIZE}; +use crate::object::paged::{Lost, MapNode, Missing, PagedHolderObject, PagedRange}; use crate::object::shm::SharedMemObject; use crate::sync::{Lock, LockGuard}; use crate::UserAddr; -type Inventory = vm::Inventory, Arc>; -type SpaceEntry = Entry, Arc>; -type SpaceKind = Kind, Arc>; +type Inventory = vm::Inventory>; +type SpaceEntry = Entry>; +type SpaceKind = Kind>; + +/// What an object range maps: a memory object's pages, fixed for its life, +/// or a paged object's, which its pager's index holds and revokes. +#[derive(Clone)] +pub enum Mapped { + Memory(Arc), + Paged(PagedRange), +} + +impl vm::Object for Mapped { + fn same(&self, other: &Self) -> bool { + match (self, other) { + (Mapped::Memory(a), Mapped::Memory(b)) => Arc::ptr_eq(a, b), + (Mapped::Paged(a), Mapped::Paged(b)) => core::ptr::eq(a.node(), b.node()), + _ => false, + } + } +} /// The bytes a copied range's pages start as. A read runs under the space /// lock, from a fault or a user copy, so it may take no lock a user copy can @@ -226,17 +245,23 @@ impl FutexWait { /// The frame behind the futex word at `addr`, populated where its range /// allows a read ([`AddressSpace::reach`]), as a translation the wait can - /// ask about later without the lock. - pub fn translate(&self, addr: crate::UserAddr) -> Option> { + /// ask about later without the lock; why there is none, waiting for no + /// mapped file's page. + pub fn translate(&self, addr: crate::UserAddr) -> Result, crate::user_ptr::Untranslated> { + use crate::user_ptr::Untranslated; let space: &Space = &self.0; let (phys, at) = { let mut held = space.lock(); - let phys = held.reach(addr.raw(), Access::Read)?; + let phys = match held.reach(addr.raw(), Access::Read) { + Populated::At(phys) => phys, + Populated::Unreachable | Populated::PastEnd(_) => return Err(Untranslated::Unreachable), + Populated::Lost(_) | Populated::Wait(_) => return Err(Untranslated::Away), + }; // Counted under this hold: a change made before it is in `at`, // and one after it moves the count. (phys, space.usage.shot_down.load(Relaxed)) }; - Some(Translation { space, va: addr.raw(), phys, at: Cell::new(at) }) + Ok(Translation { space, va: addr.raw(), phys, at: Cell::new(at) }) } } @@ -248,7 +273,10 @@ impl Drop for FutexWait { /// A futex word's frame as a hold of its space's lock found it, borrowed from /// the wait that counts it. Not a lease: nothing pins the frame, and a -/// sibling's `munmap` can free it under a parked wait. +/// sibling's `munmap` can free it under a parked wait. A mapped file's frame +/// goes back only after its pager's walk: each visit counts its clear, so +/// [`holds`](Self::holds) looks at the leaf again, and the walk's sweep ends +/// the waits keyed on the frame before it is reissued. pub struct Translation<'w> { space: &'w Space, va: u64, @@ -502,10 +530,67 @@ impl AddressSpace { self.room_at(len, at)?; let start = self.place(len, at)?; self.build(start, start + len)?; - self.put(start, Entry { end: start + len, prot, kind: Kind::Object { object, offset, max: prot } }); + self.put(start, Entry { end: start + len, prot, kind: Kind::Object { object: Mapped::Memory(object), offset, max: prot } }); Ok(UserAddr::new(start)) } + /// `len` bytes of `holder`'s object from `offset`, as `prot` allows, + /// sharing the index's frames: one node per object in a space, so a + /// second range of the object holds the first's. `space` is this space, + /// whose lock the caller holds. All or nothing, as + /// [`reserve`](Self::reserve). + pub fn map_paged(&mut self, space: &Arc, holder: &PagedHolderObject, offset: u64, len: u64, prot: Prot, at: At) -> Result { + if !offset.is_multiple_of(PAGE_SIZE) || offset.checked_add(len).is_none_or(|end| end > kernel::paged::LAST_END) { + return Err(SyscallError::InvalidArgument); + } + self.room_at(len, at)?; + let start = self.place(len, at)?; + let held = self.inventory.meeting(FLOOR, TOP).find_map(|(_, entry)| match &entry.kind { + Kind::Object { object: Mapped::Paged(range), .. } if holder.maps(range) => Some(range.clone()), + _ => None, + }); + let range = match held { + Some(range) => range, + None => holder.link(Arc::downgrade(space), &self.job)?, + }; + self.build(start, start + len)?; + self.put(start, Entry { end: start + len, prot, kind: Kind::Object { object: Mapped::Paged(range), offset, max: prot } }); + Ok(UserAddr::new(start)) + } + + /// One batch of a walk's visit of `node`'s object ([`MapNode`]): every + /// leaf of this space's ranges of it that maps one of `revoking`'s + /// `(page, frame)` is cleared, and the span shot down, under this space's + /// lock alone. The shootdown is counted ([`Translation::holds`]), so a + /// futex wait that translated before the clear looks at its leaf again + /// and does not arm on a frame the walk goes on to give back; the walk + /// ends the waits already keyed on those frames once every mapper is + /// visited. A fault meanwhile waits on this space's lock and then finds + /// the page as the index has it. + pub fn clear_paged(&mut self, node: &MapNode, revoking: &[(u64, u64)]) { + let (mut lo, mut hi) = (u64::MAX, 0); + let tables = &mut self.tables; + for (start, entry) in self.inventory.meeting(FLOOR, TOP) { + let Kind::Object { object: Mapped::Paged(range), offset, .. } = &entry.kind else { continue }; + if !core::ptr::eq(range.node(), node) { + continue; + } + for &(page, phys) in revoking { + let Some(va) = vm::page_at(start, entry.end, *offset, page) else { continue }; + tables.edit(va, va + PAGE_SIZE, |_, leaf| match leaf { + Leaf::Mapped(m) if m.phys == phys => { + (lo, hi) = (lo.min(va), hi.max(va + PAGE_SIZE)); + Some(Leaf::Empty) + } + _ => None, + }); + } + } + if lo < hi { + self.shoot_down(lo, hi, &Unlinked::none()); + } + } + /// Take every page of `[start, start + len)` away, whatever holds it: the /// POSIX `munmap` of any run of pages. Nothing held there is not a refusal. pub fn release(&mut self, start: UserAddr, len: u64) { @@ -531,6 +616,15 @@ impl AddressSpace { self.settle(); } + /// Empty the space as its process ends, every range taken away as + /// [`release`](Self::release) takes one: a paged range lets its node go + /// only after its leaves are cleared and shot down, so the space's last + /// reference, wherever it drops (a revoker's visit among them), frees + /// nothing but its empty tables. + pub fn end(&mut self) { + self.release(UserAddr::new(FLOOR), TOP - FLOOR); + } + /// Give every page of `[start, start + len)` protection `prot`: a reserved /// private page is committed unless `prot` is `None`, promised its frame /// here, and a committed one keeps its frame and commitment whatever @@ -790,7 +884,7 @@ impl AddressSpace { let mut at = start; while at < end { let next = self.inventory.meeting(at, end).find_map(|(s, entry)| match &entry.kind { - Kind::Object { object: held, offset, .. } + Kind::Object { object: Mapped::Memory(held), offset, .. } if core::ptr::eq(Arc::as_ptr(held), object) && *offset + at.saturating_sub(s) == at.max(s) - start => { Some((at.max(s), entry.end.min(end))) @@ -850,27 +944,28 @@ impl AddressSpace { } /// Serve a fault at `va` for `access`: populate its page if its range - /// allows the access and the page is empty. `false` ends the thread. - pub fn fault(&mut self, va: u64, access: Access) -> bool { - self.populate(va, access, |_| true).is_some() + /// allows the access and the page is empty, or say which paged page to + /// wait for. + pub fn fault(&mut self, va: u64, access: Access) -> Populated { + self.populate(va, access, |_| true) } /// Where the byte at `va` is, for the kernel to reach through the direct - /// map, populating its page where its range allows `access`; `None` for - /// an aperture's page, which the direct map does not reach. - pub fn reach(&mut self, va: u64, access: Access) -> Option { + /// map, populating its page where its range allows `access`; an + /// aperture's page, which the direct map does not reach, is unreachable. + pub fn reach(&mut self, va: u64, access: Access) -> Populated { self.populate(va, access, copyable) } /// The frame behind `va`, populated where its range allows `access` and /// is one `admits`. - fn populate(&mut self, va: u64, access: Access, admits: fn(&SpaceEntry) -> bool) -> Option { - let (start, entry) = self.inventory.find(va)?; + fn populate(&mut self, va: u64, access: Access, admits: fn(&SpaceEntry) -> bool) -> Populated { + let Some((start, entry)) = self.inventory.find(va) else { return Populated::Unreachable }; if !admits(entry) || !entry.prot.allows(access) { - return None; + return Populated::Unreachable; } let page = va & !(PAGE_SIZE - 1); - let prot = leaf_prot(entry.prot)?; + let Some(prot) = leaf_prot(entry.prot) else { return Populated::Unreachable }; // A range that allows an access is backed, so its tables are built. let mut leaves = self.tables.leaf_table(page).unwrap_or_else(|| panic!("space: no tables under {page:#x}")); let phys = match leaves.get(page) { @@ -886,11 +981,35 @@ impl AddressSpace { let (lo, hi) = vm::around(start, entry.end, page, |at| leaves.covers(at) && leaves.get(at) != Leaf::Empty); match &entry.kind { Kind::Private { committed: false } => unreachable!("space: a reservation at {page:#x} allows an access"), - Kind::Object { object, offset, .. } => { + Kind::Object { object: Mapped::Memory(object), offset, .. } => { let phys = |at: u64| object.page(offset + (at - start)); leaves.fill(lo, hi, |at| Leaf::Mapped(Mapping { phys: phys(at), prot, owned: false, cache: object.cache() })); phys(page) } + // The index's own frames, written under its lock: a + // revoking page reads absent, so no leaf maps a frame on + // its way out. + Kind::Object { object: Mapped::Paged(range), offset, .. } => { + let of = |at: u64| (offset + (at - start)) / PAGE_SIZE; + let found = range.node().with_index(|object| { + let own = object.fault(of(page))?; + let mut at = lo; + while at < hi { + if let (Leaf::Empty, Ok(phys)) = (leaves.get(at), object.page(of(at))) { + leaves.set(at, Leaf::Mapped(Mapping { phys, prot, owned: false, cache: CachePolicy::Normal })); + } + at += PAGE_SIZE; + } + Ok(own) + }); + match found { + Ok(phys) => phys, + Err(Missing::Absent(since)) => return Populated::Wait(PageWait { range: range.clone(), page: of(page), since }), + Err(Missing::Hole) => unreachable!("space: a fault's touch of a hole is filled"), + Err(Missing::Lost(lost)) => return Populated::Lost(lost), + Err(Missing::PastEnd(record)) => return Populated::PastEnd(record), + } + } Kind::Private { committed: true } | Kind::Copy { .. } => { let (mut filled, mut own) = (0, 0); leaves.fill(lo, hi, |at| { @@ -908,7 +1027,7 @@ impl AddressSpace { } } }; - Some(phys + va % PAGE_SIZE) + Populated::At(phys + va % PAGE_SIZE) } /// The frame and protection a present leaf maps `va` with, populating @@ -921,10 +1040,44 @@ impl AddressSpace { } } +/// What a touch of a page found. +pub enum Populated { + /// The byte's physical address. + At(u64), + /// No range allows the access there. + Unreachable, + /// A paged page its pager failed or can no longer supply. + Lost(Lost), + /// A paged page wholly past its object's end, with the record a touch + /// of it ends the process with. + PastEnd(Lost), + /// A paged page that is absent: the thread waits for it, holding nothing, + /// and touches it again. + Wait(PageWait), +} + +/// A paged page a fault waits for: the object's page, and its version when +/// it was found absent ([`MapNode::wait`]). +pub struct PageWait { + range: PagedRange, + page: u64, + since: kernel::paged::Version, +} + +impl PageWait { + /// Wait, holding no lock, until the page is no longer absent, the + /// object's version moved, or its pager is gone: `false` where a kill + /// ended the wait. + pub fn wait(self) -> bool { + self.range.node().wait(self.page, self.since) + } +} + /// Whether a range's pages are memory the direct map reaches. fn copyable(entry: &SpaceEntry) -> bool { match &entry.kind { - Kind::Object { object, .. } => object.is_ordinary(), + Kind::Object { object: Mapped::Memory(object), .. } => object.is_ordinary(), + Kind::Object { object: Mapped::Paged(_), .. } => true, Kind::Private { .. } | Kind::Copy { .. } => true, } } @@ -957,9 +1110,12 @@ fn copy_in(source: &dyn CopySource, offset: u64, len: u64, phys: u64) { /// Not through [`AddressSpace::clear`]: no thread runs in a space that drops, /// no CPU holds its root ([`Space::retire`]), and a private frame is mapped in /// no other, so no CPU caches its leaves, no futex waiter is keyed on its -/// frames and no copy pins them. +/// frames and no copy pins them. A paged range is gone by then: its process's +/// end took every range ([`AddressSpace::end`]), and a spawn lays out none. impl Drop for AddressSpace { fn drop(&mut self) { + let paged = self.inventory.meeting(FLOOR, TOP).any(|(_, entry)| matches!(entry.kind, Kind::Object { object: Mapped::Paged(_), .. })); + assert!(!paged, "space: dropped holding a paged range its end did not take"); let committed = self.committed_pages(FLOOR, TOP); self.tables.edit(FLOOR, TOP, |_, leaf| matches!(leaf, Leaf::Mapped(Mapping { owned: false, .. })).then_some(Leaf::Empty)); let mut freed = 0; diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index f331a71349..c211a8d40f 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -341,7 +341,7 @@ pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { KObjectRef::PipeRead(r) => match r.pipe().read(&crate::scheduler::Parkable::at_entry(), buf) { Ok(read) => read.map(|n| n as u64), Err(pipe::ReadStopped::Cancelled) => Some(SyscallError::Gone.to_u64()), - Err(pipe::ReadStopped::Faulted) => Some(SyscallError::BadAddress.to_u64()), + Err(pipe::ReadStopped::Faulted(faulted)) => Some(SyscallError::from(faulted).to_u64()), }, KObjectRef::Device(_) => unreachable!("a device claim is read by `read_device`"), KObjectRef::Null(_) => Some(0), @@ -376,7 +376,7 @@ pub fn pipe_written(wrote: pipe::PipeWrite) -> Option { // The reader is gone, not absent: `NotFound` means the name is not there and nothing else may say so. pipe::PipeWrite::BrokenPipe => Some(SyscallError::Gone.to_u64()), pipe::PipeWrite::NoMemory => Some(SyscallError::ResourceExhausted.to_u64()), - pipe::PipeWrite::Faulted => Some(SyscallError::BadAddress.to_u64()), + pipe::PipeWrite::Faulted(faulted) => Some(SyscallError::from(faulted).to_u64()), pipe::PipeWrite::Wrote(n) => Some(n as u64), pipe::PipeWrite::NoRoom => None, } @@ -541,8 +541,10 @@ fn write_device(claim: &DeviceClaim, buf: &UserBytes) -> u64 { return SyscallError::PermissionDenied.to_u64(); } let mut word = [0u8; 4]; - if buf.len() == word.len() && buf.read_at(0, &mut word).is_err() { - return SyscallError::BadAddress.to_u64(); + if buf.len() == word.len() { + if let Err(faulted) = buf.read_at(0, &mut word) { + return SyscallError::from(faulted).to_u64(); + } } let acknowledges = buf.len() == word.len() && u32::from_ne_bytes(word) == ack; let pid = crate::process::current_process(); diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 8f8ea45ba9..e8b03551f7 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -1,6 +1,6 @@ //! Paged objects: a pager process's memory, one object per file node, which //! holders read, write and stat through `SYS_PAGED_READ`, `SYS_PAGED_WRITE` -//! and `SYS_PAGED_STAT`. Nothing here is mapped. +//! and `SYS_PAGED_STAT`, or map read-only. //! //! **Two kinds.** A [`PagerObject`] is its process's one pager: its index of //! `capacity` pages ([`kernel::paged::Pages`]), its objects by id, its @@ -12,7 +12,7 @@ //! may be narrowed and sent. //! //! **One write at a time** ([`kernel::paged::Side`]): a holder's write -//! copies its bytes into frames held aside first, then takes the object's +//! copies its bytes into frames drawn first, then takes the object's //! write side in one hold of the lock, refused at once while the side is held //! or the pager waits for it, and is checked whole and admitted there; its //! copy into the object's frames is the kernel's own. The pager holds the @@ -25,9 +25,16 @@ //! its record of the rest is bytes the kernel keeps and never reads. //! //! **Supply copies into fresh frames**, so no frame the pager still maps -//! enters an object. A frame leaves at withdrawal, failure, shrink or the -//! object's release: out of the index under the pager's lock, then freed, -//! which a reader's pin defers to its last unpin. +//! enters an object. A frame leaves at withdrawal, failure, shrink, the +//! object's release or the pager's end: its page is revoking, keeping the +//! frame, until a walk of the object's mappers ([`MapNode`]) has cleared +//! every space's leaves of it; then it leaves the index under the pager's +//! lock and is freed, which a reader's pin defers to its last unpin. +//! +//! **A mapping shares the index's frames.** A space mapping an object holds +//! one [`MapNode`] on the object's list while it has a range of it, and +//! counts as one of its holders. A fault on an absent page requests it and +//! waits as a reader does, holding nothing. //! //! **A reader waits only on its pager and on the kernel's copies.** An //! absent page is requested in the waiter's own record, linked on its @@ -37,9 +44,11 @@ //! counted and logged overdue and goes on. A pager's own process is refused a //! holder's calls, so no pager waits on a pager. //! -//! **Lock order.** The pager's lock is innermost but for the allocator's. A -//! copy to or from user memory holds pins, never the lock; no wait holds a -//! pin. +//! **Lock order.** A space's lock, then the pager's, then the allocator's. +//! A copy to or from user memory holds pins, or a holder's write the frames +//! it drew, never the lock; no wait holds either, so such a copy stops at an +//! absent page (`user_ptr::Pinned`), lets them go, and waits for the page +//! holding nothing. //! //! **What a pager holds** (`issues/a-pager-holds-a-fixed-capacity-until-the-memory-service-reclaims-its-clean-pages.md`): //! its core — `dirty` pages and a read's headroom — committed at its making @@ -51,7 +60,7 @@ //! life (`issues/one-client-can-fill-the-tmp-servers-memory-and-deny-every-other-client.md`), //! so making an object is the one call that allocates. -use alloc::sync::Arc; +use alloc::sync::{Arc, Weak}; use core::cell::{Cell, UnsafeCell}; use core::marker::PhantomPinned; use core::pin::Pin; @@ -60,23 +69,27 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::Stack; use kernel::paged::{ - holder_span, Core, Draw, Leave, Lookup, Object, Pages, Refused, Side, Supplied, Version, Waiters, Want, Writer, - HOLDER_PAGES, PAGE, WRITES_A_READ_MEETS, + holder_span, Core, Cursor, Draw, Leave, Lookup, Mappers, Next, Object, Pages, Refused, Revoke, Revoked, Side, Supplied, Version, + Waiters, Want, Writer, HOLDER_PAGES, PAGE, WRITES_A_READ_MEETS, }; use kernel::sched::task::WaitClass; use kernel::slots::Slots; use toyos_abi::syscall::{ - PagedNotice, PagedRange, PagedStat, PagedStatus, SyscallError, NOTICE_DIRTIED, NOTICE_NO_HOLDERS, NOTICE_REQUESTED, + self as abi, PagedNotice, PagedStat, PagedStatus, SyscallError, NOTICE_DIRTIED, NOTICE_NO_HOLDERS, NOTICE_REQUESTED, PAGED_RECORD, }; -use crate::mm::charge::Charge; +use crate::mm::charge::{self, Charge}; use crate::mm::pmm; +use crate::mm::space::Space; use crate::process::Pid; -use crate::sync::Lock; +use crate::sync::{Lock, LockGuard}; use crate::time::{Deadline, Duration}; -use crate::user_ptr::{UserBytes, UserBytesMut}; +use crate::user_ptr::{Pinned, Stopped, UserBytes, UserBytesMut}; use crate::watch::{self, Watch}; + +mod ranges; +use ranges::Ranges; use crate::DirectMap; use super::{KObjectVariant, ObjectCore, ZeroHandles}; @@ -117,10 +130,10 @@ struct Request { /// One object of a pager's, at its id's slot. struct Entry { object: Object, - /// The pager's own reference, while unreleased, each holder object's, and - /// each owner read's, while it runs: the slot is free again with the last. + /// References that keep the slot: it is free again with the last. refs: u32, - /// Holder objects whose handles have not all gone. + /// Holder objects whose handles have not all gone, and spaces mapping it: + /// a mapping holds the object as a handle does. holders: u32, /// Not released: owner calls reach it. owned: bool, @@ -129,6 +142,14 @@ struct Entry { writers: u32, /// The pager's record, which holders read beside the size and time. record: [u8; PAGED_RECORD], + /// The spaces mapping it, one node each ([`MapNode`]). + mappers: Ends, +} + +/// The ends of an object's list of mappers. +struct Ends { + head: *const MapNode, + tail: *const MapNode, } struct State { @@ -332,55 +353,164 @@ impl PagerObject { let State { pages, entries, .. } = &mut *state; let made = entries.insert_with(|slot| { let object = if anonymous { pages.anonymous(slot, size) } else { pages.object(slot, size) }; - Entry { object, refs: 1, holders: 0, owned: true, writers: 0, record: [0; PAGED_RECORD] } + let mappers = Ends { head: ptr::null(), tail: ptr::null() }; + Entry { object, refs: 1, holders: 0, owned: true, writers: 0, record: [0; PAGED_RECORD], mappers } }); u64::from(made.expect("paged: a slot was made free")) } - /// `f` over the index, the live object `id` and its waiters, each frame - /// it removes given back as it goes; waiters read again. - fn removing( - &self, - id: u64, - f: impl FnOnce(&mut Pages, &mut Object, &mut Linked, &mut dyn FnMut(u64)) -> R, - ) -> Result { + /// `f` over the index, the live object `id` and its waiters, marking + /// pages revoking, then their walk ([`Self::revoked`]); waiters read again. + fn removing(&self, id: u64, f: impl FnOnce(&mut Pages, &mut Object, &mut Linked) -> R) -> Result { let mut state = self.state.lock(); let slot = state.live(id)?; - let job = self.core.job(); - let State { pages, entries, held, gone, requests, .. } = &mut *state; - let object = &mut entries.get_mut(slot).expect("live").object; - let mut waiters = Linked { head: *requests, slot }; - let answer = f(pages, object, &mut waiters, &mut |phys| held.give_back(job, *gone, phys, true)); + let mut waiters = Linked { head: state.requests, slot }; + let (pages, object) = state.split(slot); + let answer = f(pages, object, &mut waiters); + let (mut state, queued) = self.revoked(state, slot); + let Some(epoch) = queued else { + drop(state); + self.changed.post(); + return Ok(answer); + }; + // A walk runs: its next pass takes these pages, and the call returns + // once it has, with the slot held meanwhile. + state.entry_mut(slot).refs += 1; drop(state); self.changed.post(); + let _held = Ref { pager: self, slot }; + let parkable = crate::scheduler::Parkable::at_entry(); + let over = || self.state.lock().entry(slot).object.finished(epoch); + // A kill ends either wait, and its thread never reads the answer. A + // walk takes no wait of its own, so one past the report time is a + // kernel fault worth a line, and the wait goes on. + let report = Deadline::at(crate::clock::now() + self.report); + if watch::wait_until(&parkable, &self.changed, 0, WaitClass::Io, report, over).is_ok() && !over() { + log!("paged: pid {}'s removal waits past {} ms on another's walk", self.pid, self.report.nanos() / 1_000_000); + let _ = watch::wait_until(&parkable, &self.changed, 0, WaitClass::Io, Deadline::never(), over); + } Ok(answer) } + /// The pages of the object at `slot` just marked revoking leave once no + /// mapping reaches them and no futex wait is keyed on their frames: this + /// visits the object's mappers and sweeps the waits on the walk's set as + /// [`Pages::walk`] directs, the lock dropped for each visit and sweep and + /// the slot held meanwhile, until the walk's end gives each frame back + /// and names again every waiter's want a page that ends absent meets. + /// Answers the lock and, where another's walk runs and takes the pages in + /// its next pass, that pass's epoch. + fn revoked<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> (LockGuard<'a, State>, Option) { + match state.entry_mut(slot).object.revoke() { + Revoke::Done => return (state, None), + Revoke::Queued(epoch) => return (state, Some(epoch)), + Revoke::Walk => {} + } + let job = self.core.job(); + state.entry_mut(slot).refs += 1; + let mut absent = false; + loop { + let mut waiters = Linked { head: state.requests, slot }; + let State { pages, entries, held, gone, .. } = &mut *state; + let Entry { object, mappers, .. } = entries.get_mut(slot).expect("paged: a walked slot is held"); + let walked = pages.walk(object, &mut Ring(mappers), &mut waiters, &mut |phys, _, becomes| { + held.give_back(job, *gone, phys, true); + absent |= becomes == Revoked::Absent; + }); + match walked { + Next::Visit(node) => { + // SAFETY: linked, so live, under the lock; a range holds + // it, so it upgrades, and the count kept holds it past + // its unlinking while the lock is dropped. + let node = unsafe { (*node).this.upgrade() }.expect("paged: a linked mapper is held by a range"); + state = self.visit(state, slot, &node); + } + Next::Sweep => state = self.end_futex_waits(state, slot), + Next::Finished => break, + } + } + self.unref_locked(&mut state, slot); + if absent { + self.watch.post(); + } + (state, None) + } + + /// A walk's visit of `node`'s space: the object's revoking pages read a + /// batch at a time under the pager's lock alone, each batch's leaves then + /// cleared and shot down under the space's lock alone + /// ([`Space::clear_paged`]). One lock at a time, and O(revoking pages). + fn visit<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32, node: &MapNode) -> LockGuard<'a, State> { + let mut at = Cursor::Start; + loop { + let mut batch = [(0u64, 0u64); REVOKE_BATCH]; + let n = state.pages.revoking(&state.entry(slot).object, &mut at, &mut batch); + drop(state); + if n > 0 { + let Some(space) = node.space.upgrade() else { return self.state.lock() }; + space.lock().clear_paged(node, &batch[..n]); + } + state = self.state.lock(); + if n < REVOKE_BATCH { + return state; + } + } + } + + /// Every futex wait on a frame of the walk's set at `slot` ends + /// ([`Next::Sweep`], asked only of an object a mapper ever linked): the + /// frames read a batch at a time under the pager's lock, and the waits + /// ended outside it in every bucket, since a wait in any space that mapped + /// the object can be keyed on them (`sched::futex::revoke_sorted`). + /// An unmap, `discard`, `protect` to none or a space's end clears a leaf + /// without ending the waits on a frame it does not own, so a waiter keyed + /// on a file's frame through such a leaf is ended here, before the frame + /// goes back. + fn end_futex_waits<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> LockGuard<'a, State> { + let mut at = Cursor::Start; + loop { + let mut batch = [(0u64, 0u64); REVOKE_BATCH]; + let n = state.pages.revoking(&state.entry(slot).object, &mut at, &mut batch); + drop(state); + let mut frames = [0u64; REVOKE_BATCH]; + for (frame, &(_, phys)) in frames.iter_mut().zip(&batch[..n]) { + *frame = phys; + } + frames[..n].sort_unstable(); + crate::sched::futex::revoke_sorted(&frames[..n], |&phys| phys); + state = self.state.lock(); + if n < REVOKE_BATCH { + return state; + } + } + } + /// The pager lets object `id` go, in one hold of the lock: it answers /// `Gone` from now on, its pages go, and its slot with its last holder. A /// second release, or one racing the pager's end, finds it released. pub fn release(&self, id: u64) -> u64 { - let mut state = self.state.lock(); + let state = self.state.lock(); let slot = match state.live(id) { Ok(slot) => slot, Err(e) => return e.to_u64(), }; - self.let_go(&mut state, slot); - drop(state); + drop(self.let_go(state, slot)); self.changed.post(); 0 } /// Under the lock, `state`: the unreleased object at `slot` released. - fn let_go(&self, state: &mut State, slot: u32) { - let job = self.core.job(); - let State { pages, entries, held, gone, .. } = state; + fn let_go<'a>(&'a self, mut state: LockGuard<'a, State>, slot: u32) -> LockGuard<'a, State> { + let State { pages, entries, .. } = &mut *state; let entry = entries.get_mut(slot).expect("paged: a slot an object names is held"); - pages.invalidate(&mut entry.object, &mut |phys| held.give_back(job, *gone, phys, true)); + pages.invalidate(&mut entry.object); assert!(core::mem::take(&mut entry.owned), "paged: an object released twice"); entries.unmark(NO_HOLDERS, slot); entries.unmark(DIRTIED, slot); - self.unref_locked(state, slot); + // A walk already running takes the pages; nothing here waits. + let (mut state, _) = self.revoked(state, slot); + self.unref_locked(&mut state, slot); + state } /// A holder of object `id`, which writes it where `writes`, and counts @@ -424,9 +554,7 @@ impl PagerObject { return; } let entry = state.entries.remove(slot).expect("paged: a slot an object names is held"); - let job = self.core.job(); - let State { pages, held, gone, .. } = state; - pages.release(entry.object, &mut |phys| held.give_back(job, *gone, phys, true)); + assert!(entry.object.empty(), "paged: an object's slot freed while it holds pages"); } /// Fill `out` with notices, each answered once. @@ -473,9 +601,9 @@ impl PagerObject { }; // SAFETY: a `PagedNotice` is `UserSafe`: every byte of it is a // field, and the batch outlives the copy. - if unsafe { out.write_raw(filled * size, batch.as_ptr().cast::(), n * size) }.is_err() { + if let Err(faulted) = unsafe { out.write_raw(filled * size, batch.as_ptr().cast::(), n * size) } { // The notices taken are lost with the window that could not hold them. - return if filled > 0 { filled as u64 } else { SyscallError::BadAddress.to_u64() }; + return if filled > 0 { filled as u64 } else { SyscallError::from(faulted).to_u64() }; } filled += n; if n < want { @@ -539,20 +667,23 @@ impl PagerObject { }; match step { Step::Copy { count, last } => { - let mut faulted = false; + let (first, chunk) = (done, pieces[..count].iter().map(|piece| piece.len).sum::()); + let mut stopped = None; + let pinned = Pinned::begin(); for piece in &pieces[..count] { let copied = match piece.phys { // SAFETY: `phys` is pinned by `pin_chunk`, and // `within + len` lies inside its frame. - Some(phys) => unsafe { out.write_raw(done, DirectMap::from_phys(phys).as_ptr::().add(piece.within), piece.len) }, - None => out.fill_zero(done, piece.len), + Some(phys) => unsafe { pinned.write(out, done, DirectMap::from_phys(phys).as_ptr::().add(piece.within), piece.len) }, + None => pinned.zero(out, done, piece.len), }; - if copied.is_err() { - faulted = true; + if let Err(why) = copied { + stopped = Some(why); break; } done += piece.len; } + drop(pinned); for piece in &pieces[..count] { if let Some(phys) = piece.phys { pmm::unpin_range(phys, PAGE as usize); } } @@ -567,11 +698,19 @@ impl PagerObject { } if !written { // After the checks: bytes copied before the fault - // count only as every copied chunk's do. - if faulted { - return if done > 0 { done as u64 } else { SyscallError::BadAddress.to_u64() }; + // count only as every copied chunk's do. A page of + // the buffer that is absent is waited for with + // nothing pinned, and the rest of the chunk read + // again. + let faulted = match stopped { + None => None, + Some(Stopped::Absent) => out.wait_in(done, first + chunk - done).err(), + Some(Stopped::Faulted(faulted)) => Some(faulted), + }; + if let Some(faulted) = faulted { + return if done > 0 { done as u64 } else { SyscallError::from(faulted).to_u64() }; } - if last { + if last && stopped.is_none() { return done as u64; } continue; @@ -821,6 +960,7 @@ impl PagerObject { let mut ends = [0u64; CHUNK]; // The frames filled from `src`, in order: a fault stops the fill. let mut filled = 0; + let mut faulted = None; for k in 0..reserved { let page_at = (first + k as u64) * PAGE; let from = done + k * PAGE as usize; @@ -833,8 +973,12 @@ impl PagerObject { frames[k] = phys; ends[k] = page_at + (src.len() - from).min(PAGE as usize) as u64; // SAFETY: a frame just drawn, which nothing else reaches yet. - if filled == k && unsafe { src.read_raw(from, DirectMap::from_phys(phys).as_mut_ptr::(), n) }.is_ok() { - filled += 1; + if filled == k { + // SAFETY: as above. + match unsafe { src.read_raw(from, DirectMap::from_phys(phys).as_mut_ptr::(), n) } { + Ok(()) => filled += 1, + Err(why) => faulted = Some(why), + } } } let mut refused = None; @@ -875,8 +1019,8 @@ impl PagerObject { } } self.changed.post(); - if filled < reserved && !stopped { - refused.get_or_insert(SyscallError::BadAddress); + if let Some(faulted) = faulted.filter(|_| !stopped) { + refused.get_or_insert(faulted.into()); } if let Some(why) = refused { return refusal(installed, why); @@ -905,7 +1049,7 @@ impl PagerObject { let mut pinned = [(0u64, 0u64, 0usize, 0usize); CHUNK]; let mut count = 0; let mut stopped = None; - let mut faulted = false; + let mut stopped_at = None; let slot = { let mut state = self.state.lock(); let slot = match state.live(id) { @@ -936,15 +1080,18 @@ impl PagerObject { let mut from = done; // The pages whose bytes all came: a fault in the owner's buffer stops there. let mut copied = 0; + let copies = Pinned::begin(); for &(_, phys, within, n) in &pinned[..count] { // SAFETY: a frame of the object, pinned and write-pinned above. - if unsafe { src.read_raw(from, DirectMap::from_phys(phys).as_mut_ptr::().add(within), n) }.is_err() { - faulted = true; + if let Err(why) = unsafe { copies.read(src, from, DirectMap::from_phys(phys).as_mut_ptr::().add(within), n) } { + stopped_at = Some(why); break; } from += n; copied += 1; } + drop(copies); + let chunk: usize = pinned[..count].iter().map(|&(_, _, _, n)| n).sum(); { let mut state = self.state.lock(); for (k, &(page, phys, within, n)) in pinned[..count].iter().enumerate() { @@ -963,9 +1110,19 @@ impl PagerObject { for &(_, phys, _, _) in &pinned[..count] { pmm::unpin_range(phys, PAGE as usize); } + // A page of the owner's buffer that is absent is waited for with + // nothing pinned, and the rest of the chunk written again. + let faulted = match stopped_at { + None => None, + Some(Stopped::Absent) => src.wait_in(from, done + chunk - from).err(), + Some(Stopped::Faulted(faulted)) => Some(faulted), + }; done = from; - if faulted { - return refusal(done as u64, SyscallError::BadAddress); + if let Some(faulted) = faulted { + return refusal(done as u64, faulted.into()); + } + if stopped_at.is_some() { + continue; } match stopped { None => {} @@ -976,10 +1133,8 @@ impl PagerObject { done as u64 } - /// Its pages end absent, so every reader waiting on one is named in a - /// notice again (`Want::withdrawn`). pub fn withdraw(&self, id: u64, first: u64, count: u64) -> u64 { - match self.removing(id, |pages, object, waiters, out| pages.withdraw(object, first, count, waiters, &mut |f| out(f))) { + match self.removing(id, |pages, object, waiters| pages.withdraw(object, first, count, waiters)) { Ok((n, news)) => { if news { self.watch.post(); @@ -991,7 +1146,7 @@ impl PagerObject { } pub fn fail(&self, id: u64, first: u64, count: u64) -> u64 { - match self.removing(id, |pages, object, _, out| pages.fail(object, first, count, &mut |f| out(f))) { + match self.removing(id, |pages, object, _| pages.fail(object, first, count)) { Ok(Ok(())) => 0, Ok(Err(why)) => word(why).to_u64(), Err(e) => e.to_u64(), @@ -1005,8 +1160,8 @@ impl PagerObject { if let Err(e) = self.holding(id) { return e.to_u64(); } - let resized = self.removing(id, |pages, object, _, out| { - let tail = pages.resize(object, size, &mut |f| out(f))?; + let resized = self.removing(id, |pages, object, _| { + let tail = pages.resize(object, size)?; if let Some(phys) = tail { let within = (size % PAGE) as usize; // SAFETY: the last page's frame, the object's while the lock is @@ -1087,11 +1242,11 @@ impl PagerObject { /// The pages of object `id` holders dirtied since the last call: a notice /// not yet taken of them is owed no more. - pub fn take_dirtied(&self, id: u64) -> Result { + pub fn take_dirtied(&self, id: u64) -> Result { self.on_live(id, |state, slot| { state.entries.unmark(DIRTIED, slot); let (first, count) = state.entry_mut(slot).object.take_dirtied().unwrap_or((0, 0)); - PagedRange { first, count } + abi::PagedRange { first, count } }) } @@ -1107,8 +1262,10 @@ impl PagerObject { } /// A holder's write of `src` at `at`, or at the size where `None`: whole - /// or refused, and never waiting. Its bytes are copied into frames held - /// aside first, with nothing else held; then, in one hold of the lock, it + /// or refused, and never waiting for the write side. Its bytes are copied + /// into frames drawn first, with nothing else held, and a page of `src` + /// that is absent is waited for with the frames given back + /// (`user_ptr::Pinned`); then, in one hold of the lock, it /// takes the write side, refused while anyone holds or waits for it, and /// is checked and admitted whole ([`Pages::holder_fits`]). Its copy into /// the object's frames is the kernel's own, so whoever waits for the side @@ -1132,50 +1289,69 @@ impl PagerObject { let Some(_writing) = HolderWriting::begin(self, slot) else { return SyscallError::Gone.to_u64(); }; - // Frames for the bytes, and for the pages the write would install, - // as the object stands now: one fewer than it needs then refuses it. - let fresh_count = { - let mut state = self.state.lock(); - if state.gone { - return SyscallError::Gone.to_u64(); - } - let (pages, object) = state.split(slot); - let offset = at.unwrap_or(object.size()); - let span = match holder_span(offset, len as u64) { - Ok(span) => span, - Err(why) => return holder_word(why).to_u64(), + let fresh_count = loop { + // Frames for the bytes, and for the pages the write would install, + // as the object stands now: one fewer than it needs then refuses it. + let fresh_count = { + let mut state = self.state.lock(); + if state.gone { + return SyscallError::Gone.to_u64(); + } + let (pages, object) = state.split(slot); + let offset = at.unwrap_or(object.size()); + let span = match holder_span(offset, len as u64) { + Ok(span) => span, + Err(why) => return holder_word(why).to_u64(), + }; + let fresh_count = match pages.holder_fits(object, offset / PAGE, span) { + Ok(fresh) => fresh as usize, + Err(why) => return holder_word(why).to_u64(), + }; + let wanted = staged_count + fresh_count; + let got = self.draw(&mut state, &mut drawn[..wanted]); + if got < wanted { + self.give_back_drawn(&mut state, &drawn[..got]); + return SyscallError::WouldBlock.to_u64(); + } + fresh_count }; - let fresh_count = match pages.holder_fits(object, offset / PAGE, span) { - Ok(fresh) => fresh as usize, - Err(why) => return holder_word(why).to_u64(), + for (k, frame) in staged.iter_mut().take(staged_count).enumerate() { + *frame = materialize(drawn[k], false); + } + for (k, frame) in fresh.iter_mut().take(fresh_count).enumerate() { + *frame = materialize(drawn[staged_count + k], true); + } + let pinned = Pinned::begin(); + let mut stopped = None; + for (k, &frame) in staged[..staged_count].iter().enumerate() { + let n = (len - k * PAGE as usize).min(PAGE as usize); + // SAFETY: a frame this write drew, which nothing else reaches. + if let Err(why) = unsafe { pinned.read(src, k * PAGE as usize, DirectMap::from_phys(frame).as_mut_ptr::(), n) } { + stopped = Some((k * PAGE as usize, why)); + break; + } + } + drop(pinned); + let Some((from, why)) = stopped else { break fresh_count }; + { + let mut state = self.state.lock(); + self.give_frames(&mut state, &staged[..staged_count]); + self.give_frames(&mut state, &fresh[..fresh_count]); + } + let waited = match why { + Stopped::Absent => src.wait_in(from, len - from), + Stopped::Faulted(faulted) => Err(faulted), }; - let wanted = staged_count + fresh_count; - let got = self.draw(&mut state, &mut drawn[..wanted]); - if got < wanted { - self.give_back_drawn(&mut state, &drawn[..got]); - return SyscallError::WouldBlock.to_u64(); + if let Err(faulted) = waited { + return SyscallError::from(faulted).to_u64(); } - fresh_count }; - for (k, frame) in staged.iter_mut().take(staged_count).enumerate() { - *frame = materialize(drawn[k], false); - } - for (k, frame) in fresh.iter_mut().take(fresh_count).enumerate() { - *frame = materialize(drawn[staged_count + k], true); - } let give_up = |e: SyscallError| { let mut state = self.state.lock(); self.give_frames(&mut state, &staged[..staged_count]); self.give_frames(&mut state, &fresh[..fresh_count]); e.to_u64() }; - for (k, &frame) in staged[..staged_count].iter().enumerate() { - let n = (len - k * PAGE as usize).min(PAGE as usize); - // SAFETY: a frame this write drew, which nothing else reaches. - if unsafe { src.read_raw(k * PAGE as usize, DirectMap::from_phys(frame).as_mut_ptr::(), n) }.is_err() { - return give_up(SyscallError::BadAddress); - } - } let mut pinned = [(0u64, 0u64, 0usize, 0usize); HOLDER]; let (offset, count, used) = { let mut state = self.state.lock(); @@ -1263,7 +1439,7 @@ impl PagerObject { /// Under the lock: draw a frame for each of `out`'s entries from the /// core, as a supply does — one held aside, the core's promise, or a /// promise the allocator makes now: how many were drawn, each `Some` for - /// a frame held aside, which [`materialize`] completes outside the lock. + /// a frame held aside, which [`materialize`] completes. fn draw(&self, state: &mut State, out: &mut [Option]) -> usize { state.held.settle(); let mut drawn = 0; @@ -1566,7 +1742,7 @@ impl ZeroHandles for PagerObject { state.gone = true; for slot in 0..state.entries.len() as u32 { if state.entries.get(slot).is_some_and(|e| e.owned) { - self.let_go(&mut state, slot); + state = self.let_go(state, slot); } } let job = self.core.job(); @@ -1591,6 +1767,326 @@ impl Drop for PagerObject { } } +/// One space's mappings of one paged object, on the object's list from its +/// space's first range of the object until its last, which unlinks it only +/// once that range's leaves are cleared and shot down: the space drops a +/// removed range after its shootdown. It holds a reference on the object's +/// slot, so a mapped object outlives its release, and is charged to the +/// mapper's job. +pub struct MapNode { + pager: Arc, + slot: u32, + space: Weak, + /// The ranges holding it ([`PagedRange`]). + ranges: Ranges, + /// Under the pager's lock. + link: UnsafeCell, + /// Itself, for a walk to hold while it visits. + this: Weak, + _charge: Charge, +} + +struct Link { + /// The epoch of the walk that last visited it, or the one current at + /// its linking ([`kernel::paged::Mappers`]). + seen: u64, + prev: *const MapNode, + next: *const MapNode, +} + +// SAFETY: `link` is reached only under its pager's lock, and the rest is +// immutable or atomic. +unsafe impl Send for MapNode {} +// SAFETY: as for `Send`. +unsafe impl Sync for MapNode {} + +/// Why the index holds no frame a fault may map for a page. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Missing { + /// Absent, or revoking, at this version of the object: the fault waits. + Absent(Version), + /// A hole of an anonymous object, which only [`Index::fault`] fills. + Hole, + /// Its pager cannot let a mapping have it. + Lost(Lost), + /// Wholly past the object's end: nothing is there. + PastEnd(Lost), +} + +/// A page a fault was refused, as the record of its end names it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Lost { + pub pager: Pid, + pub object: u32, + pub page: u64, + pub why: Why, +} + +/// Why a pager cannot let a mapping have a page. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Why { + /// The pager failed it. + Failed, + /// The pager released the object. + Released, + /// The pager ended. + Ended, + /// A hole of an anonymous object, and the pager has no room for its page. + NoRoom, + /// The page lies wholly past the object's end: nothing is there, so a + /// touch of it ends the process and a copy from it is refused as a bad + /// address. + PastEnd, +} + +impl Why { + pub fn name(self) -> &'static str { + match self { + Why::Failed => "failed by its pager", + Why::Released => "released by its pager", + Why::Ended => "its pager ended", + Why::NoRoom => "no room in its pager", + Why::PastEnd => "past the object's end", + } + } +} + +/// A mapped object's pages as its index holds them, under the pager's lock. +pub struct Index<'a> { + pager: &'a PagerObject, + state: &'a mut State, + slot: u32, +} + +impl Index<'_> { + /// The frame present for the object's `page`, or why there is none. + pub fn page(&self, page: u64) -> Result { + let object = &self.state.entry(self.slot).object; + match self.state.pages.lookup(object, page) { + _ if self.state.gone => Err(self.lost(page, Why::Ended)), + Lookup::Present(phys) => Ok(phys), + Lookup::Absent => Err(Missing::Absent(object.version())), + Lookup::PastEnd => Err(Missing::PastEnd(self.record(page, Why::PastEnd))), + Lookup::Zero => Err(Missing::Hole), + Lookup::Failed => Err(self.lost(page, Why::Failed)), + Lookup::Gone => Err(self.lost(page, Why::Released)), + } + } + + /// As [`Self::page`], for the page a fault touched: a hole of an + /// anonymous object, a MEMORY volume's, gets a zeroed frame of its + /// pager's, installed clean and charged as a supply's is, and is refused + /// where the pager has no room for it (7b's N4). + pub fn fault(&mut self, page: u64) -> Result { + match self.page(page) { + Err(Missing::Hole) => self.zeroed(page), + found => found, + } + } + + fn zeroed(&mut self, page: u64) -> Result { + let no_room = self.lost(page, Why::NoRoom); + let mut drawn = [None]; + if self.pager.draw(self.state, &mut drawn) == 0 { + return Err(no_room); + } + let phys = materialize(drawn[0], true); + let (pages, object) = self.state.split(self.slot); + match pages.supply(object, page, phys, false, (page + 1) * PAGE) { + Ok(Supplied::Installed) => { + self.state.held.core.installed(); + Ok(phys) + } + Ok(Supplied::Present) => unreachable!("paged: a hole found under the lock held a page"), + // Inside the object and the pager alive, as the lookup found it: + // only a full table refuses. + Err(_) => { + self.pager.give_frames(self.state, &[phys]); + Err(no_room) + } + } + } + + fn lost(&self, page: u64, why: Why) -> Missing { + Missing::Lost(self.record(page, why)) + } + + fn record(&self, page: u64, why: Why) -> Lost { + Lost { pager: self.pager.pid, object: self.slot, page, why } + } +} + +/// The revoking pages a walk reads under its pager's lock at once. +/// `tests/guest/mapprobe`'s `shrunk` case revokes 79 pages so that its +/// revocation takes two batches; at 79 or more it would take one, and that +/// case would no longer reach a second. +const REVOKE_BATCH: usize = 64; + +/// The pages a fault's request names: the aligned run around its page. +const FAULT_RUN: u64 = kernel::vm::AROUND / PAGE; + +impl MapNode { + /// `f` over the object's pages, under the pager's lock. + pub fn with_index(&self, f: impl FnOnce(&mut Index<'_>) -> R) -> R { + let mut state = self.pager.state.lock(); + f(&mut Index { pager: &self.pager, state: &mut state, slot: self.slot }) + } + + /// A fault's wait for the object's `page`, found absent at `since`, its + /// space's lock not held: the page's aligned run is requested in a fresh + /// request, so a fault whose last wait a write or resize ended asks + /// afresh, and the wait lasts until the page is no longer absent, the + /// version moved, or the pager is gone. Reported overdue past the pager's + /// report time, as a holder's read is, and never ended by a timeout; a + /// file's page behaves as memory does (7b's D6). `false` where a kill + /// ended it. + pub fn wait(&self, page: u64, since: Version) -> bool { + let (pager, slot) = (&*self.pager, self.slot); + let over = |state: &State| state.gone || state.pages.wait_over(&state.entry(slot).object, page, since); + let waiting = core::pin::pin!(Waiting::new(pager, slot)); + let (first, count) = { + let mut state = pager.state.lock(); + if over(&state) { + return true; + } + // Absent, so inside the object. + let end = state.entry(slot).object.size().div_ceil(PAGE); + let first = page - page % FAULT_RUN; + let count = (first + FAULT_RUN).min(end) - first; + waiting.as_ref().request(&mut state, first, count); + (first, count) + }; + pager.wait_changed(slot, first, count, || over(&pager.state.lock())).is_ok() + } +} + +/// A range's hold on its space's [`MapNode`]: a clone is another range of +/// the same object in the space, and the last one dropped unlinks the node. +pub struct PagedRange(Arc); + +impl PagedRange { + pub fn node(&self) -> &MapNode { + &self.0 + } +} + +impl Clone for PagedRange { + fn clone(&self) -> Self { + self.0.ranges.more(); + Self(Arc::clone(&self.0)) + } +} + +impl Drop for PagedRange { + fn drop(&mut self) { + if self.0.ranges.less() { + self.0.pager.unlink(&self.0); + } + } +} + +impl PagerObject { + /// The first range of object `slot` in `space`, whose memory is counted + /// to `job`: its node linked, seen at the object's epoch now, under + /// the space's lock the caller holds. `Gone` where the object or its + /// pager is. + pub fn link(self: &Arc, slot: u32, space: Weak, job: &Arc) -> Result { + let charge = Charge::new(job, charge::arc::()).map_err(|_| SyscallError::ResourceExhausted)?; + let link = UnsafeCell::new(Link { seen: 0, prev: ptr::null(), next: ptr::null() }); + let node = Arc::new_cyclic(|this| MapNode { + pager: Arc::clone(self), + slot, + space, + ranges: Ranges::one(), + link, + this: this.clone(), + _charge: charge, + }); + let mut state = self.state.lock(); + let entry = state.entry_mut(slot); + if entry.object.gone() { + return Err(SyscallError::Gone); + } + let at = Arc::as_ptr(&node); + // SAFETY: a fresh node no list holds yet, and the tail's, linked, + // under the lock. + unsafe { + *(*at).link.get() = Link { seen: entry.object.link(), prev: entry.mappers.tail, next: ptr::null() }; + match entry.mappers.tail.is_null() { + true => entry.mappers.head = at, + false => (*(*entry.mappers.tail).link.get()).next = at, + } + } + entry.mappers.tail = at; + // A mapping holds the object as a handle does: a file mapped and + // closed, or unlinked, stays until the last mapping goes. + entry.refs += 1; + entry.holders += 1; + Ok(PagedRange(node)) + } + + /// The last range of `node`'s space went, its leaves cleared and shot + /// down: the node leaves the list, and its hold and reference on the + /// slot go. + fn unlink(&self, node: &MapNode) { + let mut state = self.state.lock(); + let entry = state.entry_mut(node.slot); + // SAFETY: linked, as are its neighbours, under the lock. + unsafe { + let Link { prev, next, .. } = *node.link.get(); + match prev.is_null() { + true => entry.mappers.head = next, + false => (*(*prev).link.get()).next = next, + } + match next.is_null() { + true => entry.mappers.tail = prev, + false => (*(*next).link.get()).prev = prev, + } + } + state.entry_mut(node.slot).holders -= 1; + self.say_no_holders(&mut state, node.slot); + self.unref_locked(&mut state, node.slot); + } +} + +/// An object's list of mappers, as its walk visits them, under the pager's +/// lock. +struct Ring<'a>(&'a mut Ends); + +impl Mappers for Ring<'_> { + type Node = *const MapNode; + + fn head(&self) -> Option<*const MapNode> { + (!self.0.head.is_null()).then_some(self.0.head) + } + + fn seen(&self, node: *const MapNode) -> u64 { + // SAFETY: linked, so live, under the lock. + unsafe { (*(*node).link.get()).seen } + } + + fn visited(&mut self, node: *const MapNode, epoch: u64) { + // SAFETY: linked, as are its neighbours, under the lock; it moves to + // the tail. + unsafe { + let link = &mut *(*node).link.get(); + link.seen = epoch; + if self.0.tail == node { + return; + } + match link.prev.is_null() { + true => self.0.head = link.next, + false => (*(*link.prev).link.get()).next = link.next, + } + (*(*link.next).link.get()).prev = link.prev; + (*(*self.0.tail).link.get()).next = node; + link.prev = self.0.tail; + link.next = ptr::null(); + self.0.tail = node; + } + } +} + /// A holder's write under way on its pager's object at `slot`, counted /// until it ends by any path. struct HolderWriting<'a> { @@ -1672,6 +2168,17 @@ impl PagedHolderObject { self.named.pager.read(self.named.slot, out, offset, true) } + /// Whether `range` maps this holder's object. + pub fn maps(&self, range: &PagedRange) -> bool { + ptr::eq(Arc::as_ptr(&range.0.pager), Arc::as_ptr(&self.named.pager)) && range.0.slot == self.named.slot + } + + /// The first range of its object in `space`, whose memory is counted to + /// `job` ([`PagerObject::link`]). + pub fn link(&self, space: Weak, job: &Arc) -> Result { + self.named.pager.link(self.named.slot, space, job) + } + /// A write of `src` at `at`, or at the size where `None`. pub fn write(&self, src: &UserBytes, at: Option) -> u64 { self.named.pager.holder_write(self.named.slot, src, at) diff --git a/kernel/src/object/paged/ranges.rs b/kernel/src/object/paged/ranges.rs new file mode 100644 index 0000000000..4243c0f29b --- /dev/null +++ b/kernel/src/object/paged/ranges.rs @@ -0,0 +1,50 @@ +//! How many ranges of one space hold its node of a mapped object +//! (`object::paged::MapNode`). A range clears and shoots down its leaves +//! before it lets go; the last to let go, which may be a parked fault's hold +//! with no space lock, sees every other's clearing before the node leaves its +//! object's list, so a walk that no longer visits the space frees no frame a +//! leaf there may still reach. `kernel/loom/tests/paged_ranges.rs` checks it. + +#[cfg(not(feature = "loom"))] +use core::sync::atomic::{fence, AtomicU32, Ordering}; + +#[cfg(feature = "loom")] +use loom::sync::atomic::{fence, AtomicU32, Ordering}; + +/// The order a letting go publishes its clearing with, and the last reads +/// every one with: `Arc`'s. Its negative control relaxes both. +#[cfg(not(feature = "ranges-less-relaxed"))] +const LESS: Ordering = Ordering::Release; +#[cfg(not(feature = "ranges-less-relaxed"))] +const LAST: Ordering = Ordering::Acquire; + +#[cfg(feature = "ranges-less-relaxed")] +const LESS: Ordering = Ordering::Relaxed; +#[cfg(feature = "ranges-less-relaxed")] +const LAST: Ordering = Ordering::Relaxed; + +pub struct Ranges(AtomicU32); + +impl Ranges { + /// A node's first range. + pub fn one() -> Self { + Self(AtomicU32::new(1)) + } + + /// Another range of a node a range holds. + pub fn more(&self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + + /// A range lets go: whether it was the last. + #[must_use] + pub fn less(&self) -> bool { + if self.0.fetch_sub(1, LESS) != 1 { + return false; + } + if LAST != Ordering::Relaxed { + fence(LAST); + } + true + } +} diff --git a/kernel/src/pipe.rs b/kernel/src/pipe.rs index 9581228fa6..a24d0b49c8 100644 --- a/kernel/src/pipe.rs +++ b/kernel/src/pipe.rs @@ -17,7 +17,7 @@ use alloc::sync::Arc; use crate::scheduler::Parkable; use crate::sync::Lock; -use crate::user_ptr::{UserBytes, UserBytesMut}; +use crate::user_ptr::{Faulted, UserBytes, UserBytesMut}; use crate::watch::{Cancelled, Watch}; /// A reference to a pipe that holds neither end: what an end's object, a @@ -173,16 +173,16 @@ pub enum PipeWrite { /// The memory its bytes need could not be taken; no wait for room makes /// it appear, so a caller must not park on it. NoMemory, - /// A byte of the writer's buffer it cannot reach: nothing was written. - Faulted, + /// A byte of the writer's buffer it cannot have: nothing was written. + Faulted(Faulted), } /// Why a read answered no count. pub enum ReadStopped { /// A kill ended the wait for another read's claim. Cancelled, - /// A byte of the reader's buffer it cannot reach: the bytes it took are lost. - Faulted, + /// A byte of the reader's buffer it cannot have: the bytes it took are lost. + Faulted(Faulted), } impl From for ReadStopped { @@ -219,7 +219,7 @@ impl PipeRef { if boost { crate::scheduler::boost_current_rt_inherited(); } - buf.write_at(0, bounce.bytes()).map_err(|_| ReadStopped::Faulted)?; + buf.write_at(0, bounce.bytes()).map_err(ReadStopped::Faulted)?; return Ok(Some(n)); } pipe.held.claim_read(n); @@ -238,7 +238,7 @@ impl PipeRef { frame_bytes::give_back(&self.0.state, |s| &mut s.held, spares); drop(unread); frame_bytes::wake(&self.0.read_watch, ended); - copied.map_err(|_| ReadStopped::Faulted)?; + copied.map_err(ReadStopped::Faulted)?; Ok(Some(n)) } @@ -249,7 +249,7 @@ impl PipeRef { let mut bounce = PipeBounce::new(); return match bounce.read(buf) { Ok(src) => self.write_bounced(p, src), - Err(_) => Ok(PipeWrite::Faulted), + Err(faulted) => Ok(PipeWrite::Faulted(faulted)), }; } let (n, first) = { @@ -273,7 +273,7 @@ impl PipeRef { let wrote = match supplied { Ok(runs) => match frame_bytes::fill(&self.0.state, |s| &mut s.held, buf, n, runs) { Ok(()) => PipeWrite::Wrote(n), - Err(_) => PipeWrite::Faulted, + Err(faulted) => PipeWrite::Faulted(faulted), }, Err(Exhausted) => PipeWrite::NoMemory, }; diff --git a/kernel/src/process.rs b/kernel/src/process.rs index 3c72a6ab2b..6d9889a1ba 100644 --- a/kernel/src/process.rs +++ b/kernel/src/process.rs @@ -663,6 +663,8 @@ fn teardown_resources( let handles = Arc::clone(&process_data_arc.lock().handles); ops::close_all(&mut handles.lock()); + let space = Arc::clone(&process_data_arc.lock().space); + space.lock().end(); // Every thread has left, and none returns to Ring 3 to use them. crate::isa::process_ends(pid); let space = Arc::clone(&process_data_arc.lock().space); @@ -923,9 +925,14 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { }; let wait = crate::user_ptr::begin_futex_wait(); - // Physical, so a futex in shared memory works across processes. - let Some(word) = futex_word(addr).and_then(|addr| wait.translate(addr)) else { - return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); + // Physical, so a futex in shared memory works across processes. A word + // of a mapped file's page that is absent or lost is a spurious wake, as + // swapped memory's is. + let unaligned = crate::user_ptr::Untranslated::Unreachable; + let word = match futex_word(addr).ok_or(unaligned).and_then(|addr| wait.translate(addr)) { + Ok(word) => word, + Err(crate::user_ptr::Untranslated::Away) => return 0, + Err(crate::user_ptr::Untranslated::Unreachable) => return toyos_abi::syscall::SyscallError::BadAddress.to_u64(), }; // 0 covers both woken and never-blocked (the caller re-checks the word either way); 1 means only the caller's own deadline ended it. @@ -937,8 +944,12 @@ pub fn futex_wait(addr: UserAddr, expected: u32, timeout_ns: u64) -> u64 { /// Wake up to `count` threads blocked on the same physical address as `addr`. pub fn futex_wake(addr: UserAddr, count: u64) -> u64 { - let Some(phys) = futex_word(addr).and_then(|addr| crate::user_ptr::translate_user(addr, crate::user_ptr::Access::Read)) else { - return toyos_abi::syscall::SyscallError::BadAddress.to_u64(); + // No wait is keyed on a page that is not there. + let unaligned = crate::user_ptr::Untranslated::Unreachable; + let phys = match futex_word(addr).ok_or(unaligned).and_then(|addr| crate::user_ptr::translate_user(addr, crate::user_ptr::Access::Read)) { + Ok(phys) => phys, + Err(crate::user_ptr::Untranslated::Away) => return 0, + Err(crate::user_ptr::Untranslated::Unreachable) => return toyos_abi::syscall::SyscallError::BadAddress.to_u64(), }; scheduler::futex_wake(phys, count as usize) } @@ -958,17 +969,116 @@ pub fn ask_join(join: &join::Join, tid: Tid, parent_pid: Pid) -> Option bool { - let Some(space) = scheduler::current_address_space() else { return false }; +/// What serving a fault came to. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Faulted { + /// The page is populated, or already held a leaf allowing the access. + Served, + /// No range allows the access there. + Unreachable, + /// A mapped file's page its pager cannot let the process have. + Lost(crate::object::paged::Lost), + /// A mapped file's page wholly past its end. + PastEnd(crate::object::paged::Lost), + /// A mapped file's page that is absent, met by a copy holding page pins + /// (`user_ptr::Pinned`), which does not wait for it. + Absent, + /// A kill ended a wait for a paged page: the thread never runs its user + /// code again, and a copy stops where it is. + Killed, +} + +impl Faulted { + /// What a copy this fault stopped short answers. + pub fn stop(self) -> crate::user_ptr::Stop { + match self { + Faulted::Lost(_) => crate::user_ptr::Stop::Lost, + Faulted::Absent => crate::user_ptr::Stop::Absent, + // Nothing is there, so a copy answers `BadAddress`. + Faulted::Served | Faulted::Unreachable | Faulted::PastEnd(_) | Faulted::Killed => crate::user_ptr::Stop::Unreachable, + } + } +} + +/// Serve a fault at `fault_addr` for `access` in the running thread's space, +/// at its trap's depth with interrupts on: its page populated under the +/// space's lock, or, for a paged page that is absent, waited for holding no +/// lock and touched again, unless the fault is a [`crate::user_ptr::Pinned`] copy's. +/// While it waits the CPU's fault mark is down, so another fault on this CPU +/// meanwhile is no recursion; it is up again on whichever CPU the thread +/// wakes on. +pub fn handle_page_fault(fault_addr: u64, access: kernel::vm::Access) -> Faulted { + use crate::arch::percpu::{self, CpuFaultState}; + touch(fault_addr, access, |page| { + if crate::user_ptr::pinned() { + return Faulted::Absent; + } + percpu::set_fault_state(CpuFaultState::Normal); + let woken = page.wait(); + let prev = percpu::swap_fault_state(CpuFaultState::PageFault); + assert_eq!(prev, CpuFaultState::Normal, "a fault woke on a CPU marked {prev:?}"); + if woken { Faulted::Served } else { Faulted::Killed } + }) +} + +/// The page at `va` populated for `access`, a paged page that is absent left +/// to `wait`, which answers `Served` to touch it again. +fn touch(va: u64, access: kernel::vm::Access, mut wait: impl FnMut(crate::mm::space::PageWait) -> Faulted) -> Faulted { + use crate::mm::space::Populated; + let Some(space) = scheduler::current_address_space() else { return Faulted::Unreachable }; let began = crate::clock::nanos_since_boot(); - let served = space.lock().fault(fault_addr, access); - if served { - space.usage().faulted(crate::clock::nanos_since_boot().saturating_sub(began)); + loop { + let populated = space.lock().fault(va, access); + match populated { + Populated::At(_) => break, + Populated::Unreachable => return Faulted::Unreachable, + Populated::Lost(lost) => return Faulted::Lost(lost), + Populated::PastEnd(record) => return Faulted::PastEnd(record), + Populated::Wait(page) => match wait(page) { + Faulted::Served => {} + stopped => return stopped, + }, + } } - served + space.usage().faulted(crate::clock::nanos_since_boot().saturating_sub(began)); + Faulted::Served +} + +/// Populate the running thread's pages over `[start, end)` for `access` from +/// a syscall holding no lock and no pin, waiting for a mapped file's absent +/// page as a fault does. +pub fn wait_in(start: u64, end: u64, access: kernel::vm::Access) -> Result<(), crate::user_ptr::Faulted> { + let mut page = start & !(crate::mm::PAGE_SIZE - 1); + while page < end { + let touched = touch(page, access, |page| if page.wait() { Faulted::Served } else { Faulted::Killed }); + match touched { + Faulted::Served => {} + stopped => { + return Err(match stopped.stop() { + crate::user_ptr::Stop::Lost => crate::user_ptr::Faulted::Lost, + _ => crate::user_ptr::Faulted::Unreachable, + }) + } + } + page += crate::mm::PAGE_SIZE; + } + Ok(()) +} + +/// A Ring 3 touch at `fault_addr` of a mapped file's page its pager cannot +/// let the process have, or of one wholly past the file's end, ends the +/// process as `SIGBUS`, with one record of what was lost. The caller has put +/// the CPU's fault mark down. +pub fn lost(fault_addr: u64, lost: crate::object::paged::Lost) -> ! { + log!( + "fault: pid {} touched {fault_addr:#x}, page {} of pid {}'s object {}: {}", + current_process(), + lost.page, + lost.pager, + lost.object, + lost.why.name() + ); + exit(End::Fault(Fault::Mapping)) } /// Dump memory around `fault_addr` and the instruction pointer for the diff --git a/kernel/src/sched/driver.rs b/kernel/src/sched/driver.rs index 36ff4acf0e..fe99fafd91 100644 --- a/kernel/src/sched/driver.rs +++ b/kernel/src/sched/driver.rs @@ -755,6 +755,11 @@ pub fn pass_on_every_other_cpu() { } } +/// `f` over the running thread's user copies; `None` off-task. +pub fn with_current_copies(f: impl FnOnce(&crate::user_ptr::Copies) -> R) -> Option { + try_with_cpu(|cpu| cpu.running().map(|t| f(t.ext().handle.copies()))).flatten() +} + /// Count the running thread's syscall `num`, which took `ticks` of the counter. pub fn note_syscall(num: u64, ticks: u64) { try_with_cpu(|cpu| cpu.running().map(|t| t.ext().handle.syscalls().note(num, ticks))); diff --git a/kernel/src/sched/payload.rs b/kernel/src/sched/payload.rs index de5509e9d2..71d10e0c3c 100644 --- a/kernel/src/sched/payload.rs +++ b/kernel/src/sched/payload.rs @@ -117,6 +117,7 @@ pub struct TaskHandle { /// The releases this thread owns while its process is torn down (`object::Releases`); kept here, as `operation` is, so they follow the thread across a migration. releases: Lock>>, syscalls: SyscallProfile, + copies: crate::user_ptr::Copies, } /// What a thread's syscalls cost it. Written only by the thread, at the end of @@ -170,6 +171,7 @@ impl TaskHandle { cancels: AtomicU32::new(0), releases: Lock::new(None), syscalls: SyscallProfile::new(), + copies: crate::user_ptr::Copies::new(), } } @@ -177,6 +179,11 @@ impl TaskHandle { &self.syscalls } + /// Its user copies, as their faults leave them. + pub fn copies(&self) -> &crate::user_ptr::Copies { + &self.copies + } + pub(crate) fn publish(&self, acct: &TaskAccounting, running_since: Option) { self.cpu_ns.store(acct.cpu_ns, Ordering::Relaxed); self.running_since diff --git a/kernel/src/syscall/device.rs b/kernel/src/syscall/device.rs index a40ade251f..e8770ea069 100644 --- a/kernel/src/syscall/device.rs +++ b/kernel/src/syscall/device.rs @@ -263,8 +263,8 @@ pub(super) fn sys_device_firmware_window(ctx: &SyscallContext, handle: RawHandle // Written first, before anything is opened, so a window the caller cannot // write as the call begins leaves no grant it was never told of. One a // sibling unmaps meanwhile can; the claim's release takes it back. - if out.fill_zero(0, len as usize).is_err() { - return SyscallError::BadAddress.to_u64(); + if let Err(faulted) = out.fill_zero(0, len as usize) { + return SyscallError::from(faulted).to_u64(); } let mut grant = toyos_abi::pci::WindowGrant { shm: toyos_abi::HANDLE_INVALID, refused: 0, offset: 0, len: 0 }; match on_function(&claim, |function| crate::pcidev::firmware_window(function, window)) { @@ -304,8 +304,8 @@ pub(super) fn sys_device_dma_alloc( let Some(mut window) = ctx.user_bytes_mut(out, len) else { return SyscallError::BadAddress.to_u64(); }; - if window.fill_zero(0, len as usize).is_err() { - return SyscallError::BadAddress.to_u64(); + if let Err(faulted) = window.fill_zero(0, len as usize) { + return SyscallError::from(faulted).to_u64(); } let granted = on_function(&claim, |function| crate::pcidev::dma_alloc(function, bytes)); let (memory, device_addr, granted) = match granted { @@ -363,8 +363,8 @@ pub(super) fn sys_device_dma_map( return SyscallError::BadAddress.to_u64(); }; // Written first, before anything is mapped, for `sys_device_dma_alloc`'s reason. - if window.fill_zero(0, len as usize).is_err() { - return SyscallError::BadAddress.to_u64(); + if let Err(faulted) = window.fill_zero(0, len as usize) { + return SyscallError::from(faulted).to_u64(); } let mapped = on_function(&claim, |function| crate::pcidev::dma_map(function, &memory)); let (device_addr, bytes) = match mapped { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index d087676f94..dd49047557 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -111,7 +111,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_MACHINE_STOPS => sys_machine_stops(RawHandle(a1 as u32), a2), SYS_PIPE => sys_pipe(), SYS_SPAWN => { - let Ok(args) = ctx.copy_in::(UserAddr::new(a1)) else { return bad_addr }; + let args = match ctx.copy_in::(UserAddr::new(a1)) { Ok(args) => args, Err(e) => return e.to_u64() }; // First: each range's object is the caller's, and an endowment // below may move the caller's own handle to it. let program = match spawn_program(&ctx, &args) { @@ -307,13 +307,13 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> } SYS_ACCEPT => sys_accept(RawHandle(a1 as u32), a2), SYS_MESSAGE_SEND => { - let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; + let args = match ctx.copy_in::(UserAddr::new(a2)) { Ok(args) => args, Err(e) => return e.to_u64() }; let Ok(out) = outgoing(&ctx, &args) else { return SyscallError::InvalidArgument.to_u64() }; let Some(out) = out else { return bad_addr }; sys_message_send(RawHandle(a1 as u32), &out, args.flags & MESSAGE_NONBLOCK != 0) } SYS_MESSAGE_RECV => { - let Ok(args) = ctx.copy_in::(UserAddr::new(a2)) else { return bad_addr }; + let args = match ctx.copy_in::(UserAddr::new(a2)) { Ok(args) => args, Err(e) => return e.to_u64() }; let Ok(into) = incoming(&ctx, &args) else { return SyscallError::InvalidArgument.to_u64() }; let Some(mut into) = into else { return bad_addr }; sys_message_recv(RawHandle(a1 as u32), &mut into, args.flags & MESSAGE_NONBLOCK != 0) @@ -337,8 +337,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> } let Some(bytes) = ctx.user_bytes(UserAddr::new(a2), len) else { return bad_addr }; let mut badge = [0u8; MAX_BADGE]; - if bytes.read_at(0, &mut badge[..len as usize]).is_err() { - return bad_addr; + if let Err(faulted) = bytes.read_at(0, &mut badge[..len as usize]) { + return SyscallError::from(faulted).to_u64(); } sys_port_mint(RawHandle(a1 as u32), &badge[..len as usize]) } @@ -484,8 +484,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> return bad_addr; }; let mut raw = [0u8; CPU_MASK_WORDS * 8]; - if bytes.read_at(0, &mut raw[..a3 as usize * 8]).is_err() { - return bad_addr; + if let Err(faulted) = bytes.read_at(0, &mut raw[..a3 as usize * 8]) { + return SyscallError::from(faulted).to_u64(); } let words: [u64; CPU_MASK_WORDS] = core::array::from_fn(|i| u64::from_le_bytes(raw[i * 8..][..8].try_into().unwrap())); @@ -501,7 +501,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> match sys_thread_get_affinity(a1) { Ok(words) => { let wrote = words.iter().enumerate().try_for_each(|(i, word)| out.write_at(i * 8, &word.to_le_bytes())); - wrote.map_or(bad_addr, |()| 0) + wrote.map_or_else(|faulted| SyscallError::from(faulted).to_u64(), |()| 0) } Err(e) => e.to_u64(), } @@ -518,8 +518,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> return bad_addr; }; let mut name = [0u8; process::THREAD_NAME_LEN]; - if bytes.read_at(0, &mut name[..len]).is_err() { - return bad_addr; + if let Err(faulted) = bytes.read_at(0, &mut name[..len]) { + return SyscallError::from(faulted).to_u64(); } process::set_current_thread_name(&name[..len]); 0 diff --git a/kernel/src/syscall/machine.rs b/kernel/src/syscall/machine.rs index 5d3e330e31..317c7b35ff 100644 --- a/kernel/src/syscall/machine.rs +++ b/kernel/src/syscall/machine.rs @@ -99,7 +99,7 @@ fn read_on_cursor( let words = |bytes: &[u8]| -> Vec { bytes.as_chunks::<8>().0.iter().map(|word| u64::from_le_bytes(*word)).collect() }; - let Ok(head) = ctx.user_vec(at, HEAD as u64) else { return SyscallError::BadAddress.to_u64() }; + let head = match ctx.user_vec(at, HEAD as u64) { Ok(head) => head, Err(e) => return e.to_u64() }; let mut head = CursorHead::from_words(words(&head).try_into().expect("the head's words")); head.shards = u32::try_from(rings).expect("`smp::size` bounds the CPUs below a u32"); let put = |head: CursorHead, positions: &[u64]| -> Result<(), SyscallError> { @@ -112,10 +112,13 @@ fn read_on_cursor( if head.slots as usize != rings { return put(head, &[]).err().unwrap_or(SyscallError::InvalidArgument).to_u64(); } - let Some(positions) = at.raw().checked_add(HEAD as u64).and_then(|p| ctx.user_vec(UserAddr::new(p), (8 * rings) as u64).ok()) - else { + let Some(after) = at.raw().checked_add(HEAD as u64) else { return SyscallError::BadAddress.to_u64(); }; + let positions = match ctx.user_vec(UserAddr::new(after), (8 * rings) as u64) { + Ok(positions) => positions, + Err(e) => return e.to_u64(), + }; let (count, walk) = match read(&words(&positions)) { Ok(read) => read, Err(e) => return put(head, &[]).err().unwrap_or(e).to_u64(), diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index 5fa6f6565a..81077a8f4f 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -116,7 +116,7 @@ pub(super) fn sys_paged_stat(ctx: &SyscallContext<'_>, h: RawHandle, out: UserAd } } -fn holder(h: RawHandle, right: Rights) -> Result, u64> { +pub(super) fn holder(h: RawHandle, right: Rights) -> Result, u64> { let handles = process::handles(); let resolved = holder_with(&handles.lock(), h, right, handles.is_pager()); match resolved { @@ -144,7 +144,7 @@ fn holder_with( } pub(super) fn sys_paged_op(ctx: &SyscallContext<'_>, h: RawHandle, op: u64, args: UserAddr) -> u64 { - let Ok(args) = ctx.copy_in::(args) else { return SyscallError::BadAddress.to_u64() }; + let args = match ctx.copy_in::(args) { Ok(args) => args, Err(e) => return e.to_u64() }; let pager = match pager(h, Rights::WRITE) { Ok(pager) => pager, Err(refused) => return refused, diff --git a/kernel/src/syscall/vm.rs b/kernel/src/syscall/vm.rs index f9fdf51b71..7769b88c5d 100644 --- a/kernel/src/syscall/vm.rs +++ b/kernel/src/syscall/vm.rs @@ -11,6 +11,7 @@ use toyos_abi::handle::{RawHandle, Rights}; use crate::mm::space::At; use crate::object::shm::SharedMemObject; +use crate::object::KObjectRef; use crate::user_ptr::SyscallContext; use crate::UserAddr; use crate::process; @@ -80,7 +81,7 @@ pub(crate) fn map_rights(prot: Prot) -> Rights { /// `toyos_abi::syscall::vm_map`: a memory object's own pages. pub(super) fn sys_vm_map(ctx: &SyscallContext<'_>, h: RawHandle, args: UserAddr) -> u64 { - let Ok(args) = ctx.copy_in::(args) else { return SyscallError::BadAddress.to_u64() }; + let args = match ctx.copy_in::(args) { Ok(args) => args, Err(e) => return e.to_u64() }; let asked = (|| { let prot = vm_prot(args.prot)?; let len = whole_pages(args.len).ok_or(SyscallError::InvalidArgument)?; @@ -90,11 +91,16 @@ pub(super) fn sys_vm_map(ctx: &SyscallContext<'_>, h: RawHandle, args: UserAddr) Ok(asked) => asked, Err(e) => return SyscallError::to_u64(e), }; + let space = process::current_address_space(); + match process::with_handles(|handles| handles.get_ref(h, Rights::NONE).map(|o| matches!(o, KObjectRef::Paged(_)))) { + Ok(true) => return map_paged(h, &space, args.offset, len, prot, at), + Ok(false) => {} + Err(e) => return e.refuse(), + } let object = match process::with_handles(|handles| handles.get::(h, map_rights(prot))) { Ok(object) => object, Err(e) => return e.refuse(), }; - let space = process::current_address_space(); let mapped = space.lock().map(alloc::sync::Arc::clone(&object), args.offset, len, prot, at); let at = match mapped { Ok(at) => at, @@ -109,6 +115,21 @@ pub(super) fn sys_vm_map(ctx: &SyscallContext<'_>, h: RawHandle, args: UserAddr) at.raw() } +/// A paged object's holder `h`: its pages shared, read-only, as `READ` +/// allows, and refused a pager's process as its reads are +/// (`syscall::paged`). +fn map_paged(h: RawHandle, space: &alloc::sync::Arc, offset: u64, len: u64, prot: Prot, at: At) -> u64 { + if prot != Prot::Read { + return SyscallError::InvalidArgument.to_u64(); + } + let holder = match super::paged::holder(h, Rights::READ) { + Ok(holder) => holder, + Err(refused) => return refused, + }; + let mapped = space.lock().map_paged(space, &holder, offset, len, prot, at); + mapped.map_or_else(SyscallError::to_u64, UserAddr::raw) +} + /// Take every page of `[addr, addr + len)` away, as POSIX `munmap` does. pub(super) fn sys_vm_release(addr: u64, len: u64) -> u64 { answer(release(addr, len)) diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index 5fe35d4d08..a876dc2177 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -1,8 +1,9 @@ //! User memory, copied in and out through the running process's own mapping //! (`arch::uaccess`): a copy reaches only what a load or store the process //! made itself could reach, and a byte it cannot reach ends the copy as -//! [`Faulted`], which a syscall answers as `BadAddress`. Values are copied -//! out, never referenced. +//! [`Faulted`], which a syscall answers as `BadAddress`, or as `Io` where it +//! is a mapped file's page its server failed or lost. Values are copied out, +//! never referenced. //! //! **A window is a range of the user half**, checked for its end, overflow //! included, when it is made, and by nothing else: whether its pages are @@ -13,10 +14,12 @@ //! only the accesses the process could make through that mapping itself //! (`arch::uaccess`). //! -//! **A copy holds no kernel lock**: its fault takes its space's lock, and in -//! 7b will wait on a pager. Every spin lock raises the preemption count, so -//! each copy refuses to run above the depth its syscall's entry runs at -//! ([`gate`]), whether or not a page faults. +//! **A copy holds no kernel lock**: its fault takes its space's lock and may +//! wait on a pager. Every spin lock raises the preemption count, so each copy +//! refuses to run above the depth its syscall's entry runs at ([`gate`]), +//! whether or not a page faults. **Nor does it wait holding a page pin**: a +//! copy made holding one goes through [`Pinned`], which stops where a page +//! would be waited for. //! //! Every single-word user read is `read_volatile`, including the futex word //! and the crash dump's walk, both outside this module. @@ -24,6 +27,7 @@ use alloc::vec; use alloc::vec::Vec; use core::marker::PhantomData; +use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use kernel::vm::Access as Asked; use toyos_abi::syscall::SyscallError; @@ -41,9 +45,24 @@ fn asked(access: Access) -> Asked { } } -/// The frame behind a user virtual address, populating its page if needed. -pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Option { - unlocked_space().lock().reach(addr.raw(), asked(access)) +/// Why a user address has no frame. +pub(crate) enum Untranslated { + /// Nothing the access may reach is there. + Unreachable, + /// A mapped file's page that is absent or lost. + Away, +} + +/// The frame behind a user virtual address, populating its page if needed, +/// but waiting for no mapped file's page. +pub(crate) fn translate_user(addr: UserAddr, access: Access) -> Result { + use crate::mm::space::Populated; + let reached = unlocked_space().lock().reach(addr.raw(), asked(access)); + match reached { + Populated::At(phys) => Ok(phys), + Populated::Unreachable | Populated::PastEnd(_) => Err(Untranslated::Unreachable), + Populated::Lost(_) | Populated::Wait(_) => Err(Untranslated::Away), + } } /// This thread's futex wait in its space. @@ -67,13 +86,129 @@ fn gate() { assert!(crate::arch::cpu::interrupts_enabled(), "user_ptr: a user copy with interrupts masked"); } -/// A copy met a user byte its process could not reach. -#[derive(Debug)] -pub struct Faulted; +/// A copy met a user byte its process could not have. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Faulted { + /// Nothing the access may reach is there. + Unreachable, + /// A mapped file's page its server failed, or can no longer supply. + Lost, +} impl From for SyscallError { - fn from(_: Faulted) -> Self { - SyscallError::BadAddress + fn from(faulted: Faulted) -> Self { + match faulted { + Faulted::Unreachable => SyscallError::BadAddress, + Faulted::Lost => SyscallError::Io, + } + } +} + +/// What stopped a copy made through [`Pinned`]. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Stopped { + Faulted(Faulted), + /// A mapped file's page that is absent: the copy did not wait for it. + Absent, +} + +/// What stopped a thread's last copy short, as its fault left it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Stop { + Unreachable, + Lost, + /// Only under [`Pinned`]. + Absent, +} + +/// A thread's user copies: whether they run holding what no wait may hold +/// ([`Pinned`]), and what stopped its last one short. Written only by the +/// thread itself, by its copies and their faults. +pub struct Copies { + pinned: AtomicBool, + stop: AtomicU8, +} + +impl Copies { + pub const fn new() -> Self { + Self { pinned: AtomicBool::new(false), stop: AtomicU8::new(Stop::Unreachable as u8) } + } +} + +/// Whether the running thread's copies hold what no wait may ([`Pinned`]), +/// so its copy's fault must not wait. +pub fn pinned() -> bool { + crate::sched::driver::with_current_copies(|c| c.pinned.load(Ordering::Relaxed)).unwrap_or(false) +} + +/// A copy's fault could not serve its page, for `why`: the copy resumes at +/// its fixup, and [`last_stop`] reads this. +pub fn stopped(why: Stop) { + crate::sched::driver::with_current_copies(|c| c.stop.store(why as u8, Ordering::Relaxed)); +} + +/// What stopped the running thread's last copy short, as its fault left it: +/// what a copy routine answers where it left bytes behind (`arch::uaccess`). +pub fn last_stop() -> Stopped { + match crate::sched::driver::with_current_copies(|c| c.stop.load(Ordering::Relaxed)) { + Some(x) if x == Stop::Lost as u8 => Stopped::Faulted(Faulted::Lost), + Some(x) if x == Stop::Absent as u8 => Stopped::Absent, + _ => Stopped::Faulted(Faulted::Unreachable), + } +} + +/// A copy outside [`Pinned`]: what stopped it is a fault. +fn faulted(copied: Result<(), Stopped>) -> Result<(), Faulted> { + copied.map_err(|why| match why { + Stopped::Faulted(faulted) => faulted, + Stopped::Absent => panic!("user_ptr: a copy outside Pinned stopped short of an absent page"), + }) +} + +/// The running thread's copies while it holds what no wait may hold, page +/// pins or frames drawn from a pager: a fault on a mapped file's absent page +/// stops the copy ([`Stopped::Absent`]) instead of waiting. The holder lets +/// them go, waits for the window's pages holding nothing (`wait_in`), and +/// copies again. Not `Send`: it marks the thread that made it. +pub struct Pinned(PhantomData<*const ()>); + +impl Pinned { + pub fn begin() -> Self { + crate::sched::driver::with_current_copies(|c| c.pinned.store(true, Ordering::Relaxed)); + Self(PhantomData) + } + + /// [`UserBytesMut::write_raw`], stopping at an absent page. + /// + /// # Safety + /// As [`UserBytesMut::write_raw`]. + pub unsafe fn write(&self, out: &mut UserBytesMut, off: usize, src: *const u8, len: usize) -> Result<(), Stopped> { + let user = out.0.at("UserBytesMut::write", off, len); + // SAFETY: as `View::write_raw`. + unsafe { uaccess::copy_to_user(user, src, len) } + } + + /// [`UserBytesMut::fill_zero`], stopping at an absent page. + pub fn zero(&self, out: &mut UserBytesMut, off: usize, len: usize) -> Result<(), Stopped> { + let user = out.0.at("UserBytesMut::fill_zero", off, len); + // SAFETY: as `View::fill_zero`. + unsafe { uaccess::clear_user(user, len) } + } + + /// [`UserBytes::read_raw`], stopping at an absent page. + /// + /// # Safety + /// As [`UserBytes::read_raw`]. + pub unsafe fn read(&self, from: &UserBytes, off: usize, dst: *mut u8, len: usize) -> Result<(), Stopped> { + let user = from.0.at("UserBytes::read", off, len); + // SAFETY: as `View::read_raw`. + unsafe { uaccess::copy_from_user(dst, user, len) } + } +} + +impl Drop for Pinned { + fn drop(&mut self) { + crate::sched::driver::with_current_copies(|c| c.pinned.store(false, Ordering::Relaxed)); } } @@ -176,7 +311,7 @@ impl View { let user = self.at("UserBytes::read", off, len); // SAFETY: the caller's `dst`; `at` keeps the user range in the half, // and `gate` holds no lock across the copy. - unsafe { uaccess::copy_from_user(dst, user, len) } + faulted(unsafe { uaccess::copy_from_user(dst, user, len) }) } /// # Safety @@ -184,13 +319,13 @@ impl View { unsafe fn write_raw(&self, off: usize, src: *const u8, len: usize) -> Result<(), Faulted> { let user = self.at("UserBytesMut::write", off, len); // SAFETY: as `read_raw`. - unsafe { uaccess::copy_to_user(user, src, len) } + faulted(unsafe { uaccess::copy_to_user(user, src, len) }) } fn fill_zero(&self, off: usize, len: usize) -> Result<(), Faulted> { let user = self.at("UserBytesMut::fill_zero", off, len); // SAFETY: as `read_raw`, with no kernel memory touched. - unsafe { uaccess::clear_user(user, len) } + faulted(unsafe { uaccess::clear_user(user, len) }) } } @@ -202,6 +337,14 @@ impl UserBytes<'_> { self.0.len } + /// Populate the window's pages under `[off, off + len)` for reading, + /// waiting for a mapped file's absent page holding nothing: what a copy + /// [`Pinned`] stopped at needs before it copies again. + pub fn wait_in(&self, off: usize, len: usize) -> Result<(), Faulted> { + let start = self.0.at("UserBytes::wait_in", off, len); + crate::process::wait_in(start, start + len as u64, Asked::Read) + } + /// Copy `dst.len()` bytes out of the window at `off`; panics if out of bounds — the kernel's own arithmetic, never user input. pub fn read_at(&self, off: usize, dst: &mut [u8]) -> Result<(), Faulted> { self.0.read_into(off, dst) @@ -249,4 +392,10 @@ impl UserBytesMut<'_> { pub fn fill_zero(&mut self, off: usize, len: usize) -> Result<(), Faulted> { self.0.fill_zero(off, len) } + + /// As [`UserBytes::wait_in`], for writing. + pub fn wait_in(&mut self, off: usize, len: usize) -> Result<(), Faulted> { + let start = self.0.at("UserBytesMut::wait_in", off, len); + crate::process::wait_in(start, start + len as u64, Asked::Write) + } } diff --git a/sdk/abi/src/end.rs b/sdk/abi/src/end.rs index fa025dba65..660207c67c 100644 --- a/sdk/abi/src/end.rs +++ b/sdk/abi/src/end.rs @@ -32,6 +32,9 @@ pub enum Fault { /// A syscall named a handle the process does not hold, which ends it /// (`kernel::object::HandleError`'s rule). Handle, + /// A touch of a mapped file's page its server failed or can no longer + /// supply (`SIGBUS`). + Mapping, } const EXITED: u64 = 0; @@ -47,6 +50,7 @@ const fn fault_kind(fault: Fault) -> u64 { Fault::Arithmetic => 5, Fault::Trap => 6, Fault::Handle => 7, + Fault::Mapping => 8, } } @@ -74,6 +78,7 @@ impl End { (5, 0) => Fault::Arithmetic, (6, 0) => Fault::Trap, (7, 0) => Fault::Handle, + (8, 0) => Fault::Mapping, _ => return None, }; (fault_kind(fault) == kind).then_some(End::Fault(fault)) @@ -102,6 +107,7 @@ impl Fault { Fault::Arithmetic => "arithmetic", Fault::Trap => "trap", Fault::Handle => "handle", + Fault::Mapping => "mapping", } } } @@ -123,8 +129,8 @@ mod tests { use super::*; use crate::syscall::SyscallError; - const FAULTS: [Fault; 6] = - [Fault::Memory, Fault::Alignment, Fault::Instruction, Fault::Arithmetic, Fault::Trap, Fault::Handle]; + const FAULTS: [Fault; 7] = + [Fault::Memory, Fault::Alignment, Fault::Instruction, Fault::Arithmetic, Fault::Trap, Fault::Handle, Fault::Mapping]; fn every_end() -> impl Iterator { let exits = [0, 1, 137, 139, -1, i32::MIN, i32::MAX].map(End::Exited); @@ -158,7 +164,7 @@ mod tests { /// or fault carrying a code. #[test] fn a_word_no_end_makes_names_none() { - let past = 8 << 32; + let past = 9 << 32; for word in [past, (KILLED << 32) | 1, End::Fault(Fault::Trap).to_word() | 7] { assert_eq!(End::from_word(word), None, "{word:#x}"); } diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 7891f43902..ea31577768 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -2137,7 +2137,23 @@ crate::user_safe! { /// last handle. [`SyscallError::Gone`] for such a window whose last handle was /// closed under the call: nothing is mapped. /// +/// **A paged holder** ([`paged_holder`]) maps its object's pages as its pager's +/// index holds them, [`VmProt::Read`] only, which needs [`Rights::READ`]; +/// any other protection is `InvalidArgument`, and a pager's own process is +/// refused `PermissionDenied`, as its holder reads are. The range may reach +/// past the object's end, where the object may grow. The mapping counts as a +/// holder of the object until it goes ([`NOTICE_NO_HOLDERS`]). A touch of a +/// page the pager has not supplied waits for it, as [`paged_read`] does, and +/// is reported when overdue, never ended by a timeout; a touch of a page the +/// pager failed, of any page once the object is released or the pager ends, +/// or of a page wholly past the object's end ends the process with +/// [`Fault::Mapping`], and a copy a call makes from such a page answers +/// [`SyscallError::Io`] (past the end, `BadAddress`). `Gone` for an object +/// released or a pager ended before the call. +/// /// [`Rights::MAP`]: crate::handle::Rights::MAP +/// [`Rights::READ`]: crate::handle::Rights::READ +/// [`Fault::Mapping`]: crate::end::Fault::Mapping /// [`Rights::WRITE`]: crate::handle::Rights::WRITE /// /// # Safety @@ -2904,7 +2920,7 @@ mod tests { // --------------------------------------------------------------------------- -// Paged objects: memory a pager supplies, read by its holders through the kernel. +// Paged objects: memory a pager supplies, read and mapped by its holders through the kernel. // --------------------------------------------------------------------------- /// The most pages one write-back range covers. @@ -2912,6 +2928,8 @@ pub const PAGED_RANGE_PAGES: u64 = 512; /// What a pager asks of one of its objects through [`SYS_PAGED_OP`], each /// with a [`PagedArgs`] naming the object by the id [`paged_create`] answered. +/// A withdraw, a failure and a shrink return only once no mapping reaches a +/// page they removed: every mapper's leaves of it are cleared. #[repr(u64)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PagedOp { @@ -3075,7 +3093,8 @@ crate::user_safe! { /// A reader waits for pages of the object. pub const NOTICE_REQUESTED: u64 = 1; -/// The object's last holder handle went: it may be released. +/// The object's last holder went, a handle or a mapping ([`vm_map`]): it may be +/// released. pub const NOTICE_NO_HOLDERS: u64 = 2; /// A holder dirtied pages of the object, which [`PagedOp::TakeDirtied`] names. pub const NOTICE_DIRTIED: u64 = 3; diff --git a/sdk/libc/arch/src/memory_refusals.rs b/sdk/libc/arch/src/memory_refusals.rs index d69741ecd5..bed36ad9a4 100644 --- a/sdk/libc/arch/src/memory_refusals.rs +++ b/sdk/libc/arch/src/memory_refusals.rs @@ -1,5 +1,5 @@ //! What libc's memory calls refuse before the kernel or the allocator is -//! asked: `mmap` a file, memory writable and executable at once, no bytes, a fixed place off a +//! asked: `mmap` a file other than read-only, memory writable and executable at once, no bytes, a fixed place off a //! page, or no one disposition; `posix_madvise` anything but its five; `madvise` the same, an //! address off a page, and `MADV_DONTNEED`'s discard. @@ -22,8 +22,13 @@ fn mmap_refuses_what_the_kernel_cannot_map_and_nothing_else() { (0, 4096, rw, MAP_ANONYMOUS, Some(MapRefusal::Invalid)), (0, 4096, rw, MAP_SHARED | MAP_PRIVATE | MAP_ANONYMOUS, Some(MapRefusal::Invalid)), (0, 4096, PROT_READ, 0, Some(MapRefusal::Invalid)), - (0, 4096, PROT_READ, MAP_PRIVATE, Some(MapRefusal::File)), + // A file read-only, shared or private, and nothing else of one. + (0, 4096, PROT_READ, MAP_PRIVATE, None), + (0, 4096, PROT_READ, MAP_SHARED, None), (0, 4096, rw, MAP_SHARED, Some(MapRefusal::File)), + (0, 4096, rw, MAP_PRIVATE, Some(MapRefusal::File)), + (0, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, Some(MapRefusal::File)), + (0, 4096, 0, MAP_SHARED, Some(MapRefusal::File)), (0, 4096, PROT_READ | PROT_EXEC, anonymous, None), (0, 4096, PROT_WRITE | PROT_EXEC, anonymous, Some(MapRefusal::WriteExec)), (0, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, anonymous, Some(MapRefusal::WriteExec)), diff --git a/sdk/libc/arch/src/spawn_requests.rs b/sdk/libc/arch/src/spawn_requests.rs index d459820bab..e39c4e613f 100644 --- a/sdk/libc/arch/src/spawn_requests.rs +++ b/sdk/libc/arch/src/spawn_requests.rs @@ -175,6 +175,7 @@ fn each_end_reads_as_its_wait_status() { (End::Fault(Fault::Memory), "SIGSEGV"), (End::Fault(Fault::Handle), "SIGSEGV"), (End::Fault(Fault::Alignment), "SIGBUS"), + (End::Fault(Fault::Mapping), "SIGBUS"), (End::Fault(Fault::Instruction), "SIGILL"), (End::Fault(Fault::Arithmetic), "SIGFPE"), (End::Fault(Fault::Trap), "SIGTRAP"), diff --git a/sdk/libc/src/memreq.rs b/sdk/libc/src/memreq.rs index 1b3f1123db..8c19848e0b 100644 --- a/sdk/libc/src/memreq.rs +++ b/sdk/libc/src/memreq.rs @@ -26,7 +26,8 @@ pub(crate) enum MapRefusal { /// No bytes, a fixed place off a page boundary, or not exactly one of /// `MAP_SHARED` and `MAP_PRIVATE`: `EINVAL`. Invalid, - /// A file's bytes, which the kernel maps for no process: `ENODEV`. + /// A file's bytes other than read-only, which no mapping of a file gives + /// yet: `ENODEV`. File, /// Memory writable and executable at once, which no page is: `ENOTSUP`. WriteExec, @@ -43,7 +44,7 @@ pub(crate) fn mmap_refusal(addr: usize, len: usize, prot: i32, flags: i32) -> Op { return Some(MapRefusal::Invalid); } - if flags & MAP_ANONYMOUS == 0 { + if flags & MAP_ANONYMOUS == 0 && prot != PROT_READ { return Some(MapRefusal::File); } if prot & PROT_EXEC != 0 && prot & PROT_WRITE != 0 { diff --git a/sdk/libc/src/posix_io.rs b/sdk/libc/src/posix_io.rs index 0da1041bfe..e19c631c89 100644 --- a/sdk/libc/src/posix_io.rs +++ b/sdk/libc/src/posix_io.rs @@ -807,7 +807,7 @@ fn range_errno(e: syscall::SyscallError) -> i32 { #[no_mangle] pub unsafe extern "C" fn mmap( - addr: *mut u8, len: usize, prot: i32, flags: i32, _fd: i32, _offset: i64, + addr: *mut u8, len: usize, prot: i32, flags: i32, fd: i32, offset: i64, ) -> *mut u8 { use crate::memreq::{self, MapRefusal}; @@ -829,6 +829,25 @@ pub unsafe extern "C" fn mmap( // POSIX takes a null address with `MAP_FIXED` as the address zero, which // no process holds; anything else without it is only a hint. let at = if flags & memreq::MAP_FIXED != 0 { syscall::VmAt::Fixed(addr) } else { syscall::VmAt::ANYWHERE }; + if flags & memreq::MAP_ANONYMOUS == 0 { + // Read-only, so private and shared are the same pages until a write. + let Some(offset) = u64::try_from(offset).ok().filter(|o| o.is_multiple_of(syscall::PAGE_SIZE as u64)) else { + crate::errno::set(EINVAL); + return MAP_FAILED; + }; + // SAFETY: a `MAP_FIXED` caller gives up what those pages held, as POSIX says. + return match unsafe { files::map(fd, offset, len, at) } { + Ok(ptr) => ptr, + Err(files::Error::Kernel(syscall::SyscallError::NotSupported)) => { + crate::errno::set(crate::errcode::ENODEV); + MAP_FAILED + } + Err(e) => { + set_errno(e); + MAP_FAILED + } + }; + } // SAFETY: a `MAP_FIXED` caller gives up what those pages held, as POSIX says. match unsafe { syscall::vm_reserve(len, prot, at) } { Ok(ptr) => ptr, diff --git a/sdk/libc/src/refused.rs b/sdk/libc/src/refused.rs index 3ec65d6dc4..33eed109c3 100644 --- a/sdk/libc/src/refused.rs +++ b/sdk/libc/src/refused.rs @@ -110,8 +110,8 @@ pub unsafe extern "C" fn getpwuid_r(_uid: u32, _pwd: *mut u8, _buf: *mut u8, _bu ENOSYS } -/// No call answers which pages of a range are mapped, and no mapping is of a -/// file (`mmap`). +/// No call answers which pages of a range are mapped, which POSIX's `ENOMEM` +/// for an unmapped one needs. #[no_mangle] pub unsafe extern "C" fn msync(_addr: *mut u8, _len: usize, _flags: i32) -> i32 { refuse() diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index e36c18d9ae..9009c0b9c3 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -223,6 +223,7 @@ extern "C" { fn toyos_fs_write(fd: Fd, buf: *const u8, len: usize) -> i64; fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u64) -> i64; fn toyos_fs_pwrite(fd: Fd, buf: *const u8, len: usize, offset: u64) -> i64; + fn toyos_fs_map(fd: Fd, offset: u64, len: usize, place: u64, fixed: bool) -> i64; fn toyos_fs_seek(fd: Fd, whence: u32, delta: i64) -> i64; fn toyos_fs_truncate(fd: Fd, size: u64) -> i64; fn toyos_fs_fsync(fd: Fd) -> i64; @@ -276,6 +277,22 @@ pub fn pread(fd: Fd, buf: &mut [u8], offset: u64) -> Result { decode(unsafe { toyos_fs_pread(fd, buf.as_mut_ptr(), buf.len(), offset) }).map(|n| n as usize) } +/// `len` bytes of the file `fd` from the page boundary `offset`, mapped +/// read-only and shared ([`crate::paged::Paged::map`]) where `at` says. +/// `Kernel(NotSupported)` for a file its server maps no pages of. +/// +/// # Safety +/// A `Fixed` range replaces what those pages held, so nothing may still +/// reach that. +pub unsafe fn map(fd: Fd, offset: u64, len: usize, at: toyos_abi::syscall::VmAt) -> Result<*mut u8, Error> { + let (place, fixed) = match at { + toyos_abi::syscall::VmAt::Fixed(addr) => (addr.expose_provenance() as u64, true), + toyos_abi::syscall::VmAt::Anywhere { .. } => (0, false), + }; + // SAFETY: the caller's. + decode(unsafe { toyos_fs_map(fd, offset, len, place, fixed) }).map(|addr| core::ptr::with_exposed_provenance_mut(addr as usize)) +} + /// Write at `offset`, leaving the descriptor's offset where it is, whether or /// not it appends. pub fn pwrite(fd: Fd, buf: &[u8], offset: u64) -> Result { diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index eab0607fc5..3fe118fcd8 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -592,6 +592,31 @@ unsafe extern "C" fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u6 answer(n.map(|n| n as u64)) } +/// `len` bytes of the file `fd` names from the page boundary `offset`, mapped +/// read-only and shared through its paged holder, at `place` where `fixed`, +/// else anywhere: the address. `NotSupported` for a file its server supplied +/// no holder for, and a descriptor that names no served file. +/// +/// # Safety +/// A fixed range replaces what those pages held, so nothing may still reach +/// that. +#[no_mangle] +unsafe extern "C" fn toyos_fs_map(fd: Fd, offset: u64, len: usize, place: u64, fixed: bool) -> i64 { + let at = match fixed { + true => syscall::VmAt::Fixed(core::ptr::with_exposed_provenance_mut(place as usize)), + false => syscall::VmAt::ANYWHERE, + }; + let mapped = on_fd_held( + fd, + |_| Ok(Err(SyscallError::NotSupported)), + is_directory, + // SAFETY: the caller's. + |holder, _| Some(Ok(unsafe { holder.map(offset, len, at) }.map(|ptr| ptr.expose_provenance() as u64))), + |_, _, _| Ok(Err(SyscallError::NotSupported)), + ); + answer(mapped.and_then(|inner| inner.map_err(Error::Kernel))) +} + #[no_mangle] unsafe extern "C" fn toyos_fs_pwrite(fd: Fd, buf: *const u8, len: usize, offset: u64) -> i64 { // SAFETY: the caller's. diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index 42d2bf200e..053ae07f3c 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -60,7 +60,7 @@ impl Refused { pub enum Notice { /// A reader waits for `count` pages from `first`. Requested { object: u64, first: u64, count: u64 }, - /// The last holder handle went. + /// The last holder, a handle or a space's mapping, went. NoHolders { object: u64 }, /// A holder dirtied pages, which [`Pager::take_dirtied`] names. Dirtied { object: u64 }, @@ -281,6 +281,19 @@ impl Paged { syscall::paged_read(self.0.raw(), out, offset) } + /// `len` bytes of the object from the page boundary `offset`, mapped + /// read-only and shared where `at` says: the pager's own frames, so a + /// write of its lands in them, and a touch of a page not yet supplied + /// waits for it as a read does. A touch of a page that failed, past the + /// object's end, or after its pager ended, ends the process. + /// + /// # Safety + /// As [`syscall::vm_map`]'s: a `Fixed` range replaces what those pages held. + pub unsafe fn map(&self, offset: u64, len: usize, at: syscall::VmAt) -> Result<*mut u8, SyscallError> { + // SAFETY: the caller's. + unsafe { syscall::vm_map(self.0.raw(), offset, len, syscall::VmProt::Read, at) } + } + /// `bytes` at `offset`, one write whole: the bytes written. `WouldBlock` /// with nothing written where another write is under way, the write spans /// more than [`PAGED_HOLDER_PAGES`] pages or ends past diff --git a/sdk/toyos/src/process.rs b/sdk/toyos/src/process.rs index 11f20821f4..875bf9dc6f 100644 --- a/sdk/toyos/src/process.rs +++ b/sdk/toyos/src/process.rs @@ -28,7 +28,7 @@ pub fn signal(end: End) -> Option { End::Exited(_) => return None, End::Killed => SIGKILL, End::Fault(Fault::Memory | Fault::Handle) => SIGSEGV, - End::Fault(Fault::Alignment) => SIGBUS, + End::Fault(Fault::Alignment | Fault::Mapping) => SIGBUS, End::Fault(Fault::Instruction) => SIGILL, End::Fault(Fault::Arithmetic) => SIGFPE, End::Fault(Fault::Trap) => SIGTRAP, @@ -250,6 +250,7 @@ mod tests { (End::Fault(Fault::Memory), "SIGSEGV"), (End::Fault(Fault::Handle), "SIGSEGV"), (End::Fault(Fault::Alignment), "SIGBUS"), + (End::Fault(Fault::Mapping), "SIGBUS"), (End::Fault(Fault::Instruction), "SIGILL"), (End::Fault(Fault::Arithmetic), "SIGFPE"), (End::Fault(Fault::Trap), "SIGTRAP"), diff --git a/tests/guest/mapprobe/Cargo.toml b/tests/guest/mapprobe/Cargo.toml new file mode 100644 index 0000000000..708c5c3129 --- /dev/null +++ b/tests/guest/mapprobe/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mapprobe" +description = "Test-only: a pager whose objects a child of its own maps, each fault answered, withheld or refused as a case needs." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[dependencies] +toyos-abi = { path = "../../../sdk/abi" } +toyos = { path = "../../../sdk/toyos" } diff --git a/tests/guest/mapprobe/src/main.rs b/tests/guest/mapprobe/src/main.rs new file mode 100644 index 0000000000..c3e0235289 --- /dev/null +++ b/tests/guest/mapprobe/src/main.rs @@ -0,0 +1,394 @@ +//! A pager whose objects a child of its own maps, started by its row's +//! `pager`: what a mapping of a paged object does that only a booted kernel +//! has, its tables, its faults and a second process. A pager may not map, +//! so each case's mapper is this program again, `--map `, endowed a +//! holder and pipe ends, and judged by its end and by what the pager hears. +//! +//! - `coherent`: a mapper sees the pager's write land in the page it maps, +//! and its mapping holds the object after it closed its handle. +//! - `parks`: a touch of an absent page waits until it is supplied; a +//! withdrawn page is asked for again and comes back with the same bytes; +//! a supply held past the report time is reported overdue, and the mapper +//! then goes on. +//! - `unmapped`: a range taken away under a thread parked in it ends the +//! process as a touch of nothing once the page comes. +//! - `fault`, `copy`: a kill ends a thread parked in its own fault, and one +//! parked in a write's copy from the mapping, which writes nothing. +//! - `hole`: an unwritten page of an anonymous object reads zero with no +//! request, and one the pager has no room for ends the mapper. +//! - `shrunk`: an object of 80 pages, every one touched by two mappers, is +//! shrunk to one; a touch of the first page past the new end ends one +//! mapper and of the last the other, each with `SIGBUS`. The shrink revokes +//! 79 pages, more than a walk reads at once, so one of the two is cleared +//! by a later batch. +//! - `written`: a holder's write whose bytes come from a page of the +//! mapping that is absent lands whole, with those bytes, once the pager +//! supplies the page: its copy stops there, waits, draws and copies again. +//! - `ends`, last: the pager's end answers a write from the mapping `Io` +//! and ends the mapper at its next touch. + +use std::os::toyos::process::{CommandExt, ExitStatusExt}; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +use toyos::endow::Endowments; +use toyos::paged::{Notice, Paged, PagedNotice, Pager, PAGE}; +use toyos::poller::{Poller, READABLE}; +use toyos::{OwnedHandle, Pipe}; +use toyos_abi::end::{End, Fault}; +use toyos_abi::syscall::{self, SyscallError, VmAt}; + +const SELF: &str = "/system/bin/mapprobe"; +const HOLDER: &str = "mapprobe-holder"; +const READY: &str = "mapprobe-ready"; +const GO: &str = "mapprobe-go"; +const OUT: &str = "mapprobe-out"; +const TARGET: &str = "mapprobe-target"; + +/// Every page the pager holds at once, and dirty at once. +const CAPACITY: u64 = 256; +const DIRTY: u64 = 2; +/// The pager's report time, which `parks` holds a supply past. +const REPORT: Duration = Duration::from_secs(1); +/// How long a step may take before the probe gives up on it: a failure +/// bound, never what ends a passing run. +const BOUND: Duration = Duration::from_secs(60); + +/// The aligned run a fault's request names, in pages. +const RUN: u64 = 16; + +fn main() { + let mut args = std::env::args().skip(1); + if args.next().as_deref() == Some("--map") { + return map(&args.next().expect("mapprobe --map: a case")); + } + let pager = Pager::create(CAPACITY, DIRTY, REPORT).expect("mapprobe: its row starts it as a pager"); + let mut served = Served { pager, poller: Poller::new(1), unheld: Vec::new() }; + coherent(&mut served); + parks(&mut served); + killed(&mut served); + hole(&mut served); + shrunk(&mut served); + written(&mut served); + ends(served); + println!("mapprobe: ok"); +} + +/// The pager, and the objects whose last holder its notices said went. +struct Served { + pager: Pager, + poller: Poller, + unheld: Vec, +} + +impl Served { + /// Wait for a request for `object` naming page `first`. A request for + /// another run is a duplicate of one answered: the kernel names a run + /// again after a withdraw meets its waiter. + fn requested(&mut self, object: u64, first: u64) { + let began = Instant::now(); + while !self.heard(|n| matches!(n, Notice::Requested { object: o, first: f, .. } if o == object && f == first)) { + self.wait(began, &format!("a request for object {object} page {first}")); + } + } + + /// Wait for the notice that `object`'s last holder went. + fn unheld(&mut self, object: u64) { + let began = Instant::now(); + while !self.unheld.contains(&object) { + self.heard(|_| false); + if !self.unheld.contains(&object) { + self.wait(began, &format!("object {object}'s last holder to go")); + } + } + } + + /// Whether a pending notice is one `want` names, the rest read and kept. + fn heard(&mut self, want: impl Fn(Notice) -> bool) -> bool { + let mut buf = [PagedNotice::default(); 8]; + let mut found = false; + for notice in self.pager.notices(&mut buf).expect("mapprobe: its notices") { + if let Notice::NoHolders { object } = notice { + self.unheld.push(object); + } + found |= want(notice); + } + found + } + + fn wait(&self, began: Instant, what: &str) { + let left = BOUND.checked_sub(began.elapsed()).unwrap_or_else(|| panic!("mapprobe: no {what} within {BOUND:?}")); + self.poller.watch(&self.pager, READABLE, 0); + self.poller.wait(1, left.as_nanos() as u64, |_| {}); + } + + /// `f` under the pager's hold of `object`'s writes, which its writes and + /// truncations need. + fn holding(&self, object: u64, f: impl FnOnce(&Pager) -> R) -> R { + self.pager.hold(object).expect("mapprobe: a hold"); + let r = f(&self.pager); + self.pager.unhold(object).expect("mapprobe: an unhold"); + r + } + + fn supply(&self, object: u64, first: u64, count: u64, byte: u8) { + let bytes = vec![byte; (count * PAGE) as usize]; + // A page already present was a duplicate's: the kernel installs nothing. + self.pager.supply(object, first * PAGE, &bytes, false).expect("mapprobe: a supply"); + } + + /// A mapper of `object` for `case`, endowed a holder and `ends`. + fn mapper(&self, object: u64, case: &str, ends: Vec<(&str, OwnedHandle)>) -> Child { + let holder = self.pager.holder(object, false).expect("mapprobe: a holder to hand on"); + let mut command = Command::new(SELF); + command.arg("--map").arg(case).endow(HOLDER, holder.into()); + for (label, end) in ends { + command.endow(label, end.into()); + } + command.spawn().expect("mapprobe: a mapper starts") + } +} + +/// A pipe: the pager's end, and the mapper's. +fn pipe() -> (Pipe, OwnedHandle) { + let (read, write) = toyos::pipe_pair().expect("mapprobe: a pipe"); + (read, OwnedHandle::from(write)) +} + +/// A pipe the other way: the mapper reads. +fn go_pipe() -> (Pipe, OwnedHandle) { + let (read, write) = toyos::pipe_pair().expect("mapprobe: a pipe"); + (write, OwnedHandle::from(read)) +} + +fn ended(child: &mut Child) -> End { + End::from_word(child.wait().expect("mapprobe: a mapper's end").into_raw()).expect("mapprobe: an end the kernel makes") +} + +fn coherent(served: &mut Served) { + let object = served.pager.object(2 * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, 1, 0x11); + let (ready, theirs) = pipe(); + let mut child = served.mapper(object, "coherent", vec![(READY, theirs)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the coherent mapper never read the supplied page"); + assert!(!served.heard(|n| n == Notice::NoHolders { object }), "mapprobe: a mapped object's last holder was said to go"); + assert_eq!(served.holding(object, |pager| pager.write(object, 0, &[0x22; PAGE as usize])), Ok(PAGE as usize)); + assert_eq!(ended(&mut child), End::Exited(0), "mapprobe: the coherent mapper"); + served.unheld(object); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a mapper sees the pager's write, and its mapping holds the object it closed"); +} + +fn parks(served: &mut Served) { + let object = served.pager.object(4 * RUN * PAGE).expect("mapprobe: an object"); + let mut child = served.mapper(object, "parks", Vec::new()); + served.requested(object, 0); + served.supply(object, 0, RUN, 0x33); + served.requested(object, RUN); + assert_eq!(served.pager.withdraw(object, 0, RUN), Ok(RUN), "mapprobe: the first run withdrawn under its mapper"); + // A stall is the stimulus here, not a wait for an event: the mapper's + // fault waits past the report time while the pager does nothing. + std::thread::sleep(2 * REPORT); + served.supply(object, RUN, RUN, 0x44); + served.requested(object, 0); + served.supply(object, 0, RUN, 0x33); + assert_eq!(ended(&mut child), End::Exited(0), "mapprobe: the parked mapper"); + println!("mapprobe: a fault waits for its page, a withdrawn page comes back, and a stalled supply is waited out"); + + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let mut child = served.mapper(object, "unmapped", vec![(READY, theirs), (GO, mine)]); + served.requested(object, 3 * RUN); + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never took its range away"); + served.supply(object, 3 * RUN, RUN, 0x55); + assert_eq!(ended(&mut child), End::Fault(Fault::Memory), "mapprobe: a thread woken in a range taken away"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a range taken away under a parked thread is nothing when its page comes"); +} + +fn killed(served: &mut Served) { + let object = served.pager.object(2 * RUN * PAGE).expect("mapprobe: an object"); + let mut child = served.mapper(object, "fault", Vec::new()); + served.requested(object, 0); + child.kill().expect("mapprobe: kill the parked mapper"); + assert_eq!(ended(&mut child), End::Killed, "mapprobe: a mapper parked in its fault"); + let (out, theirs) = pipe(); + let mut child = served.mapper(object, "copy", vec![(OUT, theirs)]); + served.requested(object, RUN); + child.kill().expect("mapprobe: kill the parked writer"); + assert_eq!(ended(&mut child), End::Killed, "mapprobe: a mapper parked in a write's copy"); + assert_eq!(out.read(&mut [0; 16]), Ok(0), "mapprobe: a write killed in its copy wrote"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a kill ends a thread parked in its fault or in a copy, and the copy's write wrote nothing"); +} + +fn hole(served: &mut Served) { + let object = served.pager.anonymous(2 * PAGE).expect("mapprobe: anonymous memory"); + let room = served.pager.room().expect("mapprobe: room"); + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let mut child = served.mapper(object, "hole", vec![(READY, theirs), (GO, mine)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: the mapper never read its hole"); + assert!(!served.heard(|n| matches!(n, Notice::Requested { object: o, .. } if o == object)), "mapprobe: a hole was requested"); + assert_eq!(served.pager.room(), Ok(room - 1), "mapprobe: a touched hole is not one page of the pager's"); + let filler = served.pager.object((room - 1) * PAGE).expect("mapprobe: an object to fill the pager"); + served.supply(filler, 0, room - 1, 0x99); + assert_eq!(served.pager.room(), Ok(0)); + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a hole touched with no room"); + served.pager.release(filler).expect("mapprobe: release"); + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room"); +} + +/// The pages `shrunk`'s object holds before its shrink: more than a walk's +/// batch, so its revocation takes two. +const SHRUNK: u64 = 80; + +fn shrunk(served: &mut Served) { + let object = served.pager.object(SHRUNK * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, SHRUNK, 0x66); + let mut mappers = Vec::new(); + for page in [1, SHRUNK - 1] { + let (ready, theirs) = pipe(); + let (go, mine) = go_pipe(); + let child = served.mapper(object, &format!("shrunk:{page}"), vec![(READY, theirs), (GO, mine)]); + assert_eq!(ready.read(&mut [0]), Ok(1), "mapprobe: a mapper never read every page"); + mappers.push((page, child, go)); + } + served.holding(object, |pager| pager.resize(object, PAGE)).expect("mapprobe: a shrink"); + for (page, mut child, go) in mappers { + assert_eq!(go.write(&[1]), Ok(1)); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a touch of page {page} past a shrunk object"); + } + served.pager.release(object).expect("mapprobe: release"); + println!("mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS"); +} + +fn written(served: &mut Served) { + let source = served.pager.object(RUN * PAGE).expect("mapprobe: an object"); + let target = served.pager.object(0).expect("mapprobe: an object to write"); + assert_eq!(served.pager.grant(target, 1), Ok(1), "mapprobe: a page of credit"); + let writer = served.pager.holder(target, true).expect("mapprobe: a write holder"); + let mut child = served.mapper(source, "written", vec![(TARGET, writer)]); + served.requested(source, 0); + served.supply(source, 0, RUN, 0x88); + assert_eq!(ended(&mut child), End::Exited(0), "mapprobe: the mapper writing from its mapping"); + let mut bytes = vec![0; PAGE as usize]; + // SAFETY: `bytes` outlives the read and nothing else reaches it. + let out = unsafe { toyos::shm::Bytes::from_raw(bytes.as_mut_ptr(), bytes.len()) }; + assert_eq!(served.pager.read(target, out, 0), Ok(PAGE as usize)); + assert!(bytes.iter().all(|&b| b == 0x88), "mapprobe: the write from an absent page landed other bytes"); + served.pager.release(target).expect("mapprobe: release"); + served.pager.release(source).expect("mapprobe: release"); + println!("mapprobe: a holder's write from a mapped page not yet supplied lands whole once it is"); +} + +fn ends(mut served: Served) { + let object = served.pager.object(2 * PAGE).expect("mapprobe: an object"); + served.supply(object, 0, 1, 0x77); + let (out, theirs) = pipe(); + let mut child = served.mapper(object, "ends", vec![(OUT, theirs)]); + served.requested(object, 0); + drop(served); + assert_eq!(ended(&mut child), End::Fault(Fault::Mapping), "mapprobe: a mapper of an ended pager's object"); + assert_eq!(out.read(&mut [0; 16]), Ok(0), "mapprobe: a write from an ended pager's page wrote"); + println!("mapprobe: a pager's end answers a write from its page Io, and ends its mapper at the next touch"); +} + +/// The object a mapper was endowed, mapped whole. +fn mapped(pages: u64) -> *mut u8 { + let handle: OwnedHandle = Endowments::get().take(HOLDER).expect("mapprobe --map: its holder"); + let holder = Paged::received(handle).expect("mapprobe --map: a holder it may read"); + // SAFETY: placed anywhere, so it replaces nothing; the holder goes and + // the mapping holds the object. + unsafe { holder.map(0, (pages * PAGE) as usize, VmAt::Anywhere { align: PAGE as usize }) }.expect("mapprobe --map: the object mapped") +} + +fn byte(at: *mut u8, page: u64) -> u8 { + // SAFETY: inside the mapping this case made, read-only. + unsafe { at.add((page * PAGE) as usize).read_volatile() } +} + +fn pipe_end(label: &str) -> Pipe { + Endowments::get().take(label).expect("mapprobe --map: a pipe end") +} + +fn map(case: &str) { + match case { + "coherent" => { + let at = mapped(2); + assert_eq!(byte(at, 0), 0x11, "mapprobe --map: the supplied page"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + let began = Instant::now(); + while byte(at, 0) != 0x22 { + assert!(began.elapsed() < BOUND, "mapprobe --map: the pager's write never reached the mapping"); + std::thread::yield_now(); + } + } + "parks" => { + let at = mapped(4 * RUN); + assert_eq!(byte(at, 4), 0x33); + assert_eq!(byte(at, RUN + 4), 0x44); + assert_eq!(byte(at, 4), 0x33, "mapprobe --map: a withdrawn page came back as other bytes"); + } + "unmapped" => { + let at = mapped(4 * RUN) as usize; + let parked = std::thread::spawn(move || byte(at as *mut u8, 3 * RUN + 2)); + assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); + // SAFETY: the mapping this case made, which the parked thread + // touches again only to fault. + unsafe { syscall::vm_release(at as *mut u8, (4 * RUN * PAGE) as usize) }.expect("mapprobe --map: unmap"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + let read = parked.join(); + panic!("mapprobe --map: a thread woken in a range taken away read {read:?}"); + } + "fault" => { + let read = byte(mapped(2 * RUN), 2); + panic!("mapprobe --map: a page nobody supplies read {read}"); + } + "copy" => { + let at = mapped(2 * RUN); + // SAFETY: a page of the mapping, which the kernel copies from. + let page = unsafe { std::slice::from_raw_parts(at.add((RUN * PAGE) as usize), PAGE as usize) }; + let wrote = pipe_end(OUT).write(page); + panic!("mapprobe --map: a write from a page nobody supplies answered {wrote:?}"); + } + "hole" => { + let at = mapped(2); + assert_eq!(byte(at, 0), 0, "mapprobe --map: an unwritten page"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); + let read = byte(at, 1); + panic!("mapprobe --map: a hole the pager had no room for read {read}"); + } + shrunk if shrunk.starts_with("shrunk:") => { + let page: u64 = shrunk["shrunk:".len()..].parse().expect("mapprobe --map: a page"); + let at = mapped(SHRUNK); + assert!((0..SHRUNK).all(|page| byte(at, page) == 0x66), "mapprobe --map: the supplied pages"); + assert_eq!(pipe_end(READY).write(&[1]), Ok(1)); + assert_eq!(pipe_end(GO).read(&mut [0]), Ok(1)); + let read = byte(at, page); + panic!("mapprobe --map: page {page}, past the object's end, read {read}"); + } + "written" => { + let at = mapped(RUN); + let handle: OwnedHandle = Endowments::get().take(TARGET).expect("mapprobe --map: a write holder"); + let target = Paged::received(handle).expect("mapprobe --map: a holder it may write"); + // SAFETY: a page of the mapping, which the kernel copies from. + let page = unsafe { std::slice::from_raw_parts(at, PAGE as usize) }; + assert_eq!(target.write(page, 0), Ok(PAGE as usize), "mapprobe --map: a write from a page not yet supplied"); + } + "ends" => { + let at = mapped(2); + assert_eq!(byte(at, 0), 0x77); + // SAFETY: a page of the mapping, which the kernel copies from. + let page = unsafe { std::slice::from_raw_parts(at.add(PAGE as usize), PAGE as usize) }; + assert_eq!(pipe_end(OUT).write(page), Err(SyscallError::Io), "mapprobe --map: a write from an ended pager's page"); + let read = byte(at, 0); + panic!("mapprobe --map: a page of an ended pager read {read}"); + } + other => panic!("mapprobe --map: no case {other:?}"), + } +} diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index a0e6e1e450..3deb6461b4 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -117,6 +117,154 @@ fn parts( each(&name("pread_8"), 100, 2000, fields, |i| io("pread", f.read_at(&mut buf[..8], page_at(i, pages))).map(drop)); } +/// The `map` group under `target`: files written there, so their pages are +/// cached, mapped read-only and shared, each line naming what served them. +/// The 16 MiB file is mapped, each page touched once and unmapped, so each +/// touch is a fault unless fault-around mapped its page already +/// (`touch_16m`, 20 after 2); the same by eight threads, each its eighth of +/// the pages (`threads_touch_16m`, 10 after 1); and 4 KiB copied out of a +/// page of one mapping every page of which was touched (`read_4k`, 2,000 +/// after 100). A 1 MiB file mapped and touched whole by 255 other processes +/// is truncated to nothing, which takes its pages from every one +/// (`truncate_255`, 5, the truncate alone timed). +pub fn map_suite(target: &Target) { + condition(&format!("map.{}.served_by", target.name), &target.served_by); + let scratch = match Scratch::new(&target.dir) { + Ok(s) => s, + Err(e) => return refuse(&format!("map.{}", target.name), format!("no scratch directory under {}: {e}", target.dir)), + }; + let name = |op: &str| format!("map.{}.{op}", target.name); + let big = scratch.at("big"); + if let Err(e) = fill(&big, 16 * MIB) { + return refuse(&name("setup"), e); + } + let fields: Vec<(&'static str, String)> = + [("file_bytes", (16 * MIB).to_string())].into_iter().chain(target.fields()).collect(); + mapped(&name, &fields, &big, 16 * MIB / 4096); + let _ = fs::remove_file(&big); + let held = scratch.at("held"); + let fields: Vec<(&'static str, String)> = + [("file_bytes", MIB.to_string()), ("mappers", MAPPERS.to_string())].into_iter().chain(target.fields()).collect(); + held_by_many(&name("truncate_255"), &fields, &held); + let _ = fs::remove_file(&held); +} + +/// Every page of `len` bytes at `at`, read once. +fn touch(at: usize, pages: std::ops::Range) -> u8 { + pages.fold(0u8, |sum, page| { + // SAFETY: a page of a read-only mapping the caller holds. + sum.wrapping_add(unsafe { (at as *const u8).add(page * 4096).read_volatile() }) + }) +} + +fn mapped(name: &dyn Fn(&str) -> String, fields: &[(&'static str, String)], big: &Path, pages: usize) { + let len = pages * 4096; + each(&name("touch_16m"), 2, 20, fields, |_| { + let at = crate::sys::map_file(big, len)?; + std::hint::black_box(touch(at as usize, 0..pages)); + // SAFETY: the mapping just made, which nothing else names. + unsafe { crate::sys::unmap(at, len) }; + Ok(()) + }); + each(&name("threads_touch_16m"), 1, 10, fields, |_| { + let at = crate::sys::map_file(big, len)? as usize; + std::thread::scope(|threads| { + for t in 0..THREADS { + threads.spawn(move || std::hint::black_box(touch(at, t * pages / THREADS..(t + 1) * pages / THREADS))); + } + }); + // SAFETY: the mapping just made, whose threads have all ended. + unsafe { crate::sys::unmap(at as *mut u8, len) }; + Ok(()) + }); + let at = match crate::sys::map_file(big, len) { + Ok(at) => at, + Err(why) => return refuse(&name("read_4k"), why), + }; + std::hint::black_box(touch(at as usize, 0..pages)); + let mut buf = [0u8; 4096]; + each(&name("read_4k"), 100, 2000, fields, |i| { + // SAFETY: a page of the mapping, every one of which is mapped. + unsafe { core::ptr::copy_nonoverlapping(at.add(page_at(i, pages) as usize), buf.as_mut_ptr(), buf.len()) }; + std::hint::black_box(&buf); + Ok(()) + }); + // SAFETY: the mapping made above, which nothing else names. + unsafe { crate::sys::unmap(at, len) }; +} + +/// The threads `threads_touch_16m` faults one mapping with. +const THREADS: usize = 8; +/// The processes holding `truncate_255`'s file mapped. +const MAPPERS: usize = 255; + +/// `held`, 1 MiB, mapped and touched whole by [`MAPPERS`] helpers of this +/// program (`osbench map-hold`), then truncated to nothing: the truncate's +/// time, five times, the file written again and the helpers started afresh +/// for each. +fn held_by_many(bench: &str, fields: &[(&'static str, String)], held: &Path) { + let exe = match std::env::current_exe() { + Ok(exe) => exe, + Err(e) => return refuse(bench, format!("no path to this program: {e}")), + }; + let mut ns = Vec::new(); + for _ in 0..5 { + let sample = (|| -> Result { + io("write", fill(held, MIB))?; + let mut helpers = Vec::with_capacity(MAPPERS); + for _ in 0..MAPPERS { + let mut helper = io( + "spawn", + std::process::Command::new(&exe) + .arg("map-hold") + .arg(held) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn(), + )?; + let mut said = [0u8; 5]; + io("read", helper.stdout.take().expect("piped").read_exact(&mut said))?; + if &said != b"held\n" { + return Err(format!("a helper said {said:?}")); + } + helpers.push(helper); + } + let file = io("open", OpenOptions::new().write(true).open(held))?; + let t = std::time::Instant::now(); + io("truncate", file.set_len(0))?; + let took = t.elapsed().as_nanos() as u64; + for mut helper in helpers { + drop(helper.stdin.take()); + let status = io("wait", helper.wait())?; + if !status.success() { + return Err(format!("a helper ended {status}")); + } + } + Ok(took) + })(); + match sample { + Ok(took) => ns.push(took), + Err(why) => return refuse(bench, why), + } + } + crate::report::result(bench, "ns", &mut ns, fields); +} + +/// `osbench map-hold `: the file at `path` mapped whole and every page +/// touched, said by "held", and held until stdin ends. +pub fn hold(args: &[String]) { + let path = Path::new(args.first().map(String::as_str).expect("osbench map-hold: a path")); + let len = fs::metadata(path).expect("osbench map-hold: the file").len() as usize; + let at = crate::sys::map_file(path, len).expect("osbench map-hold: the file mapped"); + std::hint::black_box(touch(at as usize, 0..len / 4096)); + let mut out = std::io::stdout(); + out.write_all(b"held\n").and_then(|()| out.flush()).expect("osbench map-hold: say held"); + let _ = std::io::stdin().read_to_end(&mut Vec::new()); + // SAFETY: the mapping made above; the file may have shrunk under it, and + // nothing touches it again. + unsafe { crate::sys::unmap(at, len) }; +} + /// Every benchmark under `target`, each line naming what served it and its /// medium. Each one's comment is its warm-up and repetitions. pub fn suite(target: &Target) { diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index cf184ad1b3..6886e6ddb6 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -130,18 +130,23 @@ fn files() { files::suite(&files::memory()); } +fn map() { + files::map_suite(&files::memory()); +} + /// Every group the machine alone runs, in the order `all` and the ToyOS job /// list run them: `boot` first, since it reads how long the boot took to /// reach it; and `crunch`, which heats it, last: ToyOS reads no temperature /// to cool down by, so a group after it would start at a clock its Linux run /// does not. -const GROUPS: [(&str, fn()); 16] = [ +const GROUPS: [(&str, fn()); 17] = [ ("boot", boot::suite), ("identity", identity::suite), ("system", system), ("sched", sched), ("fair", fair), ("files", files), + ("map", map), ("ipc", ipc::suite), ("net", net::suite), ("procs", procs::suite), @@ -205,6 +210,7 @@ fn main() { "fair-job" => return fair::job(rest), "fair-helper" => return fair::helper(rest), "ipc-serve" => return ipc::serve(), + "map-hold" => return files::hold(rest), "cc" => return work::cc(rest), named => match GROUPS.iter().chain(&BY_NAME).find(|(name, _)| *name == named) { Some((name, run)) => group(name, *run), diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index 4c056d177d..264875eaaa 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -58,9 +58,22 @@ pub fn map_anon(len: usize) -> *mut u8 { ptr.cast() } +/// `len` bytes of the file at `path` from its start, mapped read-only and +/// shared. +pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { + use std::os::fd::AsRawFd; + let file = std::fs::File::open(path).map_err(|e| format!("open: {e}"))?; + // SAFETY: a fresh read-only mapping of a file this process opened. + let ptr = unsafe { libc::mmap(std::ptr::null_mut(), len, libc::PROT_READ, libc::MAP_SHARED, file.as_raw_fd(), 0) }; + if ptr == libc::MAP_FAILED { + return Err(format!("mmap: {}", std::io::Error::last_os_error())); + } + Ok(ptr.cast()) +} + /// # Safety -/// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing -/// reaches it afterwards. +/// `ptr` and `len` are one whole mapping from [`map_anon`] or +/// [`map_file`], and nothing reaches it afterwards. pub unsafe fn unmap(ptr: *mut u8, len: usize) { // SAFETY: as the caller promises. assert_eq!(unsafe { libc::munmap(ptr.cast(), len) }, 0); diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index b562f789d0..861bf24e04 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -102,9 +102,24 @@ pub fn huge_backing() -> String { "0KiB".into() } +/// `len` bytes of the file at `path` from its start, mapped read-only and +/// shared: its file server's own cached pages. +pub fn map_file(path: &std::path::Path, len: usize) -> Result<*mut u8, String> { + use toyos::files::{At, Open}; + let path = path.to_str().ok_or("a path that is not UTF-8")?; + let fd = toyos::files::open(At::Cwd, path.as_bytes(), Open { read: true, cloexec: true, ..Open::default() }) + .map_err(|e| format!("open: {e:?}"))?; + // SAFETY: placed anywhere, so it replaces nothing. + let mapped = unsafe { toyos::files::map(fd, 0, len, VmAt::Anywhere { align: 4096 }) }; + // SAFETY: the descriptor this opened, which nothing else names; the + // mapping holds the file. + let _ = unsafe { toyos::fd::close(fd) }; + mapped.map_err(|e| format!("map: {e:?}")) +} + /// # Safety -/// `ptr` and `len` are one whole mapping from [`map_anon`], and nothing -/// reaches it afterwards. +/// `ptr` and `len` are one whole mapping from [`map_anon`] or +/// [`map_file`], and nothing reaches it afterwards. pub unsafe fn unmap(ptr: *mut u8, len: usize) { // SAFETY: as the caller promises. unsafe { syscall::vm_release(ptr, len) }.expect("vm_release of a whole mapping"); diff --git a/tests/guest/rust-tests/src/bin/mapped_file.rs b/tests/guest/rust-tests/src/bin/mapped_file.rs new file mode 100644 index 0000000000..1996b04eb4 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/mapped_file.rs @@ -0,0 +1,20 @@ +//! A paged object mapped by another process, on both architectures: +//! `mapprobe`, launched as its row's pager, maps its objects in children of +//! its own, answers or withholds each fault and judges each child by its end +//! (`tests/guest/mapprobe`). The probe says each claim as it holds and exits +//! 0 once every one did. +//! +//! Needs the guest: the faults, waits, revocations and refusals are a booted +//! kernel's, between two processes. + +use std::os::toyos::process::CommandExt; +use std::process::Command; + +fn main() { + let status = Command::new("/system/bin/mapprobe") + .independent() + .status() + .expect("the mapping probe launches: its image's test-runner starts it"); + assert!(status.success(), "the mapping probe ended {status:?}"); + println!("mapped_file: a mapped paged object follows its pager"); +} diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 1972b82cb3..09d19d4a9c 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -78,6 +78,7 @@ args = [ "osbench_sched", "osbench_fair", "osbench_files", + "osbench_map", "osbench_ipc", "osbench_net", "osbench_procs", @@ -103,6 +104,7 @@ args = [ "bin/osbench_sched" = "/system/bin/osbench" "bin/osbench_fair" = "/system/bin/osbench" "bin/osbench_files" = "/system/bin/osbench" +"bin/osbench_map" = "/system/bin/osbench" "bin/osbench_ipc" = "/system/bin/osbench" "bin/osbench_net" = "/system/bin/osbench" "bin/osbench_procs" = "/system/bin/osbench" diff --git a/tests/images/testcases.toml b/tests/images/testcases.toml index 9702a5fafb..6d143a9ac8 100644 --- a/tests/images/testcases.toml +++ b/tests/images/testcases.toml @@ -76,8 +76,9 @@ syscap = ["rt"] # `/state/test-runner` and nothing else of `/state`. [programs.test-runner] state = true -# `pagerprobe`, a pager row, which `a_pager_s_end_answers_its_waiting_holders_gone` launches. -starts = ["pagerprobe"] +# `pagerprobe` and `mapprobe`, pager rows, which `a_pager_s_end_answers_its_waiting_holders_gone` +# and `mapped_file` launch. +starts = ["pagerprobe", "mapprobe"] receives = ["soundserver", "volume", "power", "log"] syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace", "inventory", "jobs"] @@ -86,6 +87,10 @@ syscap = ["device", "dup", "logread", "power", "roster", "counters", "trace", "i [programs.pagerprobe] pager = true +# A pager whose objects its own children map (`tests/guest/mapprobe`). +[programs.mapprobe] +pager = true + [programs.toybox] receives = ["soundserver"] diff --git a/tests/images/virtjobcase.toml b/tests/images/virtjobcase.toml index 64a46a17f6..ef63723253 100644 --- a/tests/images/virtjobcase.toml +++ b/tests/images/virtjobcase.toml @@ -20,12 +20,18 @@ restart = true [programs.test-runner] receives = ["power"] +# `mapprobe`, a pager row, which `test_rs_mapped_file` launches. +starts = ["mapprobe"] # Four minutes from boot rather than one: a job here may run on an emulated # CPU, and a job past the bound reboots the guest with the job named. -args = ["--bound-ms=240000", "preempt", "fp_isolation", "first_entry", "unmap_touch", "debug_refused", "test_rs_ring0_timer_in_syscall", "test_rs_random_draws", "test_rs_wall_clock_now", "test_rs_user_copy_faults", "test_rs_vm_map_shares", "test_rs_relro_sealed", "test_rs_abuse_readonly_copyout"] +args = ["--bound-ms=240000", "preempt", "fp_isolation", "first_entry", "unmap_touch", "debug_refused", "test_rs_ring0_timer_in_syscall", "test_rs_random_draws", "test_rs_wall_clock_now", "test_rs_user_copy_faults", "test_rs_vm_map_shares", "test_rs_mapped_file", "test_rs_relro_sealed", "test_rs_abuse_readonly_copyout"] [programs.kernelprobe] +# A pager whose objects its own children map (`tests/guest/mapprobe`). +[programs.mapprobe] +pager = true + [symlinks] "bin/preempt" = "/system/bin/kernelprobe" "bin/fp_isolation" = "/system/bin/kernelprobe" diff --git a/tests/toyos.rs b/tests/toyos.rs index 7d266e9deb..8e9b6a94eb 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -401,6 +401,10 @@ const DRIVEN_AND_SHARED: &[&str] = &[ // Its shared run is the x86-64 verdict; `virt_jobs_at_el2` builds it for // AArch64 and runs it on that architecture's job case. "relro_sealed", + // Its shared run is the x86-64 verdict on its cases; + // `a_mapped_file_follows_its_pager` runs it again for the kernel's report + // on serial, and `virt_mapped_file` on AArch64's job case. + "mapped_file", ]; /// What the boot's audit of the kernel's tables says when every leaf passed @@ -456,6 +460,9 @@ const SCREEN_TESTS: &[ScreenTest] = &[ // `tmp_empty_files` on AArch64: the paged objects' table and its notices // are shared memory-management code. ("virt_tmp_holds_empty_files_by_what_each_costs", qemu::Profile::Virt, |profile, _| virt_tmp_holds_empty_files_by_what_each_costs(profile)), + // `a_mapped_file_follows_its_pager` on AArch64: a fault that parks on its + // pager there, the CPU's fault mark down while it waits. + ("virt_mapped_file", qemu::Profile::Virt, |profile, _| virt_job(profile, "test_rs_mapped_file")), ("virt_ring0_timer_in_syscall", qemu::Profile::Virt, |profile, _| virt_job(profile, &format!("test_rs_{VIRT_RING0_TIMER}"))), // Emulated, with `virt_el1_smp`, `virt_off_names_the_cpus_left_on` and // `virt_reboot`: under HVF a woken thread can go undispatched for seconds @@ -658,6 +665,12 @@ const MACHINE_TESTS: &[MachineTest] = &[ // endowments a job gets is the image's and the supervisor's, which no // host build has, and the benches run osbench only nightly. ("osbench_capability_groups_exit_0", |_| osbench_capability_groups_exit_0()), + // `osbench map` on tests/images/benchcase.toml with only that job, on + // QEMU x86-64's q35: its rows over a mapped file, eight threads faulting + // one mapping and 255 processes holding one while it is truncated, each + // required measured. Whether they measure correctly is a guest's to say; + // what they measure is owed on metal (the memory track record). + ("osbench_map_group_exits_0", |_| osbench_map_group_exits_0()), // What the kernel reads and writes for the `acpi` claim's holder and what // it refuses. A red here is a write the kernel made — to RAM, to the // firmware's tables, to COM1, to `PM1a_CNT`, to a function's configuration @@ -731,6 +744,15 @@ const MACHINE_TESTS: &[MachineTest] = &[ ("a_user_copy_highest_first_faults_or_ends_as_bad_address", |test_config| { a_user_copy_faults_in_mid_copy_or_ends_as_bad_address(test_config, &["copies-go-down"]) }), + // A paged object mapped by a second process (`tests/guest/mapprobe`), and + // the kernel's report that a mapper's fault waited past its pager's + // report time and then no longer did. That report is on serial and not + // the job's, and `testcases_members_exit_0` judges the member by its exit + // alone, so this boot of QEMU's x86-64 machine is its own. The job's + // cases, two processes, real tables and the kernel's fault path, which no + // host test has, are the shared run's too, and `virt_mapped_file` runs + // them on AArch64. + ("a_mapped_file_follows_its_pager", |test_config| a_mapped_file_follows_its_pager(test_config)), // The boot's log written through usbd onto the stick it booted from, and // still there once the machine has stopped: the path every metal boot's // `/log` takes, which no other QEMU row reads, because their console is @@ -3105,6 +3127,7 @@ const VIRT_JOBS: &[(&str, &str)] = &[ ("test_rs_user_copy_faults", "user copies: all checks passed"), ("test_rs_vm_map_shares", "vm_map_shares: an object's pages are mapped where asked"), ("test_rs_relro_sealed", "relro_sealed: a store to a relocated word of PT_GNU_RELRO ended its program as a memory fault"), + ("test_rs_mapped_file", "mapped_file: a mapped paged object follows its pager"), ("test_rs_abuse_readonly_copyout", "a syscall writes only where its caller could store"), ]; @@ -3443,6 +3466,32 @@ fn osbench_capability_groups_exit_0() -> Result<(), String> { Ok(()) } +/// `osbench map` on tests/images/benchcase.toml alone: it exits 0, which it +/// does only once every row it ran measured, and says each row's name. +fn osbench_map_group_exits_0() -> Result<(), String> { + const JOB: &str = "osbench_map"; + let committed = compile::repo_root().join("tests/images/benchcase.toml"); + let text = fs::read_to_string(&committed).map_err(|e| format!("{}: {e}", committed.display()))?; + let derived = toyos_metal::image::derive(&text, 600_000, &[JOB], &[]).map_err(|why| why.to_string())?; + let config = common::lane::dir().join("osbench-map-group.toml"); + fs::write(&config, derived).map_err(|e| format!("{}: {e}", config.display()))?; + let mut qemu = QemuInstance::boot_with_options( + &config, + &[], + &[], + BootOptions { ready_marker: bootlog::COMPLETE, ..Default::default() }, + ); + let mut console = format!("{}\n", qemu.boot_log()); + let end = format!("===TEST_END {JOB} "); + await_guest(&mut qemu, &mut console, "osbench's map group to end", |said| said.contains(&end))?; + let said = serial::Serial::named("osbench map", console); + said.must_say(&format!("===TEST_END {JOB} exit=0==="))?; + for row in ["touch_16m", "threads_touch_16m", "read_4k", "truncate_255"] { + said.must_say(&format!(".{row} unit="))?; + } + Ok(()) +} + /// Boot the image `--console-boot` builds, on the machine shape with a panel /// for `/system/bin/console` to claim, and wait for what that image owes at /// boot: the console up on its panel, having read the log and the keyboard, @@ -3576,6 +3625,48 @@ fn a_user_copy_faults_in_mid_copy_or_ends_as_bad_address(test_config: &Path, ker Ok(()) } +/// What `mapprobe` says as each of its cases holds (`tests/guest/mapprobe`). +const MAPPED_FILE_SAID: &[&str] = &[ + "mapprobe: a mapper sees the pager's write, and its mapping holds the object it closed", + "mapprobe: a fault waits for its page, a withdrawn page comes back, and a stalled supply is waited out", + "mapprobe: a range taken away under a parked thread is nothing when its page comes", + "mapprobe: a kill ends a thread parked in its fault or in a copy, and the copy's write wrote nothing", + "mapprobe: a touched hole is a zeroed page of the pager's, and none where it has no room", + "mapprobe: a touch of a page the object shrank past ends its mapper as SIGBUS", + "mapprobe: a holder's write from a mapped page not yet supplied lands whole once it is", + "mapprobe: a pager's end answers a write from its page Io, and ends its mapper at the next touch", + "mapped_file: a mapped paged object follows its pager", +]; + +/// `mapped_file` alone: each of the probe's claims said, and the kernel's +/// report that a mapper's fault waited past its pager's report time and then +/// no longer did, which is on the serial line and not the job's own. +fn a_mapped_file_follows_its_pager(test_config: &Path) -> Result<(), String> { + const JOB: &str = "mapped_file"; + let bin = qemu::build_toyos_bin(qemu::SUITE_ARCH, &compile::repo_root().join("tests/guest/rust-tests"), JOB); + let mut qemu = QemuInstance::boot_with_options(test_config, &[], &[(JOB.to_string(), bin)], BootOptions::default()); + serial::Serial::boot(&qemu).must_be_clean()?; + let result = qemu.run_test("test_rs_mapped_file", Duration::from_secs(120)); + if let Some(why) = &result.error { + return Err(format!("{why}\nthe job said:\n{}", result.stdout)); + } + if result.exit_code != Some(0) { + return Err(format!("the job ended {:?}:\n{}", result.exit_code, result.stdout)); + } + if let Some(claim) = MAPPED_FILE_SAID.iter().find(|claim| !result.stdout.contains(*claim)) { + return Err(format!("the job never said {claim:?}:\n{}", result.stdout)); + } + // The probe's own stalled run, pages 16 to 32, not any pager's slow read. + let overdue = |line: &str| line.contains("'s pager has a wait overdue: object ") && line.contains(" pages 16+16, "); + if !result.serial.lines().any(overdue) { + return Err(format!("the kernel never said the probe's run was overdue:\n{}", result.serial)); + } + if !result.serial.contains("'s pager has no wait overdue") { + return Err(format!("the kernel never said the overdue wait ended:\n{}", result.serial)); + } + Ok(()) +} + /// A boot whose marked slot's supervisor is bent, beside a slot that /// verifies: the loader refuses the marked slot by the section, reads nothing /// of the other, hands no kernel the machine, and the machine stops.