diff --git a/docs/src/architecture/storage.md b/docs/src/architecture/storage.md index d6c83408bc1..5087057667a 100644 --- a/docs/src/architecture/storage.md +++ b/docs/src/architecture/storage.md @@ -23,6 +23,12 @@ All four layers run in userland. block service. - A partition has one holder at a time, every request is bounded to it before the device sees it, and a flush answers for its own writer's writes. +- A driver answers every request a session publishes, or ends the session, + within one bound the block protocol states, counted from the request's + publication: the wait to be taken from the session is inside it, as are a + device's silence before the driver takes a command back and taking it back. + A driver shows that its worst case fits under the bound. A client that has + waited longer ends the session; it never waits on a driver's word alone. - Caching belongs to file servers, never to the block layer. - Partition editing and storage initialisation are userland operations shared by disk tools and the installer. Editing the table, reading payloads and diff --git a/issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md b/issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md deleted file mode 100644 index d81da843de4..00000000000 --- a/issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -status: open -kind: defect -severity: [hang] -opened: 2026-10-13 ---- - -# A block session waits forever on a server past its submission ring - -A diskserver client trusts its server's cursor on the submission ring. Where -the server's consumer index is past what the client published, -`Session::pump` (`system/diskserver/src/session.rs`) meets the transport's -`Violation::HeadPastTail` from `space` and only stops pushing, on the comment -that "the next read of the connection is where that is acted on". Nothing -acts on it: `drain` reads only the completion ring, so with no completion to -refuse `wait` blocks in `poller.wait(1, u64::MAX, ..)` until the server hangs -up. A server that keeps its connection and writes that index holds every -caller of the session for good: an unbounded wait on an untrusted peer, where -the producer's violation should end the session as `drain`'s do. - -**Exit**: a submission-ring violation ends the session, so the caller's call -is refused; held by a host test of the client's ring half over a page whose -server cursor is past what the client published, red with that end removed. - -Owner: the orchestrator. diff --git a/issues/a-read-or-write-answered-lost-is-never-answered.md b/issues/a-read-or-write-answered-lost-is-never-answered.md deleted file mode 100644 index bac9183f18e..00000000000 --- a/issues/a-read-or-write-answered-lost-is-never-answered.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -status: assigned -kind: defect -severity: [hang] -opened: 2026-09-27 ---- - -# A read or write answered Lost is never answered - -`Client::complete` (`lib/blockring/src/client.rs`) takes a completion's tag -off the wire before it looks at the status. A user read or write answered -`Status::Lost` is then refused as `Violation::Entry`, and the session ends; but -the tag is no longer on the wire, so `Client::session_ended` does not answer it -`Refused`, and nothing ever answers its ticket. blockd's client -(`userland/blockd/src/session.rs`) keeps that ticket in `pending` for good, -across every reconnect. That breaks the module's own "every request asked for -is answered exactly once". Only a server that breaks the protocol writes that -completion; blockd does not. - -Held by the orchestrator. - -**Exit condition.** A completion the client refuses leaves its tag answered by -the session's end — refused before the tag is taken off the wire, or answered -where it is refused — and a model or unit test in which a server answers a -write `Lost` goes red without the fix and green with it. diff --git a/issues/a-usb-disk-session-can-wait-past-the-block-silence-bound-behind-other-sessions.md b/issues/a-usb-disk-session-can-wait-past-the-block-silence-bound-behind-other-sessions.md new file mode 100644 index 00000000000..12f7e0271fc --- /dev/null +++ b/issues/a-usb-disk-session-can-wait-past-the-block-silence-bound-behind-other-sessions.md @@ -0,0 +1,53 @@ +--- +status: open +kind: defect +opened: 2026-11-11 +severity: [data-loss] +--- + +# A USB disk's session can wait past the block silence bound behind other sessions + +**Owner.** The storage track: usbd's mass-storage driver and the block +protocol. + +The block protocol bounds a server's silence: while it owes a session an +answer it posts a completion, or ends the session, within +`toyos_blockring::SILENCE_BOUND_NS`, and a client ends a session silent +longer. The NVMe driver shows that its figure from a command's submission +fits (`system/nvme/src/nvme.rs`, `ANSWERED_WITHIN`), which is not yet the +whole path (below). usbd cannot as it is: + +- One command at a time per device (Bulk-Only Transport 1.0 §5.1), from a + queue of up to `QUEUE` (32) commands shared by every session on the device + (`system/usbd/xhci/src/msc.rs`). Each of a command's transfers may take + `COMMAND_NS` (10 s) and still succeed, so one session's request can wait + behind other sessions' commands for about 32 × 30 s, 16 minutes, with no + failure and no completion of its own. +- A break climbs the recovery ladder (`system/usbd/xhci/src/ladder.rs`): class + reset, port reset and enumeration again, the last reset. Its steps add their + own bounds, whose sum is not derived. + +A client that ends such a session reconnects to the same running driver, +which refuses the open `Held` while it still holds the request, and the +client's disk is gone for its life (`system/diskserver/src/disk.rs`). The +acknowledged writes the client keeps that no flush has covered are then +reported not durable by its next flush: accepted data later reported lost. + +The NVMe driver's figure, `ANSWERED_WITHIN`, equals the bound with no slack and +counts from a command's submission to the device. It leaves out: + +- the wait to be pulled: a session's request waits behind other sessions' + requests whenever the device is full (`toyos_blockserve::Sessions::pull` + takes the sessions in order), and behind a reset, during which nothing is + pulled; +- the serve loop's lag between a command reaching `COMMAND_SILENCE` and the + reset; +- the reset's own register and page work; +- posting the reset's answers and ringing the sessions after it; +- the client's clock, which starts just before it publishes. + +**Exit.** Each block driver's worst case from a request's publication to its +completion, the wait behind other sessions included, is shown to fit under +`SILENCE_BOUND_NS`, with a host test of usbd's; or the protocol's bound is +redefined so a session's silence does not count other sessions' work, and +both drivers show they keep it. diff --git a/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md b/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md deleted file mode 100644 index 7ff575c5eb5..00000000000 --- a/issues/nothing-reopens-a-diskserver-session-over-a-used-page.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -status: open -kind: tooling -opened: 2026-10-02 ---- - -# Nothing reopens a diskserver session over a used page - -A nightly went red on `blockd_io: FAIL /AFTER.BIN after the restart: Io` (run -36753172688, guest 7). diskserver answered an open and only then made its ends of -the session page. A client that reconnects sends the page its last server -wrote, and one that looks before the new server's ends are made hears the dead -session's completion. The server's half is a type now: `wire::Opened::over` -makes a server's ends and its answer together, and diskserver sends the answer -it returns. - -The client's half is read off the code and not run. `Session::pump`, `drain` -and `wait` (`system/diskserver/src/session.rs`) hold a `Region`, a -`Connection` and a `Poller`, so no host reaches them. A page whose completion -ring holds the dead session's answer ends the session only through `drain`'s -refusal of a completion for no ticket in flight (`Violation::Tag`, through -`client.complete`); `disk::Served` then reconnects and asks again, and answers -its caller `Gone` once a reconnect fails or the request has been refused across -more than `MAX_RECONNECTS` ends, which a file server's caller hears as `Io`. That nothing else ends it is -`issues/a-block-session-waits-forever-on-a-server-past-its-submission-ring.md`. - -**Exit**: a host test reopens a client's session over a page a killed server -left its answer on, and the session ends and the caller's call is `Io`, by -`drain`'s `Tag` refusal: red with that refusal removed, and green with it. - -Owner: the orchestrator. diff --git a/issues/smaller-data-rows-trail-main-by-up-to-half.md b/issues/smaller-data-rows-trail-main-by-up-to-half.md index 563d0e18acd..5976e238adb 100644 --- a/issues/smaller-data-rows-trail-main-by-up-to-half.md +++ b/issues/smaller-data-rows-trail-main-by-up-to-half.md @@ -41,5 +41,14 @@ The larger gaps have records of their own Evidence: #1152's evidence, `storage-integration/r10-known-gaps` (`osbench-a1f07627e/` and `osbench-table-a1f07627e.md`). +- **Owed on metal:** both benches, `osbench_disk`'s `fs.disk.*` rows in one + A/B of main before and after #1236, which reads a DATA listing's inodes in + one pass (`issues/a-data-directory-listing-is-read-whole-for-every-readdir-request.md`). + On the AMD bench, unmeasured, main after must show `readdir.1000` below main + before on every boot. On both, no other row may resolve against main after: + in #1236's T14 A/B `open_close` and `stat` did (1.8 and 8 %), though the + code on their path is the same but for its addresses, so a second A/B is + where they recur or not. + **Exit.** Each row within 1.1 times main's on both benches, or its cause shown to be inherent. diff --git a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md index 5c6fe62371b..cc6a53ec524 100644 --- a/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md +++ b/issues/the-kernel-is-small-interrupts-post-and-threads-wait.md @@ -72,17 +72,14 @@ Beside that order, each waiting on something outside it: `what_a_flush_calls_durable_is_on_the_medium` and `what_was_on_the_wire_at_the_end_is_refused_and_what_was_not_waits` for the transport, for diskserver's part by `wire::Opened::over`, whose type - makes a server's ends before its answer, and by the client test - `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` owes, a - session reopened over the page a killed server left, by fileserver's + makes a server's ends before its answer, and by a client that opens each + session over a region no server has held, by fileserver's `a_directory_rename_is_whole_under_one_name_wherever_the_server_is_killed` for DATA, and for LOG by the FAT test step 10 owes. **Partly done** (#525, #536, #971): NVMe and the stick are driven from userland, and the kernel's NVMe driver, USB storage, block layer and refusal chain are gone; the `nvme` driver and usbd offer their disks to diskserver, the block - service. **Left**: the page cache, with step 9's file half, and the client - test `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` - owes. + service. **Left**: the page cache, with step 9's file half. 4. **Where the data filesystem lives** — owner ruling, 2026-09-26, from the roasted proposal: userland file servers, the VFS a client library, and a program's view a set of directory capabilities the supervisor hands it, @@ -257,9 +254,7 @@ Beside that order, each waiting on something outside it: same under a write to the NVMe. #971's first metal boots came back with no `logkeeper` file on the stick, which every metal row reads; QEMU's stick boot (`the_log_lands_on_the_boot_stick`) has - it. And the client test - `issues/nothing-reopens-a-diskserver-session-over-a-used-page.md` - owes, which `/log`'s evidence names. + it. 5. **USB by userland**, with discovery and recovery written once as straight-line code: step 10, and this exit. **Exit**: step 10's, and no diff --git a/issues/the-nvme-driver-resets-a-drive-that-is-slow-and-healthy.md b/issues/the-nvme-driver-resets-a-drive-that-is-slow-and-healthy.md index f1952c28b8a..e4ce49d6a11 100644 --- a/issues/the-nvme-driver-resets-a-drive-that-is-slow-and-healthy.md +++ b/issues/the-nvme-driver-resets-a-drive-that-is-slow-and-healthy.md @@ -7,9 +7,12 @@ opened: 2026-09-26 # The NVMe driver resets a drive that is slow and healthy The NVMe driver reclaims a command the controller has not answered in -`COMMAND_SILENCE`, ten seconds (`system/nvme/src/main.rs`), by resetting the -controller. NVMe bounds no command's time, and a Flush of a large volatile -cache is the slowest thing a healthy drive does: a drive still flushing at ten +`COMMAND_SILENCE`, ten seconds (`system/nvme/src/nvme.rs`), by resetting the +controller. The same ten seconds bound each admin command of the reset, and +with the reset they set the block protocol's silence bound +(`toyos_blockring::SILENCE_BOUND_NS`), which a longer one must raise. NVMe +bounds no command's time, and a Flush of a large volatile cache is the +slowest thing a healthy drive does: a drive still flushing at ten seconds is reset, every command it held is answered `Device`, and a flush retried past `toyos_blockring::client::MAX_ATTEMPTS` reaches its caller as `Device` — a loss reported for a drive that lost nothing. @@ -22,3 +25,15 @@ size and write rate, or a measured worst Flush on the T14's NVMe and on QEMU), or a silent command is first aborted (NVMe Abort, §5.1) before the controller is reset; and a guest test holds a Flush past the old bound without the session hearing `Device`. + +Every admin command, at bring-up and in a reset, is held to the same ten +seconds since #1235, where it was `CAP.TO`. The T14 brought its controller up +under it at #1235's head (a `metalcase` boot: up in 2 ms, APST on, DATA served +off the NVMe). + +- **Owed on metal:** the AMD bench, the `microcode` and `machine_reboot` rows + (the `metalcase` image) in one boot of main after #1235. Main must pass both + rows, and its log must show the nvme driver's `up:` line with no `NOT + SERVING` and no `was not answered in`, `fileserver: Data serving` off the + NVMe, and no session's server that broke the protocol, answered nothing in + the bound, or would not take the session back. diff --git a/lib/blockring/src/client.rs b/lib/blockring/src/client.rs index bf8c5012c79..f95d15faa79 100644 --- a/lib/blockring/src/client.rs +++ b/lib/blockring/src/client.rs @@ -5,7 +5,10 @@ //! **Every request asked for is answered exactly once**, by [`Client::complete`] //! or by [`Client::session_ended`], and never twice: a completion for a tag //! that is not on the wire is the server breaking the session -//! ([`Violation::Tag`]), not a second answer. +//! ([`Violation::Tag`]), not a second answer. So is a word no server says of +//! the request it names ([`Violation::Entry`]): [`Status::Lost`] of anything +//! but a flush, or [`Status::ReadOnly`] of anything but a caller's write. Its +//! request stays on the wire, for the session's end to answer. //! //! **The arena holds only what is on the wire.** A read or a write is given //! its arena blocks as it goes out ([`Client::next_request`]'s `place`) and @@ -40,7 +43,15 @@ //! **What was on the wire when a session ended is answered //! [`Outcome::Refused`]**: a write it took may or may not be on the disk, and //! the caller is told so rather than told either. What had not yet been put on -//! the wire waits for the next session, unchanged. +//! the wire waits for the next session, unchanged. A session that ended with +//! nothing on the wire ended over the first request waiting to go, which is +//! treated as if it had been on it. +//! +//! **A session's end is a failed attempt**, as a device's refusal is, of each +//! flush and each write issued again that it took with it, so a server that +//! ends every session over the same one cannot hold it, or the requests +//! behind it, for good: past [`MAX_ATTEMPTS`] the flush is answered +//! [`Outcome::Device`] and the write is given up. //! //! **Requests in flight at once are unordered.** A caller must not have two in //! flight whose ranges overlap when either is a write, nor reuse arena blocks @@ -76,9 +87,10 @@ pub enum Placing<'a, P> { /// The caller's name for one thing it asked for. pub type Ticket = u64; -/// How many times a flush is asked again after a loss before its caller is -/// told the device would not make its writes durable, and how many times one -/// write is issued again before the same. +/// How many times a flush is asked again after it failed, on the device or +/// with its session, before its caller is told the device would not make its +/// writes durable, and how many times one write is issued again before the +/// same. pub const MAX_ATTEMPTS: u32 = 4; /// What became of one ticket. @@ -174,6 +186,17 @@ impl

Sent

{ Op::Flush => None, } } + + /// Whether a server says `status` of this request: [`Status::Lost`] only + /// of a flush, and [`Status::ReadOnly`] only of a caller's write, which a + /// write issued again is not: the grant took it once. + fn admits(&self, status: Status) -> bool { + match status { + Status::Ok | Status::Invalid | Status::Device => true, + Status::Lost => self.op == Op::Flush, + Status::ReadOnly => matches!((self.queued.kind, self.op), (Kind::User(_), Op::Write { .. })), + } + } } /// The client's own flush. @@ -316,17 +339,18 @@ impl Client { Some(Request { op, tag }) } - /// What the server answered. + /// What the server answered. A completion refused answers nothing: the + /// session is over, and [`Self::session_ended`] answers what it named. pub fn complete(&mut self, completion: Completion) -> Result<(), Violation> { - let sent = self.wire.answer(completion.tag)?; + let sent = self.wire.answer_admitted(completion.tag, |sent| sent.admits(completion.status))?; // Answered, whatever the word: its arena blocks are free once a read's // data has been taken, which the caller does before it releases them. self.released.extend(sent.run()); let Sent { queued, op, covers, losses } = sent; match (queued.kind, completion.status) { (Kind::User(ticket), Status::Ok) => { - match (queued.ask, op) { - (Ask::Write { lba, blocks, payload }, Op::Write { .. }) => { + match queued.ask { + Ask::Write { lba, blocks, payload } => { let seq = self.bump(); self.kept += u64::from(blocks); let acked = Acked { lba, blocks, payload, first: seq, seq, attempts: 0 }; @@ -340,8 +364,8 @@ impl Client { self.acked.push_back(acked); } } - (Ask::Read { .. }, Op::Read { .. }) => {} - _ => return Err(Violation::Entry), + Ask::Read { .. } => {} + Ask::Flush => unreachable!("blockring: a caller's flush is not sent as its read or write"), } self.answers.push_back((ticket, Outcome::Done)); } @@ -349,18 +373,14 @@ impl Client { let outcome = match status { Status::Invalid => Outcome::Invalid, Status::Device => Outcome::Device, - Status::ReadOnly if matches!(op, Op::Write { .. }) => Outcome::ReadOnly, - // A read or a write answered `Lost`, or a read answered - // `ReadOnly`, is a server that does not know which op it - // was. - Status::ReadOnly | Status::Lost | Status::Ok => return Err(Violation::Entry), + Status::ReadOnly => Outcome::ReadOnly, + Status::Lost | Status::Ok => unreachable!("blockring: {status:?} admitted for a caller's {op:?}"), }; self.answers.push_back((ticket, outcome)); } - // A write the session took once, refused by the same grant on its - // way again, or a flush refused as a write: a server that does - // not know what it granted. - (Kind::Reissue { .. } | Kind::Flush(_) | Kind::Pushback, Status::ReadOnly) => return Err(Violation::Entry), + (Kind::Reissue { .. } | Kind::Flush(_) | Kind::Pushback, Status::ReadOnly) => { + unreachable!("blockring: ReadOnly admitted for {:?}", queued.kind) + } (Kind::Reissue { .. }, Status::Ok) => { let seq = self.bump(); let acked = queued.acked(seq); @@ -372,19 +392,7 @@ impl Client { self.insert_acked(acked); } } - (Kind::Reissue { .. }, _) => { - let mut acked = queued.acked(0); - acked.attempts += 1; - if acked.attempts > MAX_ATTEMPTS { - // The device will not take the only copy there is: every - // flush waiting is told so, and the write is gone. - self.kept -= u64::from(acked.blocks); - self.gave_up = true; - self.fail_waiting_flushes(); - } else { - self.requeue_reissue(acked); - } - } + (Kind::Reissue { .. }, _) => self.reissue_failed(queued), // A flush that succeeded after a loss it was sent before covers // writes that are being issued again: it is asked again, after them. (kind @ (Kind::Flush(_) | Kind::Pushback), Status::Ok) if losses != self.losses => self.queue_flush(kind), @@ -402,55 +410,43 @@ impl Client { } (Kind::Flush(ticket), _) => { self.lost(); - let attempts = self.attempts.entry(ticket).or_insert(0); - *attempts += 1; - if *attempts > MAX_ATTEMPTS { - self.attempts.remove(&ticket); - self.answers.push_back((ticket, Outcome::Device)); - } else { - self.queue_flush(Kind::Flush(ticket)); - } + self.flush_failed(ticket); } (Kind::Pushback, _) => { self.lost(); - // A caller's flush that covered the writes since has made it - // moot: nothing waits on it. - if let Pushback::Asked(failed) = self.pushback { - if failed + 1 > MAX_ATTEMPTS { - self.pushback = Pushback::Failed; - } else { - self.pushback = Pushback::Asked(failed + 1); - self.queue_flush(Kind::Pushback); - } - } + self.pushback_failed(); } } Ok(()) } - /// The session is over: the server hung up, broke the protocol, or died. + /// The session is over: the server hung up, broke the protocol, went + /// silent, or died. pub fn session_ended(&mut self) { self.up = false; - let mut wire = Vec::new(); - self.wire.end(|_, sent| wire.push(sent)); - let mut flushes = Vec::new(); - for sent in wire { - self.released.extend(sent.run()); - match sent.queued.kind { + let mut taken = Vec::new(); + self.wire.end(|_, sent| taken.push(sent)); + self.released.extend(taken.iter().filter_map(Sent::run)); + let mut taken: Vec> = taken.into_iter().map(|sent| sent.queued).collect(); + if taken.is_empty() { + taken.extend(self.outbox.pop_front()); + } + // Flushes first, so a write given up below answers them at once. + taken.sort_by_key(|q| matches!(q.kind, Kind::Reissue { .. })); + self.lost(); + for queued in taken { + match queued.kind { Kind::User(ticket) => { #[cfg(not(feature = "mutate-session-end-forgets"))] self.answers.push_back((ticket, Outcome::Refused)); #[cfg(feature = "mutate-session-end-forgets")] let _ = ticket; } - Kind::Reissue { .. } => self.requeue_reissue(sent.queued.acked(0)), - kind @ (Kind::Flush(_) | Kind::Pushback) => flushes.push(kind), + Kind::Reissue { .. } => self.reissue_failed(queued), + Kind::Flush(ticket) => self.flush_failed(ticket), + Kind::Pushback => self.pushback_failed(), } } - self.lost(); - for kind in flushes { - self.queue_flush(kind); - } } /// A session is open; whatever waited goes out, writes being issued @@ -474,6 +470,17 @@ impl Client { self.released.drain(..) } + /// Something asked for is waiting or on the wire: a completion is owed + /// before the caller hears of it. + pub fn owed(&self) -> bool { + !self.outbox.is_empty() || self.wire.values().next().is_some() + } + + /// Answers are decided that [`Self::take_answers`] has not taken. + pub fn answered(&self) -> bool { + !self.answers.is_empty() + } + /// Nothing is waiting, on the wire, or held for a flush. pub fn quiet(&self) -> bool { self.outbox.is_empty() && self.wire.values().next().is_none() && self.acked.is_empty() @@ -544,6 +551,49 @@ impl Client { } } + /// A write issued again failed: it goes again, or past [`MAX_ATTEMPTS`] + /// the device will not take the only copy there is, every flush waiting + /// is told so, and the write is gone. + fn reissue_failed(&mut self, queued: Queued

) { + let mut acked = queued.acked(0); + acked.attempts += 1; + if acked.attempts > MAX_ATTEMPTS { + self.kept -= u64::from(acked.blocks); + self.gave_up = true; + self.fail_waiting_flushes(); + } else { + self.requeue_reissue(acked); + } + } + + /// An attempt at the caller's flush `ticket` failed: it is asked again, or + /// past [`MAX_ATTEMPTS`] answered [`Outcome::Device`]. + fn flush_failed(&mut self, ticket: Ticket) { + let attempts = self.attempts.entry(ticket).or_insert(0); + *attempts += 1; + if *attempts > MAX_ATTEMPTS { + self.attempts.remove(&ticket); + self.answers.push_back((ticket, Outcome::Device)); + } else { + self.queue_flush(Kind::Flush(ticket)); + } + } + + /// An attempt at the client's own flush failed: it is asked again, or past + /// [`MAX_ATTEMPTS`] writes that meet the limit are refused. A caller's + /// flush that covered the writes since has made it moot: nothing waits on + /// it. + fn pushback_failed(&mut self) { + if let Pushback::Asked(failed) = self.pushback { + if failed + 1 > MAX_ATTEMPTS { + self.pushback = Pushback::Failed; + } else { + self.pushback = Pushback::Asked(failed + 1); + self.queue_flush(Kind::Pushback); + } + } + } + /// Every caller's flush waiting to be asked again is answered: a write it /// covers is gone. fn fail_waiting_flushes(&mut self) { @@ -746,11 +796,74 @@ mod tests { assert_eq!(client.take_answers().collect::>(), [(1, Outcome::ReadOnly)]); assert_eq!(client.take_released().collect::>(), [run_of(1, 1)]); assert_eq!(client.kept(), 0); - for ask in [Ask::Read { lba: 0, blocks: 1 }, Ask::Flush] { - client.submit(2, ask); + for (ticket, ask) in [(2, Ask::Read { lba: 0, blocks: 1 }), (3, Ask::Flush)] { + client.submit(ticket, ask); let r = next(&mut client).unwrap(); assert_eq!(client.complete(Completion { tag: r.tag, status: Status::ReadOnly }), Err(Violation::Entry)); } + assert_eq!(client.take_answers().count(), 0); + client.session_ended(); + assert_eq!(client.take_answers().collect::>(), [(2, Outcome::Refused)], "the flush is asked again"); + } + + /// A write issued again and answered `ReadOnly` or `Lost`, words no + /// server says of it, is refused on the wire, so the session's end keeps + /// it to issue again: it is the only copy. + #[test] + fn a_reissue_answered_a_word_no_server_says_of_it_is_kept() { + let mut client = up(ROOMY); + client.submit(1, write(1, 0, 1)); + let w = next(&mut client).unwrap(); + answer(&mut client, w, Status::Ok); + client.session_ended(); + client.session_started(); + for status in [Status::ReadOnly, Status::Lost] { + let again = next(&mut client).unwrap(); + assert_eq!(again.op, Op::Write { run: run_of(1, 1), lba: 0 }); + assert_eq!(client.complete(Completion { tag: again.tag, status }), Err(Violation::Entry), "{status:?}"); + client.session_ended(); + client.session_started(); + } + assert_eq!(next(&mut client).unwrap().op, Op::Write { run: run_of(1, 1), lba: 0 }); + assert_eq!(client.kept(), 1); + } + + /// A caller's flush on the wire at every session's end: each end is an + /// attempt, and past [`MAX_ATTEMPTS`] it is answered `Device`. + #[test] + fn a_flush_every_session_ends_over_is_answered_device() { + let mut client = up(ROOMY); + client.submit(1, Ask::Flush); + for _ in 0..=MAX_ATTEMPTS { + assert_eq!(client.take_answers().count(), 0); + assert_eq!(next(&mut client).unwrap().op, Op::Flush); + client.session_ended(); + client.session_started(); + } + assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Device)]); + assert_eq!(next(&mut client), None); + } + + /// A write issued again and on the wire at every session's end is given + /// up past [`MAX_ATTEMPTS`], and a flush waiting on it is told so. + #[test] + fn a_reissue_every_session_ends_over_is_given_up() { + let mut client = up(ROOMY); + client.submit(1, write(1, 0, 1)); + let w = next(&mut client).unwrap(); + answer(&mut client, w, Status::Ok); + assert_eq!(client.take_answers().collect::>(), [(1, Outcome::Done)]); + client.session_ended(); + client.session_started(); + client.submit(2, Ask::Flush); + for _ in 0..=MAX_ATTEMPTS { + assert_eq!(next(&mut client).unwrap().op, Op::Write { run: run_of(1, 1), lba: 0 }); + assert_eq!(next(&mut client), None, "the flush waits for it"); + client.session_ended(); + client.session_started(); + } + assert_eq!(client.kept(), 0); + assert_eq!(client.take_answers().collect::>(), [(2, Outcome::Device)]); } #[test] diff --git a/lib/blockring/src/ends.rs b/lib/blockring/src/ends.rs new file mode 100644 index 00000000000..82cc9f73731 --- /dev/null +++ b/lib/blockring/src/ends.rs @@ -0,0 +1,462 @@ +//! A client's two ends of a session page: its requests onto the request ring +//! and the server's completions off the other, and what the client does next. +//! +//! **What the server writes ends the session or is a completion the client +//! takes**: a cursor no server writes, words that are no completion, and a +//! completion [`Client::complete`] refuses are each a [`Violation`]. **So does +//! its silence**, once the client has waited [`SILENCE_BOUND_NS`], the bound +//! every server keeps, for a completion. Either way [`ClientEnds::exchange`] +//! answers [`Next::End`], the caller ends the session, and +//! [`Client::session_ended`] answers what was on the wire. +//! +//! **A wait is only ever for the server**: [`Next::Wait`] is answered only +//! once every request that can go has gone and no answer is decided. + +use toyos::transport::{Run, Violation, Word}; + +use crate::client::{Client, Placing}; +use crate::entry::Completion; +use crate::layout::{self, ClientRings, RING_WORDS}; +use crate::SILENCE_BOUND_NS; + +/// What a client does after an exchange with its server. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Next { + /// End the session. + End(Ending), + /// Answers are decided, for [`Client::take_answers`]. + Answered, + /// Nothing is decided; the server has this long to post a completion. + Wait(u64), + /// Nothing is owed and nothing decided. + Idle, +} + +/// Why a session is ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Ending { + Violation(Violation), + /// No completion in [`SILENCE_BOUND_NS`] while one was owed. + Silence, +} + +/// An exchange's outcome, and whether it published requests the server's +/// doorbell is owed for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Exchange { + pub next: Next, + pub published: bool, +} + +pub struct ClientEnds { + rings: ClientRings, + /// When the client came to be owed a completion, or last took one while + /// owed more; `None` while it is owed none. + owed_since: Option, +} + +impl ClientEnds { + /// The client's ends of `page`, every word it owns set to 0: done before + /// the page is sent to a server. + pub fn new(page: &[W; RING_WORDS]) -> Self { + Self { rings: layout::client(page), owed_since: None } + } + + /// At `now`: send what can go, take every completion posted, send what + /// those let go (a flush waiting on writes going out again), and say what + /// next. Requests go out through `place`, as [`Client::next_request`]'s. + pub fn exchange( + &mut self, + page: &[W; RING_WORDS], + client: &mut Client

, + now: u64, + mut place: impl FnMut(Placing<'_, P>) -> Option, + ) -> Exchange { + let mut published = false; + let mut trade = || -> Result<(), Violation> { + published |= self.send(page, client, now, &mut place)?; + self.receive(page, client, now)?; + published |= self.send(page, client, now, &mut place)?; + Ok(()) + }; + let next = match (trade(), self.patience(now)) { + (Err(violation), _) => Next::End(Ending::Violation(violation)), + (Ok(()), Some(0)) => Next::End(Ending::Silence), + (Ok(()), _) if client.answered() => Next::Answered, + (Ok(()), Some(left)) => Next::Wait(left), + (Ok(()), None) => Next::Idle, + }; + Exchange { next, published } + } + + /// Put every request `client` will send on the ring, at `now`; `true` if + /// any went, which the server's doorbell is owed for. + fn send( + &mut self, + page: &[W; RING_WORDS], + client: &mut Client

, + now: u64, + mut place: impl FnMut(Placing<'_, P>) -> Option, + ) -> Result { + while self.rings.0.space(page)? > 0 { + let Some(request) = client.next_request(&mut place) else { break }; + let pushed = self.rings.0.push(page, request.encode()); + assert_eq!(pushed, Ok(true), "blockring: a request past the room just counted"); + } + self.owe(client, now, false); + Ok(self.rings.0.publish(page)) + } + + /// Give `client` every completion the server has posted, at `now`. Those + /// before a violation are taken. + pub fn receive(&mut self, page: &[W; RING_WORDS], client: &mut Client

, now: u64) -> Result<(), Violation> { + let mut taken = false; + let met = loop { + match self.rings.1.pop(page) { + Ok(Some(words)) => match Completion::decode(words).ok_or(Violation::Entry).and_then(|c| client.complete(c)) { + Ok(()) => taken = true, + Err(violation) => break Err(violation), + }, + Ok(None) => break Ok(()), + Err(violation) => break Err(violation), + } + }; + self.rings.1.release(page); + self.owe(client, now, taken); + met + } + + /// How long from `now` the server still has to post a completion; `None` + /// while the client is owed none, and `Some(0)` for a server to give up. + fn patience(&self, now: u64) -> Option { + self.owed_since.map(|since| SILENCE_BOUND_NS.saturating_sub(now.saturating_sub(since))) + } + + fn owe(&mut self, client: &Client

, now: u64, taken: bool) { + self.owed_since = match self.owed_since { + _ if !client.owed() => None, + Some(since) if !taken => Some(since), + _ => Some(now), + }; + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicU32, Ordering}; + + use super::*; + use crate::client::{Ask, Outcome, Ticket, MAX_ATTEMPTS}; + use crate::entry::{Op, Request, Status}; + use crate::layout::{ServerRings, ARENA, CQ_TAIL, DEPTH, SQ_HEAD}; + + type Page = [AtomicU32; RING_WORDS]; + + /// A client, its ends of a page, and the server's ends for a test to + /// answer through, or to write past. + struct Session { + page: Page, + client: Client, + ends: ClientEnds, + server: ServerRings, + } + + fn place(p: Placing<'_, Ticket>) -> Option { + let (ticket, blocks) = match p { + Placing::Read { ticket, blocks } => (ticket, blocks), + Placing::Write { payload, blocks } => (*payload, blocks), + }; + ARENA.run(ticket as u32, blocks) + } + + impl Session { + fn open() -> Self { + Self::keeping(1 << 20) + } + + /// A client that keeps at most `limit` blocks of acknowledged writes. + fn keeping(limit: u64) -> Self { + let page: Page = core::array::from_fn(|_| AtomicU32::new(0)); + let ends = ClientEnds::new(&page); + let server = layout::server(&page); + let mut client = Client::new(limit); + client.session_started(); + Self { page, client, ends, server } + } + + /// The session ended, and the next opened over a fresh page: what the + /// caller was told at the end. + fn reopened(&mut self) -> Vec<(Ticket, Outcome)> { + let told = self.ended(); + self.page = core::array::from_fn(|_| AtomicU32::new(0)); + self.ends = ClientEnds::new(&self.page); + self.server = layout::server(&self.page); + self.client.session_started(); + told + } + + fn exchange(&mut self, now: u64) -> Next { + self.ends.exchange(&self.page, &mut self.client, now, place).next + } + + fn send(&mut self, now: u64) -> Result { + self.ends.send(&self.page, &mut self.client, now, place) + } + + fn receive(&mut self, now: u64) -> Result<(), Violation> { + self.ends.receive(&self.page, &mut self.client, now) + } + + /// The requests the server finds published. + fn taken(&mut self) -> Vec { + let mut taken = Vec::new(); + while let Some(words) = self.server.0.pop(&self.page).unwrap() { + taken.push(Request::decode(words, u64::MAX).unwrap()); + } + self.server.0.release(&self.page); + taken + } + + fn post(&mut self, tag: u32, status: Status) { + assert_eq!(self.server.1.push(&self.page, Completion { tag, status }.encode()), Ok(true)); + self.server.1.publish(&self.page); + } + + /// What the caller is told once the session is ended. + fn ended(&mut self) -> Vec<(Ticket, Outcome)> { + self.client.session_ended(); + self.client.take_answers().collect() + } + + /// Every request the server finds published answered `status`. + fn answer_all(&mut self, status: Status) -> Vec { + let taken = self.taken(); + for request in &taken { + self.post(request.tag, status); + } + taken + } + } + + #[test] + fn requests_go_out_and_their_completions_come_back() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + s.client.submit(2, Ask::Flush); + assert_eq!(s.send(0), Ok(true)); + assert_eq!(s.send(0), Ok(false), "nothing new, no doorbell"); + for request in s.taken() { + s.post(request.tag, Status::Ok); + } + assert_eq!(s.receive(0), Ok(())); + assert_eq!(s.client.take_answers().collect::>(), [(1, Outcome::Done), (2, Outcome::Durable)]); + assert_eq!(s.ends.patience(0), None, "owed nothing"); + } + + /// The client's own flush fails past its attempts while a caller's write + /// waits behind the limit, and the writes it kept go out again. When the + /// server answers those, the send after it answers the waiting write + /// `Device` unsent, and nothing is owed: the exchange answers `Answered`, + /// not a wait with an answer decided and untaken. + #[test] + fn an_answer_the_last_send_decides_is_never_waited_past() { + let mut s = Session::keeping(1); + s.client.submit(1, Ask::Write { lba: 0, blocks: 1, payload: 1 }); + assert_eq!(s.exchange(0), Next::Wait(SILENCE_BOUND_NS)); + s.answer_all(Status::Ok); + assert_eq!(s.exchange(0), Next::Answered); + assert_eq!(s.client.take_answers().collect::>(), [(1, Outcome::Done)]); + s.client.submit(2, Ask::Write { lba: 4, blocks: 1, payload: 2 }); + assert_eq!(s.exchange(0), Next::Wait(SILENCE_BOUND_NS), "the client's own flush goes, and the write waits"); + for attempt in 0..=MAX_ATTEMPTS { + assert_eq!(s.answer_all(Status::Device).len(), 1, "the client's own flush"); + assert!(matches!(s.exchange(0), Next::Wait(_)), "the kept write goes out again"); + assert_eq!(s.answer_all(Status::Ok).len(), 1, "the kept write"); + let next = s.exchange(0); + if attempt < MAX_ATTEMPTS { + assert!(matches!(next, Next::Wait(_)), "the client's own flush goes again"); + } else { + assert_eq!(next, Next::Answered, "the send after the last completion answered the waiting write"); + } + } + assert_eq!(s.client.take_answers().collect::>(), [(2, Outcome::Device)]); + assert_eq!(s.exchange(0), Next::Idle); + } + + /// A server whose cursor on the request ring is past what the client + /// published: the send that meets it says so, and the session's end + /// answers the request that was out. + #[test] + fn a_server_past_the_request_ring_ends_the_session() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + assert_eq!(s.send(0), Ok(true)); + s.page[SQ_HEAD].store(2, Ordering::Release); + s.client.submit(2, Ask::Read { lba: 1, blocks: 1 }); + assert_eq!(s.send(0), Err(Violation::HeadPastTail)); + assert_eq!(s.ended(), [(1, Outcome::Refused)]); + } + + /// The same before anything went: the session ended over the first + /// request waiting, which its end answers; the one behind it waits. + #[test] + fn a_server_past_the_request_ring_before_anything_went_ends_over_what_waited() { + let mut s = Session::open(); + s.page[SQ_HEAD].store(1, Ordering::Release); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + s.client.submit(2, Ask::Read { lba: 1, blocks: 1 }); + assert_eq!(s.exchange(0), Next::End(Ending::Violation(Violation::HeadPastTail))); + assert_eq!(s.reopened(), [(1, Outcome::Refused)]); + assert_eq!(s.exchange(0), Next::Wait(SILENCE_BOUND_NS)); + assert_eq!(s.taken().iter().map(|r| r.op).collect::>(), [Op::Read { run: ARENA.run(2, 1).unwrap(), lba: 1 }]); + } + + /// A server that answers a write issued again with a word no server says + /// of it, at every session: each end counts against the write, which is + /// given up past its attempts, and the request behind it goes out. + #[test] + fn a_server_that_ends_every_session_over_one_write_holds_nothing_for_good() { + let mut s = Session::open(); + s.client.submit(1, Ask::Write { lba: 0, blocks: 1, payload: 1 }); + s.exchange(0); + s.answer_all(Status::Ok); + assert_eq!(s.exchange(0), Next::Answered); + assert_eq!(s.reopened(), [(1, Outcome::Done)]); + s.client.submit(2, Ask::Read { lba: 4, blocks: 1 }); + for _ in 0..=MAX_ATTEMPTS { + assert!(matches!(s.exchange(0), Next::Wait(_))); + let sent: Vec = s.answer_all(Status::Lost).iter().map(|r| r.op).collect(); + assert_eq!(sent, [Op::Write { run: ARENA.run(1, 1).unwrap(), lba: 0 }], "the write alone, again"); + assert_eq!(s.exchange(0), Next::End(Ending::Violation(Violation::Entry))); + assert_eq!(s.reopened(), []); + } + assert_eq!(s.client.kept(), 0, "given up"); + assert!(matches!(s.exchange(0), Next::Wait(_))); + assert_eq!(s.taken().iter().map(|r| r.op).collect::>(), [Op::Read { run: ARENA.run(2, 1).unwrap(), lba: 4 }]); + } + + #[test] + fn a_server_past_the_completion_ring_ends_the_session() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + assert_eq!(s.send(0), Ok(true)); + s.page[CQ_TAIL].store(DEPTH + 1, Ordering::Release); + assert_eq!(s.receive(0), Err(Violation::TailPastDepth)); + assert_eq!(s.ended(), [(1, Outcome::Refused)]); + } + + /// The first answer is the caller's; the second is refused and answers + /// nothing, and the request beside it is the session's end's. + #[test] + fn a_second_answer_for_one_request_ends_the_session() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + s.client.submit(2, Ask::Read { lba: 1, blocks: 1 }); + assert_eq!(s.send(0), Ok(true)); + let tag = s.taken()[0].tag; + s.post(tag, Status::Ok); + s.post(tag, Status::Ok); + assert_eq!(s.receive(0), Err(Violation::Tag)); + assert_eq!(s.client.take_answers().collect::>(), [(1, Outcome::Done)]); + assert_eq!(s.ended(), [(2, Outcome::Refused)]); + } + + #[test] + fn an_answer_for_a_request_never_asked_ends_the_session() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + assert_eq!(s.send(0), Ok(true)); + let tag = s.taken()[0].tag; + s.post(tag + 1, Status::Ok); + assert_eq!(s.receive(0), Err(Violation::Tag)); + assert_eq!(s.ended(), [(1, Outcome::Refused)]); + } + + /// Words that are no completion: a status no server writes. + #[test] + fn a_completion_that_decodes_to_nothing_ends_the_session() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + assert_eq!(s.send(0), Ok(true)); + let mut words = Completion { tag: s.taken()[0].tag, status: Status::Ok }.encode(); + words[1] = u32::MAX; + assert_eq!(s.server.1.push(&s.page, words), Ok(true)); + s.server.1.publish(&s.page); + assert_eq!(s.receive(0), Err(Violation::Entry)); + assert_eq!(s.ended(), [(1, Outcome::Refused)]); + } + + /// `Lost` is a flush's word. A write answered it is refused with the + /// write still on the wire, so the session's end answers its caller. + #[test] + fn a_write_answered_lost_is_answered_by_the_sessions_end() { + let mut s = Session::open(); + s.client.submit(1, Ask::Write { lba: 0, blocks: 1, payload: 1 }); + assert_eq!(s.send(0), Ok(true)); + let tag = s.taken()[0].tag; + s.post(tag, Status::Lost); + assert_eq!(s.receive(0), Err(Violation::Entry)); + assert_eq!(s.client.take_answers().count(), 0); + assert_eq!(s.ended(), [(1, Outcome::Refused)]); + assert_eq!(s.client.take_released().collect::>(), [ARENA.run(1, 1).unwrap()]); + } + + /// A server that takes a request and never answers it: its patience runs + /// from the send, and is gone at the bound. + #[test] + fn a_dropped_answer_runs_out_the_servers_patience() { + let mut s = Session::open(); + assert_eq!(s.ends.patience(5), None); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + assert_eq!(s.send(5), Ok(true)); + s.taken(); + assert_eq!(s.ends.patience(5), Some(SILENCE_BOUND_NS)); + assert_eq!(s.receive(9), Ok(())); + assert_eq!(s.send(9), Ok(false)); + assert_eq!(s.ends.patience(5 + SILENCE_BOUND_NS - 1), Some(1), "looking again buys it nothing"); + assert_eq!(s.ends.patience(5 + SILENCE_BOUND_NS), Some(0)); + assert_eq!(s.ended(), [(1, Outcome::Refused)]); + } + + /// A completion taken is the server at work: what it still owes is + /// counted from there. + #[test] + fn a_completion_renews_the_servers_patience() { + let mut s = Session::open(); + s.client.submit(1, Ask::Read { lba: 0, blocks: 1 }); + s.client.submit(2, Ask::Read { lba: 1, blocks: 1 }); + assert_eq!(s.send(0), Ok(true)); + let tag = s.taken()[0].tag; + s.post(tag, Status::Ok); + assert_eq!(s.receive(40), Ok(())); + assert_eq!(s.ends.patience(40), Some(SILENCE_BOUND_NS)); + } + + /// A server that answers and never gives a request entry back: the + /// requests that cannot go out are owed as those on the wire are. + #[test] + fn a_server_that_frees_no_request_entry_is_owed_for_what_cannot_go() { + let mut s = Session::open(); + for round in 0..2 { + for ticket in 0..Ticket::from(DEPTH) { + s.client.submit(ticket, Ask::Read { lba: ticket, blocks: 1 }); + } + assert_eq!(s.send(7), Ok(round == 0)); + if round == 1 { + break; + } + // Read without a release: the client's entries stay taken. + let mut tags = Vec::new(); + while let Some(words) = s.server.0.pop(&s.page).unwrap() { + tags.push(Request::decode(words, u64::MAX).unwrap().tag); + } + for tag in tags { + s.post(tag, Status::Ok); + } + assert_eq!(s.receive(7), Ok(())); + assert_eq!(s.ends.patience(7), None); + } + assert_eq!(s.client.on_the_wire().count(), 0); + assert_eq!(s.ends.patience(7 + SILENCE_BOUND_NS), Some(0)); + } +} diff --git a/lib/blockring/src/layout.rs b/lib/blockring/src/layout.rs index 97e65bd8c64..bce8455020b 100644 --- a/lib/blockring/src/layout.rs +++ b/lib/blockring/src/layout.rs @@ -53,10 +53,9 @@ pub type ClientRings = (Producer, Consumer, Producer); -/// The client's ends of a session page, every word it owns set to 0. Done -/// before the page is sent to a server, and again before it is sent to the -/// next one. -pub fn client(page: &[W; RING_WORDS]) -> ClientRings { +/// The client's ends of a session page, every word it owns set to 0. A +/// client's way to them is [`crate::ends::ClientEnds::new`]. +pub(crate) fn client(page: &[W; RING_WORDS]) -> ClientRings { (Producer::new(page, REQUESTS), Consumer::new(page, COMPLETIONS)) } diff --git a/lib/blockring/src/lib.rs b/lib/blockring/src/lib.rs index a8d94ed35e4..0bb1f88e471 100644 --- a/lib/blockring/src/lib.rs +++ b/lib/blockring/src/lib.rs @@ -8,7 +8,7 @@ //! data into and out of directly. Nothing on the page is a pointer and nothing on it //! is trusted by the end that did not write it: a consumer bounds every index //! and every field before it acts ([`entry::Request::decode`], the -//! transport's cursor bounds). +//! transport's cursor bounds, and for a client [`ends::ClientEnds`]). //! //! **A doorbell is an event each way**, rung after the entries it announces //! are published. The server makes both and hands the client its sides with @@ -29,6 +29,12 @@ //! **Requests in flight at once are unordered**, as on the device: a client //! that needs one to follow another waits for the first's answer. //! +//! **A server is never silent past [`SILENCE_BOUND_NS`] while it owes an +//! answer**: from a request it has not answered, or from its last completion, +//! it posts the next completion or ends the session within it. A client that +//! has waited longer ends the session ([`ends`]); a driver shows its own worst +//! case fits under the bound. +//! //! **One service, many disks.** The block service serves [`PORT`] to clients //! and [`disk::PORT`] to disk drivers, each of which offers its disks and is //! handed the sessions opened on them ([`disk`]); data moves between a @@ -46,6 +52,7 @@ extern crate alloc; pub mod client; pub mod disk; +pub mod ends; pub mod entry; pub mod layout; pub mod server; @@ -54,6 +61,13 @@ pub mod wire; #[cfg(test)] mod model; +/// The protocol's bound on a server's silence. It is set from the NVMe +/// driver's figure, counted from a command's submission to the device: the +/// command's silence before the driver resets the controller, and the reset. +/// What that figure leaves out, and usbd's worst case, which does not fit, are +/// `issues/a-usb-disk-session-can-wait-past-the-block-silence-bound-behind-other-sessions.md`. +pub const SILENCE_BOUND_NS: u64 = 365_000_000_000; + pub use entry::{Completion, Op, Request, Status}; pub use toyos::transport::Run; pub use layout::{BLOCK_BYTES, DEPTH, MAX_REQUEST_BLOCKS, SESSION_BYTES}; diff --git a/lib/blockring/src/model.rs b/lib/blockring/src/model.rs index 0709f14f4e6..6a7a165b3fa 100644 --- a/lib/blockring/src/model.rs +++ b/lib/blockring/src/model.rs @@ -22,9 +22,9 @@ //! meet a second completion for one tag ([`Law::Answers`]); //! - a flush answered durable left on the medium, for every block, the last //! write acknowledged before the flush was asked for, or one asked for after -//! it; and a flush ends answered durable unless the run spent more failures -//! than [`MAX_ATTEMPTS`], which is the fewest that can make the client give -//! a write or a flush up ([`Law::Durable`]). +//! it; and a flush ends answered durable unless the run's failures can cost +//! one flush or one write more attempts than [`MAX_ATTEMPTS`], the fewest +//! that make the client give it up ([`Law::Durable`], [`Failures::attempts`]). use alloc::collections::{BTreeMap, BTreeSet, VecDeque}; use alloc::format; @@ -58,8 +58,12 @@ pub struct Failures { } impl Failures { - fn total(self) -> u32 { - u32::from(self.resets) + u32::from(self.crashes) + u32::from(self.errors) + /// The most attempts at one flush or one write these can cost: a reset + /// two (the request it fails, and the loss the next flush is told of), a + /// server's death one (the request its session's end took), a device + /// error one. + fn attempts(self) -> u32 { + 2 * u32::from(self.resets) + u32::from(self.crashes) + u32::from(self.errors) } } @@ -264,7 +268,7 @@ pub fn explore(script: &[Step], failures: Failures) -> Explored { ends: 0, given_up: 0, filled: [false; 2], - may_give_up: failures.total() > MAX_ATTEMPTS, + may_give_up: failures.attempts() > MAX_ATTEMPTS, path: Vec::new(), }; dfs(&mut run, start(failures, LIMIT)); @@ -720,10 +724,13 @@ mod tests { /// given up in. const GIVE_UP: [Step; 2] = [Step::Write { block: 0, value: 1 }, Step::Flush]; - /// What it is held under: two deaths and resets, and a death followed by - /// more device errors than [`MAX_ATTEMPTS`] — what makes the client give - /// an acknowledged write up. - const GIVE_UP_BOUNDS: [Failures; 3] = [ + /// What it is held under: a reset and two deaths, failures that cost + /// exactly [`MAX_ATTEMPTS`] attempts, the boundary where the law forbids a + /// give-up; two deaths and resets, and a death followed by more device + /// errors than [`MAX_ATTEMPTS`] — what makes the client give an + /// acknowledged write up. + const GIVE_UP_BOUNDS: [Failures; 4] = [ + at_most(1, 2, 0), at_most(2, 2, 0), at_most(0, 1, MAX_ATTEMPTS as u8 + 1), at_most(1, 1, MAX_ATTEMPTS as u8), @@ -796,9 +803,9 @@ mod tests { assert_eq!(explored.filled, [true, true], "a ring never held its depth"); } - /// Nor is the give-up path out of its reach: past [`MAX_ATTEMPTS`] failures - /// some run ends with a flush answered the device's refusal, and the laws - /// above were held on it. + /// Nor is the give-up path out of its reach: with failures that cost more + /// than [`MAX_ATTEMPTS`] attempts some run ends with a flush answered the + /// device's refusal, and the laws above were held on it. #[test] fn the_model_reaches_a_write_given_up() { let explored = verdict(&GIVE_UP, at_most(0, 1, MAX_ATTEMPTS as u8 + 1)); diff --git a/lib/blockring/src/wire.rs b/lib/blockring/src/wire.rs index 1209e973416..a910f7312d2 100644 --- a/lib/blockring/src/wire.rs +++ b/lib/blockring/src/wire.rs @@ -13,8 +13,8 @@ //! and a connection with no grant reaches nothing. //! //! **The answer comes with the server's ends of the page.** A client looks at -//! the page the moment it hears, and one that opens the same region again -//! sends the cursors its last server left on it. So [`Opened::over`] makes a +//! the page the moment it hears, and a region it sends may carry the cursors +//! another server left on it. So [`Opened::over`] makes a //! server's ends and its answer together: nothing else makes the ends, and an //! answer is otherwise only read off the wire ([`Opened::decode`]). diff --git a/sdk/toyos/src/transport/inflight.rs b/sdk/toyos/src/transport/inflight.rs index 1f1042009fe..e19a8bac470 100644 --- a/sdk/toyos/src/transport/inflight.rs +++ b/sdk/toyos/src/transport/inflight.rs @@ -54,11 +54,21 @@ impl Inflight { /// What the peer's `tag` answers; a tag nothing is in flight under is a /// violation. pub fn answer(&mut self, tag: u32) -> Result { + self.answer_admitted(tag, |_| true) + } + + /// [`Self::answer`], for a reply `admits` judges by what it answers: one + /// it refuses is [`Violation::Entry`] and leaves the request in flight, + /// for [`Self::end`]. + pub fn answer_admitted(&mut self, tag: u32, admits: impl FnOnce(&T) -> bool) -> Result { let index = usize::try_from(tag & Self::INDEX_MASK).map_err(|_| Violation::Tag)?; let slot = self.slots.get_mut(index).ok_or(Violation::Tag)?; if tag.wrapping_shr(Self::INDEX_BITS) != slot.seq { return Err(Violation::Tag); } + if !admits(slot.value.as_ref().ok_or(Violation::Tag)?) { + return Err(Violation::Entry); + } slot.value.take().ok_or(Violation::Tag) } @@ -125,6 +135,16 @@ mod tests { assert!(inflight.insert("d").is_ok() && inflight.insert("e").is_ok(), "every slot is free again"); } + #[test] + fn a_reply_refused_leaves_its_request_in_flight() { + let mut inflight = Inflight::<&str, 2>::new(); + let a = inflight.insert("a").unwrap(); + assert_eq!(inflight.answer_admitted(a, |_| false), Err(Violation::Entry)); + let mut ended = Vec::new(); + inflight.end(|tag, value| ended.push((tag, value))); + assert_eq!(ended, [(a, "a")]); + } + /// A slot filled 2^16 times over still refuses its first tag: the index /// takes six bits of a 64-slot table's tag, not sixteen. #[test] diff --git a/system/diskserver/src/session.rs b/system/diskserver/src/session.rs index 99a3e7f8270..114506ce7e8 100644 --- a/system/diskserver/src/session.rs +++ b/system/diskserver/src/session.rs @@ -13,9 +13,22 @@ //! reads every completion the server posted before it hung up, answers what //! was still on the wire [`Outcome::Refused`], and says the session ended; the //! caller decides when to [`Session::reconnect`], which reaches the service -//! through the same name, opens the same partition over the same region, and -//! issues every acknowledged write no flush had covered again before anything -//! else. Nothing here restarts a service: whoever holds its claim does. +//! through the same name, opens the same partition, and issues every +//! acknowledged write no flush had covered again before anything else. +//! Nothing here restarts a service: whoever holds its claim does. +//! +//! **No wait is on the server's word alone.** A server that breaks the +//! protocol, or leaves the session unanswered past the bound every server +//! keeps ([`SILENCE_BOUND_NS`]), is ended as one that hung up is +//! (`toyos_blockring::ends`). That server is still running, and the +//! reconnect reaches it again. One that still holds a request of the ended +//! session refuses the open [`Refusal::Held`] until the request is answered, +//! which is the reconnect failing: the caller's disk is gone and nothing is +//! issued again. +//! +//! **Every session is opened over a region no server has held**, so nothing a +//! server writes to an earlier session's page reaches this one, whether or not +//! that server has let the page go. use std::collections::BTreeMap; @@ -26,10 +39,10 @@ use toyos::wake::{Bell, Waker}; use toyos_abi::syscall::SyscallError; use toyos_blockdev::SharedBlock; use toyos_blockring::client::{Ask, Client, Outcome, Placing, Ticket}; -use toyos_blockring::entry::Completion; -use toyos_blockring::layout::{self, ClientRings, ARENA, MAX_REQUEST_BLOCKS}; +use toyos_blockring::ends::{ClientEnds, Ending, Exchange, Next}; +use toyos_blockring::layout::{ARENA, MAX_REQUEST_BLOCKS}; use toyos_blockring::wire::{self, Opened, Refusal}; -use toyos_blockring::{Run, BLOCK_BYTES}; +use toyos_blockring::{Run, BLOCK_BYTES, SILENCE_BOUND_NS}; use toyos_blockserve::region::Region; @@ -129,7 +142,7 @@ pub struct Session { region: Region, /// The open session's doorbells, `None` once it ended. bells: Option, - rings: ClientRings, + ends: ClientEnds, client: Client>, opened: Opened, arena: Arena, @@ -137,8 +150,6 @@ pub struct Session { pending: BTreeMap, /// Each read on the wire's arena blocks. reads: BTreeMap, - /// The most requests this session has had on the wire at once. - peak: usize, } impl Session { @@ -146,7 +157,7 @@ impl Session { /// `names` calls `service`. pub fn open(names: Namespace, service: &str, guid: [u8; wire::GUID_BYTES]) -> Result { let region = Region::create().map_err(Error::Kernel)?; - let rings = layout::client(region.words()); + let ends = ClientEnds::new(region.words()); let (bells, opened) = handshake(&names, service, guid, ®ion)?; let mut client = Client::new(KEPT_BLOCKS); client.session_started(); @@ -156,14 +167,13 @@ impl Session { guid, region, bells: Some(bells), - rings, + ends, client, opened, arena: Arena::new(), next_ticket: 0, pending: BTreeMap::new(), reads: BTreeMap::new(), - peak: 0, }) } @@ -172,11 +182,6 @@ impl Session { self.opened.blocks() } - /// The most requests this session has had on the wire at once. - pub fn peak_on_the_wire(&self) -> usize { - self.peak - } - /// How many acknowledged writes have gone out again after a loss. pub fn reissued(&self) -> u64 { self.client.reissued() @@ -225,13 +230,13 @@ impl Session { Ok(ticket) } - /// Put what the client will send on the ring, and ring the doorbell once - /// for all of it. - fn pump(&mut self) { - let Self { bells, region, rings, client, arena, reads, .. } = self; - let Some(bells) = bells else { return }; - let page = region.words(); - let mut place = |p: Placing<'_, Vec>| match p { + /// Exchange requests and completions with the server at `now` + /// ([`ClientEnds::exchange`]), ring its doorbell once for what went, and + /// take every answer decided into `answers`. + fn exchange(&mut self, now: u64, answers: &mut Vec) -> Next { + let Self { bells, region, ends, client, arena, reads, .. } = self; + let bells = bells.as_ref().expect("diskserver: an exchange over a session that ended"); + let place = |p: Placing<'_, Vec>| match p { Placing::Read { ticket, blocks } => { let run = arena.alloc(blocks)?; reads.insert(ticket, run); @@ -246,28 +251,18 @@ impl Session { Some(run) } }; - loop { - match rings.0.space(page) { - Ok(0) => break, - Ok(_) => {} - // The server wrote an index no server writes: the session is - // over, and the next drain is where that is acted on. - Err(_) => break, - } - let Some(request) = client.next_request(&mut place) else { break }; - let pushed = rings.0.push(page, request.encode()); - assert_eq!(pushed, Ok(true), "diskserver: a request past the room just counted"); - } - let on_the_wire = client.on_the_wire().count(); - if rings.0.publish(page) { + let Exchange { next, published } = ends.exchange(region.words(), client, now, place); + if published { // A server that has ended is found by the wait. bells.ring.wake(); } - self.peak = self.peak.max(on_the_wire); + self.collect(answers); + next } /// Every answer there is; with `block`, wait until there is at least one - /// or the service has ended. + /// or the session has ended, which it has within [`SILENCE_BOUND_NS`] of + /// the server's last completion. pub fn wait(&mut self, block: bool) -> Waited { let mut waited = Waited::default(); if self.bells.is_none() { @@ -275,57 +270,29 @@ impl Session { return waited; } loop { - self.pump(); - let violated = self.drain(); - self.collect(&mut waited.answers); - if violated { - self.end(&mut waited); - return waited; - } - // Again after every completion: one can make a request sendable - // that was not — a flush waiting on writes going out again — and - // a wait with it unsent would wait for an answer nobody asked for. - self.pump(); - if !waited.answers.is_empty() || !block { - return waited; - } - let Some(bells) = &self.bells else { - waited.ended = true; - return waited; - }; - if bells.answered.wait_for(u64::MAX).is_err() { - // What the server posted before it went is real: read it - // before the rest is answered refused. - self.drain(); - self.collect(&mut waited.answers); - self.end(&mut waited); - return waited; - } - } - } - - /// Read every completion posted; `true` if the server broke the protocol. - fn drain(&mut self) -> bool { - let page = self.region.words(); - let mut violated = false; - loop { - match self.rings.1.pop(page) { - Ok(Some(words)) => match Completion::decode(words) { - Some(c) if self.client.complete(c).is_ok() => {} - _ => { - violated = true; - break; + let now = toyos_abi::clock::nanos_since_boot(); + match self.exchange(now, &mut waited.answers) { + Next::End(Ending::Violation(violation)) => { + println!("{}: the session's server broke the protocol ({violation:?})", self.service); + } + Next::End(Ending::Silence) => { + println!("{}: the session's server answered nothing in {} s", self.service, SILENCE_BOUND_NS / 1_000_000_000); + } + Next::Wait(patience) if block => { + let bells = self.bells.as_ref().expect("diskserver: the session is open"); + if bells.answered.wait_for(patience).is_ok() { + continue; } - }, - Ok(None) => break, - Err(_) => { - violated = true; - break; + // What the server posted before it went is real: read it + // before the rest is answered refused. + let _ = self.ends.receive(self.region.words(), &mut self.client, now); } + Next::Idle if block => panic!("diskserver: a wait with nothing asked would never end"), + Next::Answered | Next::Wait(_) | Next::Idle => return waited, } + self.end(&mut waited); + return waited; } - self.rings.1.release(page); - violated } fn collect(&mut self, answers: &mut Vec) { @@ -361,24 +328,24 @@ impl Session { waited.ended = true; } - /// Open the same partition over the same region through the same name, - /// after the service ended; writes the client holds go out again first. + /// Open the same partition through the same name over a fresh region, + /// after the session ended; at the next [`Self::wait`] the writes the + /// client holds go out again, first. Refused [`Refusal::Held`] while the + /// server still holds a request of the session that ended. /// /// Blocks until the service answers, within [`wire::ANSWER_BOUND_NS`]: a /// name whose server is being restarted queues the connection until the /// new one accepts it. pub fn reconnect(&mut self) -> Result<(), Error> { assert!(self.bells.is_none(), "diskserver: reconnect while a session is open"); - // Before the region goes to the new server: it must find this end's - // two indices at zero, as it will set its own. - self.rings = layout::client(self.region.words()); - let (bells, opened) = handshake(&self.names, &self.service, self.guid, &self.region)?; + let region = Region::create().map_err(Error::Kernel)?; + let ends = ClientEnds::new(region.words()); + let (bells, opened) = handshake(&self.names, &self.service, self.guid, ®ion)?; if opened != self.opened { return Err(Error::Protocol); } - self.bells = Some(bells); + (self.region, self.ends, self.bells) = (region, ends, Some(bells)); self.client.session_started(); - self.pump(); Ok(()) } } diff --git a/system/nvme/src/main.rs b/system/nvme/src/main.rs index c9d82d69975..da5129113a0 100644 --- a/system/nvme/src/main.rs +++ b/system/nvme/src/main.rs @@ -28,12 +28,7 @@ use toyos_blockserve::{Device, Sessions}; mod nvme; -use nvme::{Controller, Done, Owner}; - -/// How long a command may go unanswered before the controller is reset to -/// take it back. Policy: NVMe defines no per-command bound, and a flush of a -/// large cache is the slowest thing a healthy drive does. -const COMMAND_SILENCE: Duration = Duration::from_secs(10); +use nvme::{Controller, Done, Owner, COMMAND_SILENCE}; /// How long the stop's normal shutdown is given: a policy, below the /// supervisor's wait for every disk driver (`toyos_quiesce::DRIVERS_MS`). diff --git a/system/nvme/src/nvme.rs b/system/nvme/src/nvme.rs index 94b25c138e7..09b518e716c 100644 --- a/system/nvme/src/nvme.rs +++ b/system/nvme/src/nvme.rs @@ -29,7 +29,9 @@ //! //! **A command the controller does not answer is reclaimed by a controller //! reset** (§3.7.2): `CC.EN` cleared, every command it held is gone, the queues -//! are made again. Whoever owned one is told it was not done. +//! are made again. Whoever owned one is told it was not done. Every wait of +//! the reset is bounded, so a session's command is answered within the block +//! protocol's bound on a server's silence ([`ANSWERED_WITHIN`]). use std::time::{Duration, Instant}; @@ -110,6 +112,96 @@ pub enum Power { Autonomous { states: Vec, plan: Vec> }, } +/// How long a command may go unanswered before the controller is reset to +/// take it back. Policy: NVMe defines no per-command bound, and a flush of a +/// large cache is the slowest thing a healthy drive does. +pub const COMMAND_SILENCE: Duration = Duration::from_secs(10); + +/// How long an admin command is waited for, at bring-up and in a reset: the +/// same policy as any other command's. +const ADMIN_BOUND: Duration = COMMAND_SILENCE; + +/// `CAP.TO`'s unit, and its largest value: the controller's own bound on a +/// `CSTS.RDY` transition, in 500 ms units, at most 255 (NVMe Base 2.0 +/// §3.1.4.1). +const READY_UNIT: Duration = Duration::from_millis(500); +const READY_BOUND_MOST: Duration = READY_UNIT.checked_mul(255).unwrap(); + +/// One admin command of a reset, in the order it is sent: the queue count, a +/// completion and a submission queue for each I/O queue the controller grants, +/// and the APST table. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Step { + QueueCount, + CompletionQueue(u16), + SubmissionQueue(u16), + PowerTable, +} + +impl Step { + fn named(self) -> &'static str { + match self { + Self::QueueCount => "Set Features (Number of Queues)", + Self::CompletionQueue(_) => "Create I/O Completion Queue", + Self::SubmissionQueue(_) => "Create I/O Submission Queue", + Self::PowerTable => "Set Features (Autonomous Power State Transition)", + } + } +} + +/// Every step [`Controller::configure`] takes, at bring-up and in a reset, +/// each at most one admin command: the reset waits for no other. +const STEPS: [Step; 2 + 2 * MAX_IO_QUEUES as usize] = { + let mut steps = [Step::QueueCount; 2 + 2 * MAX_IO_QUEUES as usize]; + let mut q = 0; + while q < MAX_IO_QUEUES { + steps[1 + 2 * q as usize] = Step::CompletionQueue(q); + steps[2 + 2 * q as usize] = Step::SubmissionQueue(q); + q += 1; + } + steps[steps.len() - 1] = Step::PowerTable; + steps +}; + +/// The most admin commands a reset waits for. +const RESET_ADMIN_COMMANDS: u32 = STEPS.len() as u32; + +/// The longest a command goes unanswered, from its submission to the device: +/// [`COMMAND_SILENCE`], then the reset, whose two `CSTS.RDY` transitions are +/// each bounded by `CAP.TO` and whose admin commands each by [`ADMIN_BOUND`]. +const ANSWERED_WITHIN: Duration = COMMAND_SILENCE + .checked_add(READY_BOUND_MOST.checked_mul(2).unwrap()) + .unwrap() + .checked_add(ADMIN_BOUND.checked_mul(RESET_ADMIN_COMMANDS).unwrap()) + .unwrap(); +const _: () = assert!( + ANSWERED_WITHIN.as_nanos() <= toyos_blockring::SILENCE_BOUND_NS as u128, + "a command the controller leaves silent would be answered past the block protocol's bound" +); + +/// The submission and completion queue pages of I/O queue `q`. +fn queue_pages(q: u16) -> (usize, usize) { + let sq = OFF_IO + usize::from(q) * 2 * PAGE; + (sq, sq + PAGE) +} + +/// What `step` answers, asked until it answers or `bound` has passed by +/// `elapsed`. `step` is given how long it may wait before it looks: nothing +/// the first time, then never more than is left of the bound, so a wake just +/// before the bound buys no second one. +fn within(bound: Duration, elapsed: impl Fn() -> Duration, mut step: impl FnMut(Duration) -> Option) -> Option { + let mut left = Duration::ZERO; + loop { + if let Some(answer) = step(left) { + return Some(answer); + } + left = bound.saturating_sub(elapsed()); + if left.is_zero() { + return None; + } + } +} + /// Whose a command is. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Owner { @@ -257,7 +349,8 @@ pub struct Controller { dma: Window, dma_addr: u64, stride: usize, - /// `CAP.TO`: the controller's own worst case for a `CSTS.RDY` transition. + /// `CAP.TO`: the controller's own worst case for a `CSTS.RDY` transition, + /// at most [`READY_BOUND_MOST`]. ready_bound: Duration, admin: Queue, io: Vec, @@ -321,7 +414,7 @@ impl Controller { dma, dma_addr, stride, - ready_bound: Duration::from_millis(to * 500), + ready_bound: READY_UNIT.saturating_mul(to as u32), admin: Queue::new(0, ADMIN_DEPTH, OFF_ADMIN_SQ, OFF_ADMIN_CQ), io: Vec::new(), lba_bytes: 0, @@ -336,8 +429,7 @@ impl Controller { }; ctrl.enable()?; ctrl.identify()?; - ctrl.make_io_queues()?; - let refused = ctrl.set_power()?; + let refused = ctrl.configure()?; Ok((ctrl, refused)) } @@ -356,16 +448,18 @@ impl Controller { /// (§3.1.4.1, §3.5.1). fn settle(&self, ready: bool, what: &'static str) -> Result<(), Refusal> { let start = Instant::now(); - while (self.csts() & CSTS_RDY != 0) != ready { - if self.csts() & CSTS_CFS != 0 { - return Err(Refusal::Fatal(what)); + let settled = within(self.ready_bound, || start.elapsed(), |_| { + let csts = self.csts(); + if (csts & CSTS_RDY != 0) == ready { + return Some(Ok(())); } - if start.elapsed() > self.ready_bound { - return Err(Refusal::Silent(what, self.ready_bound)); + if csts & CSTS_CFS != 0 { + return Some(Err(Refusal::Fatal(what))); } std::thread::yield_now(); - } - Ok(()) + None + }); + settled.unwrap_or(Err(Refusal::Silent(what, self.ready_bound))) } /// The machine stops: a normal shutdown (NVMe Base 2.0 §3.6.2), which has @@ -446,80 +540,84 @@ impl Controller { Ok(()) } - /// Set the APST table Identify built, through the identify page, which - /// nothing reads after bring-up; nothing is sent where it built none. A - /// table with no transition is sent with APST disabled (APSTE, CDW11 bit - /// 0). The answer is the status the controller refused it with, if it - /// did, after which it serves from its operational state. - fn set_power(&mut self) -> Result, Refusal> { - let Power::Autonomous { plan, .. } = &self.power else { - return Ok(None); - }; - let enable = plan.iter().any(Option::is_some); - self.dma.copy_in(OFF_IDENTIFY, &nvme_apst::encode(plan)); - let cmd = Command { - opcode: ADMIN_SET_FEATURES, - prp1: self.dma_addr + OFF_IDENTIFY as u64, - cdw10: FEATURE_APST, - cdw11: u32::from(enable), - ..Command::default() - }; - match self.admin_command("Set Features (Autonomous Power State Transition)", cmd) { - Ok(_) => Ok(None), - Err(Refusal::Admin(_, status)) => Ok(Some(status)), - Err(why) => Err(why), - } - } - - fn make_io_queues(&mut self) -> Result<(), Refusal> { - let asked = MAX_IO_QUEUES as u32 - 1; - let answer = self.admin_command( - "Set Features (Number of Queues)", - Command { - opcode: ADMIN_SET_FEATURES, - cdw10: FEATURE_NUMBER_OF_QUEUES, - cdw11: (asked << 16) | asked, - ..Command::default() - }, - )?; - let sqs = (answer & 0xFFFF) + 1; - let cqs = (answer >> 16) + 1; - let n = sqs.min(cqs).min(MAX_IO_QUEUES as u32) as u16; + /// Make the I/O queues and set the APST table Identify built, at bring-up + /// and after a reset: the admin command of each of [`STEPS`] that applies + /// and no other. The answer is the status the controller refused the + /// table with, if it did, after which it serves from its operational + /// state. + fn configure(&mut self) -> Result, Refusal> { + let asked = u32::from(MAX_IO_QUEUES) - 1; + let size = (u32::from(IO_DEPTH) - 1) << 16; + let mut granted = 0; + let mut refused = None; self.io.clear(); - for q in 0..n { - let qid = q + 1; - let sq = OFF_IO + q as usize * 2 * PAGE; - let cq = sq + PAGE; - self.dma.sub(sq, PAGE).zero(); - self.dma.sub(cq, PAGE).zero(); - let size = (IO_DEPTH as u32 - 1) << 16; - // Physically contiguous, interrupts on, on the claim's vector 0. - self.admin_command( - "Create I/O Completion Queue", - Command { - opcode: ADMIN_CREATE_IO_CQ, - prp1: self.dma_addr + cq as u64, - cdw10: size | qid as u32, - cdw11: 0b11, - ..Command::default() - }, - )?; - self.admin_command( - "Create I/O Submission Queue", - Command { - opcode: ADMIN_CREATE_IO_SQ, - prp1: self.dma_addr + sq as u64, - cdw10: size | qid as u32, - cdw11: ((qid as u32) << 16) | 1, + for step in STEPS { + let cmd = match step { + Step::QueueCount => Command { + opcode: ADMIN_SET_FEATURES, + cdw10: FEATURE_NUMBER_OF_QUEUES, + cdw11: (asked << 16) | asked, ..Command::default() }, - )?; - self.io.push(Queue::new(qid, IO_DEPTH, sq, cq)); + // Physically contiguous, interrupts on, on the claim's vector 0. + Step::CompletionQueue(q) if q < granted => { + let (_, cq) = queue_pages(q); + self.dma.sub(cq, PAGE).zero(); + Command { + opcode: ADMIN_CREATE_IO_CQ, + prp1: self.dma_addr + cq as u64, + cdw10: size | u32::from(q + 1), + cdw11: 0b11, + ..Command::default() + } + } + Step::SubmissionQueue(q) if q < granted => { + let (sq, _) = queue_pages(q); + self.dma.sub(sq, PAGE).zero(); + Command { + opcode: ADMIN_CREATE_IO_SQ, + prp1: self.dma_addr + sq as u64, + cdw10: size | u32::from(q + 1), + cdw11: (u32::from(q + 1) << 16) | 1, + ..Command::default() + } + } + Step::CompletionQueue(_) | Step::SubmissionQueue(_) => continue, + // Through the identify page, which nothing reads after + // bring-up; nothing is sent where Identify built no table. A + // table with no transition is sent with APST disabled (APSTE, + // CDW11 bit 0). + Step::PowerTable => { + let Power::Autonomous { plan, .. } = &self.power else { continue }; + let enable = plan.iter().any(Option::is_some); + self.dma.copy_in(OFF_IDENTIFY, &nvme_apst::encode(plan)); + Command { + opcode: ADMIN_SET_FEATURES, + prp1: self.dma_addr + OFF_IDENTIFY as u64, + cdw10: FEATURE_APST, + cdw11: u32::from(enable), + ..Command::default() + } + } + }; + match (step, self.admin_command(step.named(), cmd)) { + (Step::QueueCount, Ok(answer)) => { + let (sqs, cqs) = ((answer & 0xFFFF) + 1, (answer >> 16) + 1); + granted = sqs.min(cqs).min(u32::from(MAX_IO_QUEUES)) as u16; + } + (Step::SubmissionQueue(q), Ok(_)) => { + let (sq, cq) = queue_pages(q); + self.io.push(Queue::new(q + 1, IO_DEPTH, sq, cq)); + } + (Step::PowerTable, Err(Refusal::Admin(_, status))) => refused = Some(status), + (_, Ok(_)) => {} + (_, Err(why)) => return Err(why), + } } if self.issued.len() != self.io.len() { self.issued = vec![0; self.io.len()]; } - Ok(()) + Ok(refused) } pub fn queues(&self) -> usize { @@ -637,20 +735,22 @@ impl Controller { /// An admin command, waited for: bring-up and reset are the only callers, /// and nothing else is on the device then. fn admin_command(&mut self, what: &'static str, cmd: Command) -> Result { - let cid = self.submit(None, cmd, Owner::Driver, |_| 0); + self.submit(None, cmd, Owner::Driver, |_| 0); let start = Instant::now(); let mut out = Vec::new(); - loop { - self.reap_queue(None, &mut out); - if let Some(done) = out.pop() { - debug_assert!(out.is_empty()); - let _ = cid; - return if done.ok { Ok(done.dw0) } else { Err(Refusal::Admin(what, done.status)) }; - } - if start.elapsed() > self.ready_bound { - return Err(Refusal::Silent(what, self.ready_bound)); + let done = within(ADMIN_BOUND, || start.elapsed(), |left| { + if !left.is_zero() { + self.wait_interrupt(left); } - self.wait_interrupt(self.ready_bound); + self.reap_queue(None, &mut out); + let done = out.pop(); + debug_assert!(out.is_empty()); + done + }); + match done { + Some(done) if done.ok => Ok(done.dw0), + Some(done) => Err(Refusal::Admin(what, done.status)), + None => Err(Refusal::Silent(what, ADMIN_BOUND)), } } @@ -748,8 +848,7 @@ impl Controller { .map(|slot| slot.owner) .collect(); self.enable()?; - self.make_io_queues()?; - let refused = self.set_power()?; + let refused = self.configure()?; Ok((aborted, refused)) } @@ -761,7 +860,64 @@ impl Controller { #[cfg(test)] mod tests { - use super::{page_size, sector_shift}; + use core::cell::Cell; + use std::time::Duration; + + use super::{page_size, sector_shift, within, Step, ANSWERED_WITHIN, STEPS}; + + /// A command left silent is answered `Device` after 10 s and a reset of two + /// `CSTS.RDY` transitions at the largest `CAP.TO`, 127.5 s each, and ten + /// admin commands at 10 s each: 365 s, the block protocol's bound. + #[test] + fn a_command_left_silent_is_answered_within_365_s() { + assert_eq!(ANSWERED_WITHIN, Duration::from_secs(365)); + assert_eq!(ANSWERED_WITHIN.as_nanos(), u128::from(toyos_blockring::SILENCE_BOUND_NS)); + } + + /// The queue count, then each of four queues' completion and submission + /// queue, then the APST table. + #[test] + fn a_reset_steps_through_the_queues_then_the_power_table() { + let mut want = vec![Step::QueueCount]; + for q in 0..4 { + want.extend([Step::CompletionQueue(q), Step::SubmissionQueue(q)]); + } + want.push(Step::PowerTable); + assert_eq!(STEPS[..], want[..]); + } + + /// A wake for something else 1 ms before the bound, then nothing: the wait + /// after it is for the 1 ms left, so the whole takes the bound. + #[test] + fn a_wake_before_the_bound_buys_no_second_bound() { + let bound = Duration::from_secs(10); + let clock = Cell::new(Duration::ZERO); + let mut waits = Vec::new(); + let answer: Option<()> = within(bound, || clock.get(), |left| { + waits.push(left); + let woke = if waits.len() == 2 { left.saturating_sub(Duration::from_millis(1)) } else { left }; + clock.set(clock.get() + woke); + None + }); + assert_eq!(answer, None); + assert_eq!(clock.get(), bound); + assert_eq!(waits, [Duration::ZERO, bound, Duration::from_millis(1)]); + } + + /// An answer at the first look ends the wait without one, and an answer + /// after a wait is the answer. + #[test] + fn an_answer_ends_the_wait() { + assert_eq!(within(Duration::from_secs(10), || Duration::ZERO, |_| Some(7)), Some(7)); + let clock = Cell::new(Duration::ZERO); + let mut looks = 0; + let answer = within(Duration::from_secs(10), || clock.get(), |left| { + looks += 1; + clock.set(clock.get() + left.min(Duration::from_secs(3))); + (looks == 3).then_some("done") + }); + assert_eq!((answer, looks, clock.get()), (Some("done"), 3, Duration::from_secs(6))); + } /// An Identify Namespace page: `formats` LBA formats, each `(MS, LBADS)`. fn namespace(formats: &[(u16, u8)]) -> impl Fn(usize) -> u32 {