From b5d31eabad2324244eab55e233825bbf1f53a01e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 9 Oct 2026 11:38:34 +0200 Subject: [PATCH 1/5] fuse: format fuse_dlm_cache.c to kernel style Convert space indentation to tabs, put block comment delimiters on their own lines, align continuation lines to the open parenthesis and drop the double dash as comment punctuation. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 43 +++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 9284880b2e96c6..95a2f7a9002fb6 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -620,7 +620,8 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); } -/* Context for a fuse_get_dlm_lock() request, embedding struct +/* + * Context for a fuse_get_dlm_lock() request, embedding struct * fuse_args as required by the request API. */ struct fuse_dlm_lock_args { @@ -652,7 +653,7 @@ struct fuse_dlm_lock_args { * there instead if still necessary, right before it touches the page * cache. * - * Does not re-validate the granted range against what was requested -- + * Does not re-validate the granted range against what was requested; * fuse_get_dlm_lock() still does that itself after being woken. */ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, @@ -669,7 +670,7 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, * range is treated as covered) here: either way fuse_get_dlm_lock() * goes on to consider the range usable. Any other error fails the * IO instead, and the caller releases the still-INIT rlock directly - * without ever touching the page cache under it -- leave it alone. + * without ever touching the page cache under it, so leave it alone. */ if (error && error != -ENOSYS) return; @@ -687,7 +688,7 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, * moved to READY as part of processing a reply that leaves the range * covered, before this function's caller is even woken up, or * directly to LOCKED when the range is already covered without a - * round trip -- see fuse_get_dlm_lock_complete() and the declaration + * round trip, see fuse_get_dlm_lock_complete() and the declaration * in fuse_dlm_cache.h. In the later case, the grant is re-validated * one LOCKED (fencing further revokes) before being trusted; if a * concurrent invalidate is found to have revoked it, @rlock is put @@ -720,7 +721,8 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (!length) return 0; - /* note that this can be run from different processes + /* + * note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care * of any races in lock requests. @@ -729,13 +731,13 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, * never disagree about what counts as covered. * * rlock is still INIT here, invisible to invalidation. Move - * it to LOCKED -- which blocks until any invalidate that is + * it to LOCKED, which blocks until any invalidate that is * concurrently draining an overlapping range (one that got * past its own conflict check while we were INIT) has fully - * revoked the grant and released -- then re-check: only once + * revoked the grant and released, then re-check: only once * LOCKED, which fences any further revoke, can is_held() be * trusted. If the grant was revoked, undo back to INIT (never - * blocks) and continue to request a fresh grant below. + * blocks) and fall through to request a fresh grant below. */ if (rlock) { err = fuse_range_lock_mark_locked(fi, rlock); @@ -750,7 +752,8 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned + /* + * note that the offset and length don't have to be page aligned * here but since we only get here on writeback caching we will * send out page aligned requests */ @@ -827,7 +830,7 @@ static inline uint64_t fuse_range_lock_last(struct fuse_range_lock *lock) INTERVAL_TREE_DEFINE(struct fuse_range_lock, rb, uint64_t, __subtree_end, fuse_range_lock_start, fuse_range_lock_last, static, - fuse_range_it); + fuse_range_it); /** * fuse_range_lock_tree_init - Initialize the range lock manager @@ -852,14 +855,14 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) * @lock itself) and either @lock or that range is a WRITE lock (READ * ranges may overlap each other freely). When @locked_only is set, a * range still in INIT state (reserved, not yet touching the page cache) - * is not considered, but a range already in READY or LOCKED state is -- + * is not considered, but a range already in READY or LOCKED state is, * see fuse_range_lock_acquire_locked(). * * A range held by @lock's own task never conflicts: the only way that * happens is the task faulting in its own user buffer mid-IO, where * fuse_filemap_fault() takes a range lock on the faulting page while * the read/write path already holds one covering it. Waiting here - * would then deadlock on ourselves, and skipping is safe -- the outer + * would then deadlock on ourselves, and skipping is safe: the outer * hold already gives invalidation an overlapping range to conflict * with for as long as both are held. * @@ -869,7 +872,7 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) */ static bool fuse_range_conflicts(struct fuse_range_lock_tree *tree, struct fuse_range_lock *lock, - bool locked_only) + bool locked_only) { struct fuse_range_lock *cur; @@ -892,7 +895,7 @@ static bool fuse_range_conflicts(struct fuse_range_lock_tree *tree, * @lock: The range lock to try to acquire * * Conflict tested against every existing range regardless of state, same - * as a plain exclusive acquire -- this is what keeps two local IOs on an + * as a plain exclusive acquire. This is what keeps two local IOs on an * overlapping range serialized against each other even while both are * still reserving (INIT), not yet touching the page cache. * @@ -987,7 +990,7 @@ static bool fuse_range_try_mark_locked(struct fuse_range_lock_tree *tree, * reboot clears. * * Return: 0 with the range reserved, or -ERESTARTSYS if a fatal signal - * arrived first -- the lock was never inserted then and must not be + * arrived first. The lock was never inserted then and must not be * passed to fuse_range_lock_release(). */ int fuse_range_lock_acquire_init(struct fuse_inode *inode, @@ -1019,7 +1022,7 @@ int fuse_range_lock_acquire_init(struct fuse_inode *inode, * callers that must not block, e.g. the fault path under mmap_lock. * * Return: true with the range reserved, false if a conflicting range is - * currently held -- the lock was not inserted then and must not be + * currently held. The lock was not inserted then and must not be * passed to fuse_range_lock_release(). */ bool fuse_range_lock_try_acquire_init(struct fuse_inode *inode, @@ -1070,7 +1073,7 @@ void fuse_range_lock_mark_ready(struct fuse_inode *inode, * Killable, like fuse_range_lock_acquire_init(). * * Return: 0 once LOCKED, or -ERESTARTSYS if a fatal signal arrived - * first -- @lock then keeps its previous state (INIT or READY) and is + * first. @lock then keeps its previous state (INIT or READY) and is * still held, so the caller must still release it. */ int fuse_range_lock_mark_locked(struct fuse_inode *inode, @@ -1091,7 +1094,7 @@ int fuse_range_lock_mark_locked(struct fuse_inode *inode, * Same semantics as fuse_range_lock_mark_locked() but never waits. * * Return: true if @lock is now LOCKED, false if a conflicting READY or - * LOCKED range is held -- @lock then keeps its previous state and is + * LOCKED range is held. @lock then keeps its previous state and is * still held, so the caller must still release it. */ bool fuse_range_lock_try_mark_locked(struct fuse_inode *inode, @@ -1138,8 +1141,8 @@ void fuse_range_lock_mark_init(struct fuse_inode *inode, */ void fuse_range_lock_acquire_locked(struct fuse_inode *inode, struct fuse_range_lock *lock, - uint64_t start, uint64_t end, - enum fuse_range_lock_mode mode) + uint64_t start, uint64_t end, + enum fuse_range_lock_mode mode) { struct fuse_range_lock_tree *tree = &inode->io_range_lock; From 1801c0e64a81fc83d6857e5e8fb0f7c49f6ddba0 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Fri, 9 Oct 2026 00:40:33 +0000 Subject: [PATCH 2/5] fuse: throttle foreground io-uring requests with the writeback cache With the writeback cache, foreground requests can wait in userspace on DLM revokes. A revoke waits on page invalidation, which waits on writeback. If foreground requests held every entry of an io-uring queue, background writeback could not be sent and none of them would complete. When the connection has the writeback cache, each queue now keeps a share of its entries for background requests. Background requests may use a quarter of the entries, but at least 2. Foreground requests may use the rest, but at least 8. A foreground request over the limit waits on the new fuse_req_fg_queue and only moves to fuse_req_queue when a slot frees up. Background requests keep their own path to free entries and are held to their cap, except that each queue may always have one active background request. If a queue has fewer than 10 entries both minimums cannot hold, and the foreground minimum wins. The foreground count is released on completion, on commit errors and on entry teardown. Requests marked no_fg_limit skip the foreground limit. These are the page cache reads and writes (redfs_do_readfolio, redfs_send_readpages and redfs_send_write_pages) and the flush from write_inode. A page fault holds the inode range lock while its read is outstanding, and the revoke that is waiting for that range lock would never finish if the read were queued behind other foreground requests. The flush is sent by writeback itself. Without the writeback cache, dispatch is unchanged. Signed-off-by: Hai Zhong Zhou --- fs/fuse/dev_uring.c | 140 +++++++++++++++++++++++++++++++++++++++--- fs/fuse/dev_uring_i.h | 13 ++++ fs/fuse/dir.c | 2 + fs/fuse/file.c | 3 + fs/fuse/fuse_i.h | 3 + 5 files changed, 154 insertions(+), 7 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 1d372cede3ee69..1f6b51a2069258 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -99,6 +99,67 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } +/* + * With the writeback cache, foreground requests may hold at most half of the + * entries. They might wait in userspace on DLM revokes, which wait on page + * invalidation, which waits on writeback - if they held all entries, + * background writeback could not be sent and none of them would complete. + */ +#define FUSE_URING_MIN_FOREGROUND 8U +#define FUSE_URING_MIN_BACKGROUND 2U + +static unsigned int fuse_uring_max_background(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents / 4, FUSE_URING_MIN_BACKGROUND); +} + +static unsigned int fuse_uring_max_foreground(struct fuse_ring_queue *queue) +{ + unsigned int bg = fuse_uring_max_background(queue); + + if (queue->nr_ents <= bg) { + pr_warn_once("fuse-io-uring configuration value is too small, could cause hung issue\n"); + return queue->nr_ents; + } + + return queue->nr_ents - bg; +} + +static bool fuse_uring_req_fg_limited(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !(req->args && req->args->no_fg_limit); +} + +static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) +{ + lockdep_assert_held(&queue->lock); + + while (queue->active_foreground < fuse_uring_max_foreground(queue) && + !list_empty(&queue->fuse_req_fg_queue)) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_fg_queue, + struct fuse_req, list); + set_bit(FR_URING_FG, &req->flags); + queue->active_foreground++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_end_foreground(struct fuse_ring_queue *queue, + struct fuse_req *req) +{ + lockdep_assert_held(&queue->lock); + + if (test_and_clear_bit(FR_URING_FG, &req->flags)) { + queue->active_foreground--; + fuse_uring_flush_queue_fg(queue); + } +} + static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { @@ -111,6 +172,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); + fuse_uring_end_foreground(queue, req); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); @@ -133,9 +195,15 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) LIST_HEAD(req_list); spin_lock(&queue->lock); - list_for_each_entry(req, &queue->fuse_req_queue, list) + list_for_each_entry(req, &queue->fuse_req_queue, list) { + clear_bit(FR_PENDING, &req->flags); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; + } + list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); list_splice_init(&queue->fuse_req_queue, &req_list); + list_splice_init(&queue->fuse_req_fg_queue, &req_list); queue->nr_reqs = 0; spin_unlock(&queue->lock); @@ -384,6 +452,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); queue->fpq.processing = pq; @@ -447,7 +516,10 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; } + queue->nr_ents--; /* * The entry must not be freed immediately, due to access of direct @@ -1061,6 +1133,8 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -1074,6 +1148,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; + struct fuse_ring_ent *avail; + struct fuse_req *next = NULL; err = -ENOTCONN; if (!ring) @@ -1112,7 +1188,32 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + if (test_bit(FR_BACKGROUND, &req->flags)) { + queue->active_background--; + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); + } + + /* + * The freed foreground or background slot might have moved a + * queued request to fuse_req_queue. This entry is not usable, + * so hand the request to another available entry. + */ + avail = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (avail) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(avail, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(avail, true); + req->out.h.error = err; clear_bit(FR_SENT, &req->flags); fuse_request_end(req); @@ -1357,6 +1458,8 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; + queue->nr_ents++; + fuse_uring_flush_queue_fg(queue); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1593,6 +1696,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; + struct fuse_req *next; int err; err = -EINVAL; @@ -1607,18 +1711,25 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); queue->nr_reqs++; - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else + if (fuse_uring_req_fg_limited(req)) { + list_add_tail(&req->list, &queue->fuse_req_fg_queue); + fuse_uring_flush_queue_fg(queue); + } else { list_add_tail(&req->list, &queue->fuse_req_queue); + } + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + next = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); - if (ent) + if (next) fuse_uring_dispatch_ent(ent, false); return; @@ -1685,12 +1796,27 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; bool removed = fuse_remove_pending_req(req, &queue->lock); + struct fuse_ring_ent *ent; + struct fuse_req *next = NULL; if (removed) { /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + + /* the freed foreground slot might let a queued request run */ + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (ent) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(ent, true); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 778924ea342c76..27e48f218a1881 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -98,6 +98,19 @@ struct fuse_ring_queue { /* background fuse requests */ struct list_head fuse_req_bg_queue; + /* foreground fuse requests waiting for the foreground limit */ + struct list_head fuse_req_fg_queue; + + /* number of registered ring entries */ + unsigned int nr_ents; + + /* + * foreground requests on fuse_req_queue or in userspace, limited to + * the entries not reserved for background requests with writeback cache; + * see fuse_uring_max_foreground() + */ + unsigned int active_foreground; + /* number of requests queued or in userspace */ unsigned int nr_reqs; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index a58e586754ef58..7ba18745d21c81 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2119,6 +2119,8 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) if (fm->fc->setattr_writeback) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); + /* might be called from writeback, like a background request */ + args.no_fg_limit = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index fffef155ceb586..822056293dfbbf 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1002,6 +1002,7 @@ static int fuse_do_readpage(struct file *file, struct page *page) struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, + .ap.args.no_fg_limit = true, .ap.num_pages = 1, .ap.pages = &page, .ap.descs = &desc, @@ -1112,6 +1113,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; + ap->args.no_fg_limit = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1558,6 +1560,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + ap->args.no_fg_limit = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 532ef7aff6e06a..0632fd25dceb1c 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -422,6 +422,7 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + bool no_fg_limit:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -510,6 +511,7 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring + * FR_URING_FG: request is counted in the io-uring queue foreground limit */ enum fuse_req_flag { FR_ISREPLY, @@ -526,6 +528,7 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_URING_FG, }; /** From a0cd7d5b861ab4857121adfca03f3f1cbda48a00 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Sat, 10 Oct 2026 09:03:53 +0000 Subject: [PATCH 3/5] fuse: reserve io-uring entries for critical requests with writeback cache With the writeback cache, a request can wait in userspace on a DLM revoke. The revoke waits on page invalidation on this node, which waits on writeback and on locked pages being read. Those page reads and writes are themselves fuse requests. If requests that may block in userspace held every entry of an io-uring queue, the page I/O could never be sent and none of the requests would complete. Add a uring_critical flag to fuse_args for requests on the path that invalidation waits for. These are the page reads in fuse_do_readfolio() and fuse_send_readpages() (both sync and async readahead), the page writes in fuse_send_write_pages() and fuse_send_writepage(), and fuse_flush_times(), which writeback calls. The daemon must never block a critical request on a DLM revoke that waits on page invalidation on this node. When the connection has the writeback cache, a quarter of the entries of each queue, but at least 2, are critical entries. Non-critical requests, foreground and background together, may hold only the other entries, counted in active_noncritical. A queue always keeps at least one entry for non-critical requests, and at least one critical entry when it has 2 or more entries. Critical requests are not counted in that limit and may use any entry. A non-critical foreground request over the limit waits on the new fuse_req_fg_queue. A non-critical background request waits on fuse_req_bg_queue, now limited by both the background limits and the non-critical limit. Critical background requests wait on the new fuse_req_bg_crit_queue, so they never sit behind non-critical ones, and are admitted first. They are still held to max_background, except that each queue may always have one critical background request active. Critical foreground requests go straight to fuse_req_queue. When a non-critical slot is freed, it is offered first to the same kind of request, foreground or background, that freed it, so neither kind can starve the other. Registering a new entry raises the limit and admits waiting requests of both kinds. The counts are released through one helper, fuse_uring_end_active(), on completion, on commit errors and on request removal. Entry teardown and queue abort also drop the non-critical count. Without the writeback cache, dispatch is unchanged. Signed-off-by: Hai Zhong Zhou --- fs/fuse/dev.c | 13 +- fs/fuse/dev_uring.c | 294 ++++++++++++++++++++++++++---------------- fs/fuse/dev_uring_i.h | 22 +++- fs/fuse/dir.c | 2 +- fs/fuse/file.c | 7 +- fs/fuse/fuse_i.h | 23 +++- 6 files changed, 239 insertions(+), 122 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 6520500d01bc41..8d87f330a6b036 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -649,10 +649,21 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, return -ENOMEM; __set_bit(FR_BACKGROUND, &req->flags); } else { + /* + * A critical request must not wait for the background budget: + * async readahead holds its folios locked here, and the budget + * might be held by requests blocked in userspace on a DLM revoke + * that waits for those folios. It still counts in the budget, + * and the limits on active background requests still apply. + */ + bool critical = args->uring_critical && fm->fc->writeback_cache; + WARN_ON(args->nocreds); - req = fuse_get_req(&invalid_mnt_idmap, fm, true); + req = fuse_get_req(&invalid_mnt_idmap, fm, !critical); if (IS_ERR(req)) return PTR_ERR(req); + if (critical) + __set_bit(FR_BACKGROUND, &req->flags); } fuse_args_to_req(req, args); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 1f6b51a2069258..c80de864d83183 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -71,6 +71,46 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) return pdu->ent; } +/* + * With the writeback cache, some entries of a queue are critical entries, used + * only by uring_critical requests. Other requests, foreground and background, + * might wait in userspace on DLM revokes, which wait on page invalidation, + * which waits on writeback and page reads - if they held all entries, the + * critical requests could not be sent and none of them would complete. + * + * The critical entries are a quarter of the entries, but at least 2. + * Non-critical requests share the rest, but always keep at least one entry, + * and always leave at least one critical entry when there are 2 or more. + */ +#define FUSE_URING_MIN_CRITICAL 2U + +static unsigned int fuse_uring_nr_critical(struct fuse_ring_queue *queue) +{ + if (queue->nr_ents < 2) + return 0; + + return min(max(queue->nr_ents / 4, FUSE_URING_MIN_CRITICAL), + queue->nr_ents - 1); +} + +static unsigned int fuse_uring_max_noncritical(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents - fuse_uring_nr_critical(queue), 1U); +} + +static bool fuse_uring_req_critical(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + req->args && req->args->uring_critical; +} + +static bool fuse_uring_req_fg_noncritical(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !fuse_uring_req_critical(req); +} + static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) { struct fuse_ring *ring = queue->ring; @@ -84,101 +124,119 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) * This prevents a single queue from consuming all resources and * eliminates the need for remote queue wake-ups when global * limits are met but this queue has no more waiting requests. + * + * Critical bg requests wait on their own list, so they never wait + * behind non-critical bg requests, and are admitted first. They are + * subject to the bg limits, but one critical bg request per queue is + * always allowed. It is not counted in the non-critical limit and can + * use the critical entries, so critical bg requests always progress. + * + * With the writeback cache, non-critical bg requests also share the + * non-critical limit with foreground requests, see + * fuse_uring_max_noncritical(). */ while ((fc->active_background < fc->max_background || - !queue->active_background) && - (!list_empty(&queue->fuse_req_bg_queue))) { + !queue->active_bg_critical) && + !list_empty(&queue->fuse_req_bg_crit_queue)) { struct fuse_req *req; - req = list_first_entry(&queue->fuse_req_bg_queue, + req = list_first_entry(&queue->fuse_req_bg_crit_queue, struct fuse_req, list); fc->active_background++; queue->active_background++; + set_bit(FR_URING_CRITICAL, &req->flags); + queue->active_bg_critical++; list_move_tail(&req->list, &queue->fuse_req_queue); } -} -/* - * With the writeback cache, foreground requests may hold at most half of the - * entries. They might wait in userspace on DLM revokes, which wait on page - * invalidation, which waits on writeback - if they held all entries, - * background writeback could not be sent and none of them would complete. - */ -#define FUSE_URING_MIN_FOREGROUND 8U -#define FUSE_URING_MIN_BACKGROUND 2U - -static unsigned int fuse_uring_max_background(struct fuse_ring_queue *queue) -{ - return max(queue->nr_ents / 4, FUSE_URING_MIN_BACKGROUND); -} + while ((fc->active_background < fc->max_background || + !queue->active_background) && + (!fc->writeback_cache || queue->stopped || + queue->active_noncritical < fuse_uring_max_noncritical(queue)) && + (!list_empty(&queue->fuse_req_bg_queue))) { + struct fuse_req *req; -static unsigned int fuse_uring_max_foreground(struct fuse_ring_queue *queue) -{ - unsigned int bg = fuse_uring_max_background(queue); + req = list_first_entry(&queue->fuse_req_bg_queue, + struct fuse_req, list); + fc->active_background++; + queue->active_background++; + if (fc->writeback_cache) { + set_bit(FR_URING_NONCRIT, &req->flags); + queue->active_noncritical++; + } - if (queue->nr_ents <= bg) { - pr_warn_once("fuse-io-uring configuration value is too small, could cause hung issue\n"); - return queue->nr_ents; + list_move_tail(&req->list, &queue->fuse_req_queue); } - - return queue->nr_ents - bg; -} - -static bool fuse_uring_req_fg_limited(struct fuse_req *req) -{ - return req->fm->fc->writeback_cache && - !test_bit(FR_BACKGROUND, &req->flags) && - !(req->args && req->args->no_fg_limit); } static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) { lockdep_assert_held(&queue->lock); - while (queue->active_foreground < fuse_uring_max_foreground(queue) && + while (queue->active_noncritical < fuse_uring_max_noncritical(queue) && !list_empty(&queue->fuse_req_fg_queue)) { struct fuse_req *req; req = list_first_entry(&queue->fuse_req_fg_queue, struct fuse_req, list); - set_bit(FR_URING_FG, &req->flags); - queue->active_foreground++; + set_bit(FR_URING_NONCRIT, &req->flags); + queue->active_noncritical++; list_move_tail(&req->list, &queue->fuse_req_queue); } } -static void fuse_uring_end_foreground(struct fuse_ring_queue *queue, - struct fuse_req *req) +/* + * Undo the queue counting of an admitted request, and admit waiting requests + * into the freed slots. Requests that were never admitted are not counted. + */ +static void fuse_uring_end_active(struct fuse_ring_queue *queue, + struct fuse_req *req) { + struct fuse_conn *fc = queue->ring->fc; + bool noncritical, background; + lockdep_assert_held(&queue->lock); - if (test_and_clear_bit(FR_URING_FG, &req->flags)) { - queue->active_foreground--; + noncritical = test_and_clear_bit(FR_URING_NONCRIT, &req->flags); + if (noncritical) + queue->active_noncritical--; + + background = test_bit(FR_BACKGROUND, &req->flags); + if (background) { + queue->active_background--; + if (test_and_clear_bit(FR_URING_CRITICAL, &req->flags)) + queue->active_bg_critical--; + } + + /* + * The non-critical limit is shared by foreground and bg requests. A + * freed non-critical slot is offered first to the kind of request that + * freed it, so under load neither kind can starve the other. + */ + if (noncritical && !background) fuse_uring_flush_queue_fg(queue); + if (noncritical || background) { + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); } + if (noncritical && background) + fuse_uring_flush_queue_fg(queue); } static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; lockdep_assert_not_held(&queue->lock); spin_lock(&queue->lock); ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); - fuse_uring_end_foreground(queue, req); - if (test_bit(FR_BACKGROUND, &req->flags)) { - queue->active_background--; - spin_lock(&fc->bg_lock); - fuse_uring_flush_queue_bg(queue); - spin_unlock(&fc->bg_lock); - } + fuse_uring_end_active(queue, req); spin_unlock(&queue->lock); if (error) @@ -197,8 +255,8 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) spin_lock(&queue->lock); list_for_each_entry(req, &queue->fuse_req_queue, list) { clear_bit(FR_PENDING, &req->flags); - if (test_and_clear_bit(FR_URING_FG, &req->flags)) - queue->active_foreground--; + if (test_and_clear_bit(FR_URING_NONCRIT, &req->flags)) + queue->active_noncritical--; } list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); @@ -452,6 +510,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_bg_crit_queue); INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); @@ -516,8 +575,8 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); - if (test_and_clear_bit(FR_URING_FG, &req->flags)) - queue->active_foreground--; + if (test_and_clear_bit(FR_URING_NONCRIT, &req->flags)) + queue->active_noncritical--; } queue->nr_ents--; @@ -1095,6 +1154,40 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, fuse_uring_req_end(ent, req, err); } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + +/* + * Hand each request on fuse_req_queue to an available entry. Freeing a slot + * or background budget can admit more than one request at once, and every one + * of them needs an entry: a request left on fuse_req_queue next to an idle + * entry waits for the next event on this queue, which never comes when the + * requests in userspace are all blocked on a DLM revoke. + * + * Must be called without the queue lock. The entries are sent in task context, + * as the caller might hold the io_uring submission lock. + */ +static void fuse_uring_dispatch_pending(struct fuse_ring_queue *queue) +{ + struct fuse_ring_ent *ent; + struct fuse_req *req; + + for (;;) { + spin_lock(&queue->lock); + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + req = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (req) + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + + if (!req) + return; + + fuse_uring_dispatch_ent(ent, true); + } +} + /* * Get the next fuse req and send it */ @@ -1104,11 +1197,15 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, { int err; struct fuse_req *req; + bool more; retry: spin_lock(&queue->lock); fuse_uring_ent_avail(ent, queue); req = fuse_uring_ent_assign_req(ent); + /* the request that ended might have admitted more than this one */ + more = !list_empty(&queue->fuse_req_queue) && + !list_empty(&queue->ent_avail_queue); spin_unlock(&queue->lock); if (req) { @@ -1116,6 +1213,9 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, if (err) goto retry; } + + if (more) + fuse_uring_dispatch_pending(queue); } static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) @@ -1133,8 +1233,6 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } -static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); - /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -1148,8 +1246,6 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; - struct fuse_ring_ent *avail; - struct fuse_req *next = NULL; err = -ENOTCONN; if (!ring) @@ -1166,6 +1262,17 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped)) return err; + /* + * With a single entry no entry is critical, so a non-critical request + * blocked in userspace on a DLM revoke can hold the only entry, see + * fuse_uring_nr_critical(). Checked on commit rather than on register, + * as a queue has a single entry while its entries are being registered, + * and libfuse registers all of them before it commits a reply. + */ + if (fc->writeback_cache && READ_ONCE(queue->nr_ents) < 2) + pr_warn_once("fuse-io-uring: qid=%u has a single entry, page I/O might hang behind a request blocked in userspace; use an io-uring queue depth of 2 or more\n", + qid); + spin_lock(&queue->lock); /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list @@ -1189,30 +1296,16 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); queue->nr_reqs--; - fuse_uring_end_foreground(queue, req); - if (test_bit(FR_BACKGROUND, &req->flags)) { - queue->active_background--; - spin_lock(&fc->bg_lock); - fuse_uring_flush_queue_bg(queue); - spin_unlock(&fc->bg_lock); - } + fuse_uring_end_active(queue, req); - /* - * The freed foreground or background slot might have moved a - * queued request to fuse_req_queue. This entry is not usable, - * so hand the request to another available entry. - */ - avail = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); - if (avail) - next = list_first_entry_or_null(&queue->fuse_req_queue, - struct fuse_req, list); - if (next) - fuse_uring_add_req_to_ring_ent(avail, next); spin_unlock(&queue->lock); - if (next) - fuse_uring_dispatch_ent(avail, true); + /* + * The freed non-critical or background slot might have moved + * queued requests to fuse_req_queue. This entry is not usable, + * so hand them to the available entries. + */ + fuse_uring_dispatch_pending(queue); req->out.h.error = err; clear_bit(FR_SENT, &req->flags); @@ -1459,7 +1552,11 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; queue->nr_ents++; + /* the new entry might raise the non-critical limit */ fuse_uring_flush_queue_fg(queue); + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1713,7 +1810,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) req->ring_queue = queue; queue->nr_reqs++; - if (fuse_uring_req_fg_limited(req)) { + if (fuse_uring_req_fg_noncritical(req)) { list_add_tail(&req->list, &queue->fuse_req_fg_queue); fuse_uring_flush_queue_fg(queue); } else { @@ -1747,7 +1844,6 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) struct fuse_conn *fc = req->fm->fc; struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; - struct fuse_ring_ent *ent = NULL; queue = fuse_uring_select_queue(ring, true); if (!queue) @@ -1761,33 +1857,26 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - list_add_tail(&req->list, &queue->fuse_req_bg_queue); + if (fuse_uring_req_critical(req)) + list_add_tail(&req->list, &queue->fuse_req_bg_crit_queue); + else + list_add_tail(&req->list, &queue->fuse_req_bg_queue); queue->nr_reqs++; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); spin_lock(&fc->bg_lock); fc->num_background++; if (fc->num_background == fc->max_background) fc->blocked = 1; fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); + spin_unlock(&queue->lock); /* * Due to bg_queue flush limits there might be other bg requests - * in the queue that need to be handled first. Or no further req - * might be available. + * in the queue that need to be handled first, or this one might have + * to wait. Budget freed on other queues might also admit several. */ - req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, - list); - if (ent && req) { - fuse_uring_add_req_to_ring_ent(ent, req); - spin_unlock(&queue->lock); - - fuse_uring_dispatch_ent(ent, true); - } else { - spin_unlock(&queue->lock); - } + fuse_uring_dispatch_pending(queue); return true; } @@ -1796,27 +1885,16 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; bool removed = fuse_remove_pending_req(req, &queue->lock); - struct fuse_ring_ent *ent; - struct fuse_req *next = NULL; if (removed) { /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; - fuse_uring_end_foreground(queue, req); - - /* the freed foreground slot might let a queued request run */ - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); - if (ent) - next = list_first_entry_or_null(&queue->fuse_req_queue, - struct fuse_req, list); - if (next) - fuse_uring_add_req_to_ring_ent(ent, next); + fuse_uring_end_active(queue, req); spin_unlock(&queue->lock); - if (next) - fuse_uring_dispatch_ent(ent, true); + /* the freed non-critical slot might let queued requests run */ + fuse_uring_dispatch_pending(queue); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 27e48f218a1881..da7e8a17301ccc 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -95,21 +95,25 @@ struct fuse_ring_queue { /* fuse requests waiting for an entry slot */ struct list_head fuse_req_queue; - /* background fuse requests */ + /* non-critical background fuse requests */ struct list_head fuse_req_bg_queue; - /* foreground fuse requests waiting for the foreground limit */ + /* uring_critical background fuse requests */ + struct list_head fuse_req_bg_crit_queue; + + /* non-critical foreground fuse requests waiting for the non-critical limit */ struct list_head fuse_req_fg_queue; /* number of registered ring entries */ unsigned int nr_ents; /* - * foreground requests on fuse_req_queue or in userspace, limited to - * the entries not reserved for background requests with writeback cache; - * see fuse_uring_max_foreground() + * non-critical requests, foreground and background, on fuse_req_queue + * or in userspace. With the writeback cache they are limited to the + * entries that are not critical entries, which are used only by + * uring_critical requests, see fuse_uring_max_noncritical(). */ - unsigned int active_foreground; + unsigned int active_noncritical; /* number of requests queued or in userspace */ unsigned int nr_reqs; @@ -118,6 +122,12 @@ struct fuse_ring_queue { unsigned int active_background; + /* + * uring_critical background requests on fuse_req_queue or in + * userspace, one is always allowed regardless of the bg limits + */ + unsigned int active_bg_critical; + bool stopped; }; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 7ba18745d21c81..6c12580c1a7e49 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2120,7 +2120,7 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); /* might be called from writeback, like a background request */ - args.no_fg_limit = true; + args.uring_critical = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 822056293dfbbf..89c547962c31c7 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1002,7 +1002,7 @@ static int fuse_do_readpage(struct file *file, struct page *page) struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, - .ap.args.no_fg_limit = true, + .ap.args.uring_critical = true, .ap.num_pages = 1, .ap.pages = &page, .ap.descs = &desc, @@ -1113,7 +1113,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; - ap->args.no_fg_limit = true; + ap->args.uring_critical = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1560,7 +1560,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; - ap->args.no_fg_limit = true; + ap->args.uring_critical = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; @@ -2782,6 +2782,7 @@ __acquires(fi->lock) args->in_args[1].size = inarg->size; args->force = true; args->nocreds = true; + args->uring_critical = true; err = fuse_simple_background(fm, args, GFP_ATOMIC); if (err == -ENOMEM) { diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 0632fd25dceb1c..ea98502532ebb0 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -422,7 +422,20 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; - bool no_fg_limit:1; + /* + * Request is on the path invalidation waits for: the page reads in + * fuse_do_readfolio() and fuse_send_readpages(), the page writes in + * fuse_send_write_pages() and fuse_send_writepage(), and + * fuse_flush_times(), which writeback calls. With the writeback cache, + * it must never be held back behind requests that may be blocked in + * userspace on a DLM revoke: foreground or background, it is exempt + * from the io-uring entry limits and may use the critical entries, see + * fuse_uring_req_critical(), and a background one does not wait for + * the background budget, see fuse_simple_background(). Critical + * requests are not limited, so the daemon must never block them on a + * DLM revoke that waits on page invalidation on this node. + */ + bool uring_critical:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -511,7 +524,10 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring - * FR_URING_FG: request is counted in the io-uring queue foreground limit + * FR_URING_NONCRIT: request is counted in the io-uring queue non-critical limit, + * shared by non-critical foreground and background requests + * FR_URING_CRITICAL: critical background request counted in the io-uring + * queue active critical background requests */ enum fuse_req_flag { FR_ISREPLY, @@ -528,7 +544,8 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, - FR_URING_FG, + FR_URING_NONCRIT, + FR_URING_CRITICAL, }; /** From 17b00667d026cd6178624a66ab73d0b9064dcf13 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Sat, 10 Oct 2026 08:42:16 -0700 Subject: [PATCH 4/5] fuse: upgrade only the granted subrange in the DLM grant cache fuse_dlm_lock_range() flipped the mode of every READ range overlapping a newly recorded WRITE grant in its entirety, not just the part the grant covers. A whole-file READ grant -- read(2) leaves one behind via readahead -- overlapped by any WRITE grant was thus recorded as a whole-file WRITE grant, and fuse_dlm_lock_is_held() then reported WRITE coverage the server never gave. Every write-side check trusts that record: the fault fast path, fuse_get_dlm_lock()'s early exit, and the write path's re-validation all see the range as covered, no lock request reaches the server, and writeback sends WRITEs the server observes under a read lock. Split the overlapping READ range at the grant boundaries and upgrade only the intersection. The splits preserve their mode, so an allocation failure partway through still leaves the tree describing exactly the grants held. Found probing the fault-around fix in the next commit: the probe's first-touch write fault should have requested a WRITE grant for its batch window, but the poisoned cache reported the page already covered and no request reached the server. Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 48 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 95a2f7a9002fb6..e6080f312cc047 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -176,7 +176,10 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * For overlapping ranges, handle lock compatibility: * - READ locks are compatible with existing READ locks * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * - WRITE locks need to upgrade existing READ locks, but only over the + * intersection with [start, end]: the server granted nothing outside + * it, so the parts of a READ range beyond the grant are split off and + * keep their mode * * Return: 0 on success, negative error code on failure */ @@ -210,6 +213,49 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, /* Check lock compatibility */ if (lock_mode == FUSE_PCACHE_LK_WRITE && lock_mode != range->mode) { + /* + * Upgrade only what the server granted: the grant + * covers [start, end], not the whole of an + * overlapping READ range. Split off the pieces + * outside the grant first -- mode-preserving, so + * an allocation failure below still leaves the + * tree describing exactly the grants held -- and + * upgrade just the intersection. + */ + if (range->start < start) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = range->start; + new_range->end = start - 1; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->start = start; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } + if (range->end > end) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = end + 1; + new_range->end = range->end; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->end = end; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } /* we own the lock but have to update it. */ list_add_tail(&range->list, &to_upgrade); } From 89c800bf207d766452ca7fe0b32935ba5a1859c9 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Sat, 10 Oct 2026 08:42:17 -0700 Subject: [PATCH 5/5] fuse: drop fault-around for shared-writable mappings under DLM With the writeback cache on, page_mkwrite skipped the DLM lock request entirely, on the assumption that fuse_filemap_fault() had already taken a write grant for any page a shared-writable mapping could dirty. That assumption does not hold: fault-around (filemap_map_pages) PTE-maps any resident folio in its window without calling ->fault for it, so a folio read in under a READ grant -- by read(2) or readahead -- can take its first write fault with no WRITE grant ever requested. The page is dirtied anyway and writeback later sends a WRITE the server sees under a read lock (DFS cases 040, 080, 081, 090 and 200). page_mkwrite cannot take the grant itself: it is called with mmap_lock (or the per-VMA lock) held and has no VM_FAULT_RETRY protocol to drop it, while the buffered write path holds the IO range lock LOCKED while faulting in its source buffer, which takes mmap_lock. With a pending mmap_lock writer queued between the two, blocking in page_mkwrite on the range lock or the DLM round trip recreates the three-party deadlock that 94effa2 ("fuse: do not block the fault path under mmap_lock") removed from the fault path. Instead, give shared-writable VMAs under the writeback cache with DLM a vm_operations_struct without ->map_pages, so the first touch of every page goes through fuse_filemap_fault(), which records a WRITE grant for its batch window before any PTE is installed and already drops mmap_lock to block. page_mkwrite keeps a non-blocking check of the recorded grant -- the same LOCKED fence the fault fast path uses -- and returns VM_FAULT_NOPAGE on a miss. A verified miss can only mean a revoke in flight, whose invalidation clears the PTE, so the re-fault resolves through fuse_filemap_fault(); a trylock-contended miss re-faults with mmap_lock released in between, so the range lock holder can make progress. This also makes fuse_filemap_fault()'s FAULT_FLAG_TRIED pass take the drop-and-retry slow path instead of blocking in place: with every first touch of a shared-writable page now funnelled through ->fault, blocking there on the range lock with mmap_lock held steps into page_mkwrite's old role in the same three-party cycle, and did so instantly under a buffered write sourced from an mmap of the file racing munmap and a write fault. Retries may repeat (4064b98 "mm: allow VM_FAULT_RETRY for multiple times"), so waiting for a contended range must happen with mmap_lock dropped, every time; blocking in place remains only for callers that allow no retries at all. The non-writeback-cache arm keeps the existing single-page fuse_get_page_mkwrite_lock() behaviour. Reported-by: Hai Zhong Zhou Suggested-by: Yong Ze Chen Signed-off-by: Allison Henderson --- fs/fuse/file.c | 162 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 39 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 89c547962c31c7..b10aa4ae3021fa 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3379,6 +3379,36 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l } return err; } + +/* + * Batch size for fuse_fault_read_private_folios(): one fault-around + * window (64k with 4k pages), centred on the faulted page. A + * spanning access (one instruction touching the tail of one page and + * the head of the next -- any unaligned copy does this constantly) + * needs BOTH pages mapped at once to retire, and every server round + * trip gets this client's grant revoked and its page cache dropped + * (the server does not exempt the requester, and the notification + * runs one op behind). A one-page grant-and-populate therefore + * ping-pongs forever: mapping each page costs a round trip whose + * revoke drops the other. A window covering both pages makes the + * neighbour's fault a pure fast-path hit -- recorded grant, resident + * folio, NO server op and no new revoke -- and the spanning access + * completes. Centred, not aligned: an aligned window has edges + * (every 16th page boundary), and a spanning access sitting on an + * edge gets two one-page-covering windows and ping-pongs just the + * same (generic/263); a centred window always covers both of the + * faulted page's neighbours, so no boundary has the problem. + */ +#define FUSE_FAULT_BATCH_PAGES 16 + +/* First page of the populate/grant window centred on @index. */ +static pgoff_t fuse_fault_batch_base(pgoff_t index) +{ + if (index < FUSE_FAULT_BATCH_PAGES / 2) + return 0; + return index - FUSE_FAULT_BATCH_PAGES / 2; +} + /* * Wait for writeback against this page to complete before allowing it * to be marked dirty again, and hence written back again, possibly @@ -3400,9 +3430,60 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct file *file = vmf->vma->vm_file; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = vmf->pgoff << PAGE_SHIFT; - if (!fm->fc->writeback_cache && fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; + if (fm->fc->writeback_cache && fm->fc->dlm) { + /* + * Writenotify routes every clean-to-dirty transition of a + * shared mapping through here, but the WRITE grant itself + * is taken by fuse_filemap_fault(): shared-writable VMAs + * get no ->map_pages under DLM (see fuse_file_mmap()), so + * every PTE in such a VMA was installed by a fault that + * recorded a WRITE grant for its batch window. Only + * verify here, and only without blocking -- nothing may + * sleep on the range lock or the server while mmap_lock + * (or the per-VMA lock) is held: the buffered write path + * holds the range lock LOCKED while faulting in its + * source buffer (which takes mmap_lock), so with a + * pending mmap_lock writer queued between the two, + * blocking here deadlocks three ways -- the cycle + * 94effa2 ("fuse: do not block the fault path under + * mmap_lock") removed from the fault path -- and + * page_mkwrite has no VM_FAULT_RETRY protocol to drop + * the lock and come back with. + * + * Same fence as fuse_filemap_fault()'s fast path: only + * once LOCKED can the recorded grant be trusted. On a + * miss, NOPAGE. A contended trylock re-faults with + * mmap_lock released in between, so the holder can make + * progress; a verified miss means a revoke is in flight + * (only an invalidate removes a recorded grant), whose + * unmap_mapping_range() clears the PTE, so the re-fault + * goes through fuse_filemap_fault(), which re-takes the + * grant with the proper drop-and-retry dance. + */ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + bool held = false; + + if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, + pos + PAGE_SIZE - 1, + FUSE_RANGE_LOCK_WRITE)) { + bool locked = + fuse_range_lock_try_mark_locked(fi, &rlock); + + held = locked && + fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, + FUSE_PAGE_LOCK_WRITE); + if (locked && !held) + pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", + fi->nodeid, vmf->pgoff); + fuse_range_lock_release(fi, &rlock); + } + + if (!held) + return VM_FAULT_NOPAGE; + } else if (fm->fc->dlm) { size_t length = PAGE_SIZE; int err = fuse_get_page_mkwrite_lock(file, pos, length); if (err < 0) { @@ -3446,35 +3527,6 @@ static bool fuse_fault_needs_read(struct inode *inode, pgoff_t index) return !uptodate; } -/* - * Batch size for fuse_fault_read_private_folios(): one fault-around - * window (64k with 4k pages), centred on the faulted page. A - * spanning access (one instruction touching the tail of one page and - * the head of the next -- any unaligned copy does this constantly) - * needs BOTH pages mapped at once to retire, and every server round - * trip gets this client's grant revoked and its page cache dropped - * (the server does not exempt the requester, and the notification - * runs one op behind). A one-page grant-and-populate therefore - * ping-pongs forever: mapping each page costs a round trip whose - * revoke drops the other. A window covering both pages makes the - * neighbour's fault a pure fast-path hit -- recorded grant, resident - * folio, NO server op and no new revoke -- and the spanning access - * completes. Centred, not aligned: an aligned window has edges - * (every 16th page boundary), and a spanning access sitting on an - * edge gets two one-page-covering windows and ping-pongs just the - * same (generic/263); a centred window always covers both of the - * faulted page's neighbours, so no boundary has the problem. - */ -#define FUSE_FAULT_BATCH_PAGES 16 - -/* First page of the populate/grant window centred on @index. */ -static pgoff_t fuse_fault_batch_base(pgoff_t index) -{ - if (index < FUSE_FAULT_BATCH_PAGES / 2) - return 0; - return index - FUSE_FAULT_BATCH_PAGES / 2; -} - /* * Read a batch of pages centred on @index into freshly * allocated folios that are NOT in the page cache. Unlike @@ -3707,9 +3759,19 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) * miss and come back here; each pass leaves a recorded * grant behind, which only a real invalidation takes away * again. + * + * FAULT_FLAG_TRIED does not demote a pass to the blocking + * path below: retries may repeat (4064b98 "mm: allow + * VM_FAULT_RETRY for multiple times"), and blocking on + * the range lock with mmap_lock held closes a three-party + * cycle whenever the lock holder is a buffered write + * faulting in its source buffer (which waits on + * mmap_lock) behind a queued mmap_lock writer. A miss on + * the retried fast path only means the range is contended + * again; waiting for it must happen with mmap_lock + * dropped, every time. */ - if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && - !(vmf->flags & FAULT_FLAG_TRIED)) { + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) return VM_FAULT_RETRY; @@ -3763,11 +3825,12 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) } /* - * Retries exhausted or not allowed: block in place. The - * waits are killable, and a same-task fault-in never - * conflicts with its own IO's range lock, so this cannot - * deadlock on itself; it can still stack behind a pending - * mmap_lock writer the way any blocking fault can. + * Retry not allowed (a caller without + * FAULT_FLAG_ALLOW_RETRY): block in place. The waits are + * killable, and a same-task fault-in never conflicts with + * its own IO's range lock, so this cannot deadlock on + * itself; it can still stack behind a pending mmap_lock + * writer the way any blocking fault can. * * Nothing held once the range lock is LOCKED may wait for * the server (see fuse_cache_read_iter), and @@ -3906,6 +3969,23 @@ static const struct vm_operations_struct fuse_file_vm_ops = { .page_mkwrite = fuse_page_mkwrite, }; +/* + * Shared-writable VMAs under the writeback cache with DLM: no + * ->map_pages. Fault-around installs PTEs for any resident folio in + * its window without calling ->fault, so a folio read in under a READ + * grant (read(2), readahead) could take its first write in + * ->page_mkwrite with no WRITE grant ever requested -- and + * page_mkwrite cannot take one there (no retry protocol; see + * fuse_page_mkwrite()). Without fault-around, every first touch goes + * through fuse_filemap_fault(), which records a WRITE grant for the + * batch window before any PTE is installed. + */ +static const struct vm_operations_struct fuse_file_wb_dlm_vm_ops = { + .close = fuse_vma_close, + .fault = fuse_filemap_fault, + .page_mkwrite = fuse_page_mkwrite, +}; + static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; @@ -3987,7 +4067,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) } file_accessed(file); - vma->vm_ops = &fuse_file_vm_ops; + if (fc->writeback_cache && fc->dlm && + (vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + vma->vm_ops = &fuse_file_wb_dlm_vm_ops; + else + vma->vm_ops = &fuse_file_vm_ops; return 0; }