From fc08cf9a2fe51d4726ad54f442693cf86efd6474 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Fri, 9 Oct 2026 00:40:33 +0000 Subject: [PATCH] fuse: throttle foreground io-uring requests with the writeback cache With the writeback cache, foreground requests can wait in userspace on DLM revokes. A revoke waits on page invalidation, which waits on writeback. If foreground requests held every entry of an io-uring queue, background writeback could not be sent and none of them would complete. When the connection has the writeback cache, each queue now keeps a share of its entries for background requests. Background requests may use a quarter of the entries, but at least 2. Foreground requests may use the rest, but at least 8. A foreground request over the limit waits on the new fuse_req_fg_queue and only moves to fuse_req_queue when a slot frees up. Background requests keep their own path to free entries and are held to their cap, except that each queue may always have one active background request. If a queue has fewer than 10 entries both minimums cannot hold, and the foreground minimum wins. The foreground count is released on completion, on commit errors and on entry teardown. Requests marked no_fg_limit skip the foreground limit. These are the page cache reads and writes (redfs_do_readfolio, redfs_send_readpages and redfs_send_write_pages) and the flush from write_inode. A page fault holds the inode range lock while its read is outstanding, and the revoke that is waiting for that range lock would never finish if the read were queued behind other foreground requests. The flush is sent by writeback itself. Without the writeback cache, dispatch is unchanged. Signed-off-by: Hai Zhong Zhou --- fs/fuse/dev_uring.c | 141 +++++++++++++++++++++++++++++++++++++++--- fs/fuse/dev_uring_i.h | 13 ++++ fs/fuse/dir.c | 2 + fs/fuse/file.c | 3 + fs/fuse/fuse_i.h | 3 + 5 files changed, 155 insertions(+), 7 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 71249d1119533c..8ab950418259db 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -100,6 +100,67 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } +/* + * With the writeback cache, foreground requests may hold at most half of the + * entries. They might wait in userspace on DLM revokes, which wait on page + * invalidation, which waits on writeback - if they held all entries, + * background writeback could not be sent and none of them would complete. + */ +#define FUSE_URING_MIN_FOREGROUND 8U +#define FUSE_URING_MIN_BACKGROUND 2U + +static unsigned int fuse_uring_max_background(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents / 4, FUSE_URING_MIN_BACKGROUND); +} + +static unsigned int fuse_uring_max_foreground(struct fuse_ring_queue *queue) +{ + unsigned int bg = fuse_uring_max_background(queue); + + if (queue->nr_ents <= bg) { + pr_warn_once("fuse-io-uring configuration value is too small, could cause hung issue\n"); + return queue->nr_ents; + } + + return queue->nr_ents - bg; +} + +static bool fuse_uring_req_fg_limited(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !(req->args && req->args->no_fg_limit); +} + +static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) +{ + lockdep_assert_held(&queue->lock); + + while (queue->active_foreground < fuse_uring_max_foreground(queue) && + !list_empty(&queue->fuse_req_fg_queue)) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_fg_queue, + struct fuse_req, list); + set_bit(FR_URING_FG, &req->flags); + queue->active_foreground++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_end_foreground(struct fuse_ring_queue *queue, + struct fuse_req *req) +{ + lockdep_assert_held(&queue->lock); + + if (test_and_clear_bit(FR_URING_FG, &req->flags)) { + queue->active_foreground--; + fuse_uring_flush_queue_fg(queue); + } +} + static void __fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { @@ -112,6 +173,7 @@ static void __fuse_uring_req_end(struct fuse_ring_ent *ent, ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); + fuse_uring_end_foreground(queue, req); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); @@ -171,9 +233,15 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) LIST_HEAD(req_list); spin_lock(&queue->lock); - list_for_each_entry(req, &queue->fuse_req_queue, list) + list_for_each_entry(req, &queue->fuse_req_queue, list) { + clear_bit(FR_PENDING, &req->flags); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; + } + list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); list_splice_init(&queue->fuse_req_queue, &req_list); + list_splice_init(&queue->fuse_req_fg_queue, &req_list); queue->nr_reqs = 0; spin_unlock(&queue->lock); @@ -237,6 +305,7 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) spin_lock(&queue->lock); if (fuse_request_expired(fc, &queue->fuse_req_queue) || fuse_request_expired(fc, &queue->fuse_req_bg_queue) || + fuse_request_expired(fc, &queue->fuse_req_fg_queue) || ent_list_request_expired(fc, &queue->ent_w_req_queue) || ent_list_request_expired(fc, &queue->ent_in_userspace)) { spin_unlock(&queue->lock); @@ -461,6 +530,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); queue->fpq.processing = pq; @@ -524,7 +594,10 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; } + queue->nr_ents--; /* * The entry must not be freed immediately, due to access of direct @@ -1138,6 +1211,8 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -1151,6 +1226,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; + struct fuse_ring_ent *avail; + struct fuse_req *next = NULL; err = -ENOTCONN; if (!ring) @@ -1189,7 +1266,32 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + if (test_bit(FR_BACKGROUND, &req->flags)) { + queue->active_background--; + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); + } + + /* + * The freed foreground or background slot might have moved a + * queued request to fuse_req_queue. This entry is not usable, + * so hand the request to another available entry. + */ + avail = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (avail) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(avail, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(avail, true); + req->out.h.error = err; clear_bit(FR_SENT, &req->flags); fuse_request_end(req); @@ -1434,6 +1536,8 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; + queue->nr_ents++; + fuse_uring_flush_queue_fg(queue); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1670,6 +1774,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; + struct fuse_req *next; int err; err = -EINVAL; @@ -1684,18 +1789,25 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); queue->nr_reqs++; - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else + if (fuse_uring_req_fg_limited(req)) { + list_add_tail(&req->list, &queue->fuse_req_fg_queue); + fuse_uring_flush_queue_fg(queue); + } else { list_add_tail(&req->list, &queue->fuse_req_queue); + } + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + next = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); - if (ent) + if (next) fuse_uring_dispatch_ent(ent, false); return; @@ -1762,12 +1874,27 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; bool removed = fuse_remove_pending_req(req, &queue->lock); + struct fuse_ring_ent *ent; + struct fuse_req *next = NULL; if (removed) { /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + + /* the freed foreground slot might let a queued request run */ + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (ent) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(ent, true); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 667f668ef3277f..4f86f64e801777 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -98,6 +98,19 @@ struct fuse_ring_queue { /* background fuse requests */ struct list_head fuse_req_bg_queue; + /* foreground fuse requests waiting for the foreground limit */ + struct list_head fuse_req_fg_queue; + + /* number of registered ring entries */ + unsigned int nr_ents; + + /* + * foreground requests on fuse_req_queue or in userspace, limited to + * the entries not reserved for background requests with writeback cache; + * see fuse_uring_max_foreground() + */ + unsigned int active_foreground; + /* number of requests queued or in userspace */ unsigned int nr_reqs; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 4553980c1461a8..fee1864d73d47b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2100,6 +2100,8 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) if (fm->fc->setattr_writeback) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); + /* might be called from writeback, like a background request */ + args.no_fg_limit = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 748260f92e7cc7..1154e4acb3ca05 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1012,6 +1012,7 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, + .ap.args.no_fg_limit = true, .ap.num_folios = 1, .ap.folios = &folio, .ap.descs = &desc, @@ -1164,6 +1165,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; + ap->args.no_fg_limit = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1593,6 +1595,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + ap->args.no_fg_limit = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 5c403f0b71d21f..e0fd2418567e5c 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -432,6 +432,7 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + bool no_fg_limit:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -520,6 +521,7 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring + * FR_URING_FG: request is counted in the io-uring queue foreground limit */ enum fuse_req_flag { FR_ISREPLY, @@ -536,6 +538,7 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_URING_FG, }; /**