From b63a9a8f8091e5ff61d6bf6ef5a3bcb0882b1859 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Sat, 10 Oct 2026 09:03:53 +0000 Subject: [PATCH] fuse: reserve io-uring entries for critical requests with writeback cache With the writeback cache, a request can wait in userspace on a DLM revoke. The revoke waits on page invalidation on this node, which waits on writeback and on locked pages being read. Those page reads and writes are themselves fuse requests. If requests that may block in userspace held every entry of an io-uring queue, the page I/O could never be sent and none of the requests would complete. Add a uring_critical flag to fuse_args for requests on the path that invalidation waits for. These are the page reads in fuse_do_readfolio() and fuse_send_readpages() (both sync and async readahead), the page writes in fuse_send_write_pages() and fuse_send_writepage(), and fuse_flush_times(), which writeback calls. The daemon must never block a critical request on a DLM revoke that waits on page invalidation on this node. When the connection has the writeback cache, a quarter of the entries of each queue, but at least 2, are critical entries. Non-critical requests, foreground and background together, may hold only the other entries, counted in active_noncritical. A queue always keeps at least one entry for non-critical requests, and at least one critical entry when it has 2 or more entries. Critical requests are not counted in that limit and may use any entry. A non-critical foreground request over the limit waits on the new fuse_req_fg_queue. A non-critical background request waits on fuse_req_bg_queue, now limited by both the background limits and the non-critical limit. Critical background requests wait on the new fuse_req_bg_crit_queue, so they never sit behind non-critical ones, and are admitted first. They are still held to max_background, except that each queue may always have one critical background request active. Critical foreground requests go straight to fuse_req_queue. When a non-critical slot is freed, it is offered first to the same kind of request, foreground or background, that freed it, so neither kind can starve the other. Registering a new entry raises the limit and admits waiting requests of both kinds. The counts are released through one helper, fuse_uring_end_active(), on completion, on commit errors and on request removal. Entry teardown and queue abort also drop the non-critical count. Without the writeback cache, dispatch is unchanged. Signed-off-by: Hai Zhong Zhou --- fs/fuse/dev.c | 13 +- fs/fuse/dev_uring.c | 295 ++++++++++++++++++++++++++---------------- fs/fuse/dev_uring_i.h | 22 +++- fs/fuse/dir.c | 2 +- fs/fuse/file.c | 7 +- fs/fuse/fuse_i.h | 23 +++- 6 files changed, 240 insertions(+), 122 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index e2e68e7b39ded1..2d2cf732a93728 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -764,10 +764,21 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, return -ENOMEM; __set_bit(FR_BACKGROUND, &req->flags); } else { + /* + * A critical request must not wait for the background budget: + * async readahead holds its folios locked here, and the budget + * might be held by requests blocked in userspace on a DLM revoke + * that waits for those folios. It still counts in the budget, + * and the limits on active background requests still apply. + */ + bool critical = args->uring_critical && fm->fc->writeback_cache; + WARN_ON(args->nocreds); - req = fuse_get_req(&invalid_mnt_idmap, fm, true); + req = fuse_get_req(&invalid_mnt_idmap, fm, !critical); if (IS_ERR(req)) return PTR_ERR(req); + if (critical) + __set_bit(FR_BACKGROUND, &req->flags); } fuse_args_to_req(req, args); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 8ab950418259db..4b64045e027ae6 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -72,6 +72,46 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) return pdu->ent; } +/* + * With the writeback cache, some entries of a queue are critical entries, used + * only by uring_critical requests. Other requests, foreground and background, + * might wait in userspace on DLM revokes, which wait on page invalidation, + * which waits on writeback and page reads - if they held all entries, the + * critical requests could not be sent and none of them would complete. + * + * The critical entries are a quarter of the entries, but at least 2. + * Non-critical requests share the rest, but always keep at least one entry, + * and always leave at least one critical entry when there are 2 or more. + */ +#define FUSE_URING_MIN_CRITICAL 2U + +static unsigned int fuse_uring_nr_critical(struct fuse_ring_queue *queue) +{ + if (queue->nr_ents < 2) + return 0; + + return min(max(queue->nr_ents / 4, FUSE_URING_MIN_CRITICAL), + queue->nr_ents - 1); +} + +static unsigned int fuse_uring_max_noncritical(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents - fuse_uring_nr_critical(queue), 1U); +} + +static bool fuse_uring_req_critical(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + req->args && req->args->uring_critical; +} + +static bool fuse_uring_req_fg_noncritical(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !fuse_uring_req_critical(req); +} + static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) { struct fuse_ring *ring = queue->ring; @@ -85,101 +125,119 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) * This prevents a single queue from consuming all resources and * eliminates the need for remote queue wake-ups when global * limits are met but this queue has no more waiting requests. + * + * Critical bg requests wait on their own list, so they never wait + * behind non-critical bg requests, and are admitted first. They are + * subject to the bg limits, but one critical bg request per queue is + * always allowed. It is not counted in the non-critical limit and can + * use the critical entries, so critical bg requests always progress. + * + * With the writeback cache, non-critical bg requests also share the + * non-critical limit with foreground requests, see + * fuse_uring_max_noncritical(). */ while ((fc->active_background < fc->max_background || - !queue->active_background) && - (!list_empty(&queue->fuse_req_bg_queue))) { + !queue->active_bg_critical) && + !list_empty(&queue->fuse_req_bg_crit_queue)) { struct fuse_req *req; - req = list_first_entry(&queue->fuse_req_bg_queue, + req = list_first_entry(&queue->fuse_req_bg_crit_queue, struct fuse_req, list); fc->active_background++; queue->active_background++; + set_bit(FR_URING_CRITICAL, &req->flags); + queue->active_bg_critical++; list_move_tail(&req->list, &queue->fuse_req_queue); } -} -/* - * With the writeback cache, foreground requests may hold at most half of the - * entries. They might wait in userspace on DLM revokes, which wait on page - * invalidation, which waits on writeback - if they held all entries, - * background writeback could not be sent and none of them would complete. - */ -#define FUSE_URING_MIN_FOREGROUND 8U -#define FUSE_URING_MIN_BACKGROUND 2U - -static unsigned int fuse_uring_max_background(struct fuse_ring_queue *queue) -{ - return max(queue->nr_ents / 4, FUSE_URING_MIN_BACKGROUND); -} + while ((fc->active_background < fc->max_background || + !queue->active_background) && + (!fc->writeback_cache || queue->stopped || + queue->active_noncritical < fuse_uring_max_noncritical(queue)) && + (!list_empty(&queue->fuse_req_bg_queue))) { + struct fuse_req *req; -static unsigned int fuse_uring_max_foreground(struct fuse_ring_queue *queue) -{ - unsigned int bg = fuse_uring_max_background(queue); + req = list_first_entry(&queue->fuse_req_bg_queue, + struct fuse_req, list); + fc->active_background++; + queue->active_background++; + if (fc->writeback_cache) { + set_bit(FR_URING_NONCRIT, &req->flags); + queue->active_noncritical++; + } - if (queue->nr_ents <= bg) { - pr_warn_once("fuse-io-uring configuration value is too small, could cause hung issue\n"); - return queue->nr_ents; + list_move_tail(&req->list, &queue->fuse_req_queue); } - - return queue->nr_ents - bg; -} - -static bool fuse_uring_req_fg_limited(struct fuse_req *req) -{ - return req->fm->fc->writeback_cache && - !test_bit(FR_BACKGROUND, &req->flags) && - !(req->args && req->args->no_fg_limit); } static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) { lockdep_assert_held(&queue->lock); - while (queue->active_foreground < fuse_uring_max_foreground(queue) && + while (queue->active_noncritical < fuse_uring_max_noncritical(queue) && !list_empty(&queue->fuse_req_fg_queue)) { struct fuse_req *req; req = list_first_entry(&queue->fuse_req_fg_queue, struct fuse_req, list); - set_bit(FR_URING_FG, &req->flags); - queue->active_foreground++; + set_bit(FR_URING_NONCRIT, &req->flags); + queue->active_noncritical++; list_move_tail(&req->list, &queue->fuse_req_queue); } } -static void fuse_uring_end_foreground(struct fuse_ring_queue *queue, - struct fuse_req *req) +/* + * Undo the queue counting of an admitted request, and admit waiting requests + * into the freed slots. Requests that were never admitted are not counted. + */ +static void fuse_uring_end_active(struct fuse_ring_queue *queue, + struct fuse_req *req) { + struct fuse_conn *fc = queue->ring->fc; + bool noncritical, background; + lockdep_assert_held(&queue->lock); - if (test_and_clear_bit(FR_URING_FG, &req->flags)) { - queue->active_foreground--; + noncritical = test_and_clear_bit(FR_URING_NONCRIT, &req->flags); + if (noncritical) + queue->active_noncritical--; + + background = test_bit(FR_BACKGROUND, &req->flags); + if (background) { + queue->active_background--; + if (test_and_clear_bit(FR_URING_CRITICAL, &req->flags)) + queue->active_bg_critical--; + } + + /* + * The non-critical limit is shared by foreground and bg requests. A + * freed non-critical slot is offered first to the kind of request that + * freed it, so under load neither kind can starve the other. + */ + if (noncritical && !background) fuse_uring_flush_queue_fg(queue); + if (noncritical || background) { + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); } + if (noncritical && background) + fuse_uring_flush_queue_fg(queue); } static void __fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; lockdep_assert_not_held(&queue->lock); spin_lock(&queue->lock); ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); - fuse_uring_end_foreground(queue, req); - if (test_bit(FR_BACKGROUND, &req->flags)) { - queue->active_background--; - spin_lock(&fc->bg_lock); - fuse_uring_flush_queue_bg(queue); - spin_unlock(&fc->bg_lock); - } + fuse_uring_end_active(queue, req); spin_unlock(&queue->lock); if (error) @@ -235,8 +293,8 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) spin_lock(&queue->lock); list_for_each_entry(req, &queue->fuse_req_queue, list) { clear_bit(FR_PENDING, &req->flags); - if (test_and_clear_bit(FR_URING_FG, &req->flags)) - queue->active_foreground--; + if (test_and_clear_bit(FR_URING_NONCRIT, &req->flags)) + queue->active_noncritical--; } list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); @@ -305,6 +363,7 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) spin_lock(&queue->lock); if (fuse_request_expired(fc, &queue->fuse_req_queue) || fuse_request_expired(fc, &queue->fuse_req_bg_queue) || + fuse_request_expired(fc, &queue->fuse_req_bg_crit_queue) || fuse_request_expired(fc, &queue->fuse_req_fg_queue) || ent_list_request_expired(fc, &queue->ent_w_req_queue) || ent_list_request_expired(fc, &queue->ent_in_userspace)) { @@ -530,6 +589,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_bg_crit_queue); INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); @@ -594,8 +654,8 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); - if (test_and_clear_bit(FR_URING_FG, &req->flags)) - queue->active_foreground--; + if (test_and_clear_bit(FR_URING_NONCRIT, &req->flags)) + queue->active_noncritical--; } queue->nr_ents--; @@ -1173,6 +1233,40 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, fuse_uring_req_end_deferred(ent, req, err, issue_flags); } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + +/* + * Hand each request on fuse_req_queue to an available entry. Freeing a slot + * or background budget can admit more than one request at once, and every one + * of them needs an entry: a request left on fuse_req_queue next to an idle + * entry waits for the next event on this queue, which never comes when the + * requests in userspace are all blocked on a DLM revoke. + * + * Must be called without the queue lock. The entries are sent in task context, + * as the caller might hold the io_uring submission lock. + */ +static void fuse_uring_dispatch_pending(struct fuse_ring_queue *queue) +{ + struct fuse_ring_ent *ent; + struct fuse_req *req; + + for (;;) { + spin_lock(&queue->lock); + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + req = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (req) + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + + if (!req) + return; + + fuse_uring_dispatch_ent(ent, true); + } +} + /* * Get the next fuse req and send it */ @@ -1182,11 +1276,15 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, { int err; struct fuse_req *req; + bool more; retry: spin_lock(&queue->lock); fuse_uring_ent_avail(ent, queue); req = fuse_uring_ent_assign_req(ent); + /* the request that ended might have admitted more than this one */ + more = !list_empty(&queue->fuse_req_queue) && + !list_empty(&queue->ent_avail_queue); spin_unlock(&queue->lock); if (req) { @@ -1194,6 +1292,9 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, if (err) goto retry; } + + if (more) + fuse_uring_dispatch_pending(queue); } static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) @@ -1211,8 +1312,6 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } -static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); - /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -1226,8 +1325,6 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; - struct fuse_ring_ent *avail; - struct fuse_req *next = NULL; err = -ENOTCONN; if (!ring) @@ -1244,6 +1341,17 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped)) return err; + /* + * With a single entry no entry is critical, so a non-critical request + * blocked in userspace on a DLM revoke can hold the only entry, see + * fuse_uring_nr_critical(). Checked on commit rather than on register, + * as a queue has a single entry while its entries are being registered, + * and libfuse registers all of them before it commits a reply. + */ + if (fc->writeback_cache && READ_ONCE(queue->nr_ents) < 2) + pr_warn_once("fuse-io-uring: qid=%u has a single entry, page I/O might hang behind a request blocked in userspace; use an io-uring queue depth of 2 or more\n", + qid); + spin_lock(&queue->lock); /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list @@ -1267,30 +1375,16 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); queue->nr_reqs--; - fuse_uring_end_foreground(queue, req); - if (test_bit(FR_BACKGROUND, &req->flags)) { - queue->active_background--; - spin_lock(&fc->bg_lock); - fuse_uring_flush_queue_bg(queue); - spin_unlock(&fc->bg_lock); - } + fuse_uring_end_active(queue, req); - /* - * The freed foreground or background slot might have moved a - * queued request to fuse_req_queue. This entry is not usable, - * so hand the request to another available entry. - */ - avail = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); - if (avail) - next = list_first_entry_or_null(&queue->fuse_req_queue, - struct fuse_req, list); - if (next) - fuse_uring_add_req_to_ring_ent(avail, next); spin_unlock(&queue->lock); - if (next) - fuse_uring_dispatch_ent(avail, true); + /* + * The freed non-critical or background slot might have moved + * queued requests to fuse_req_queue. This entry is not usable, + * so hand them to the available entries. + */ + fuse_uring_dispatch_pending(queue); req->out.h.error = err; clear_bit(FR_SENT, &req->flags); @@ -1537,7 +1631,11 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; queue->nr_ents++; + /* the new entry might raise the non-critical limit */ fuse_uring_flush_queue_fg(queue); + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1791,7 +1889,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) req->ring_queue = queue; queue->nr_reqs++; - if (fuse_uring_req_fg_limited(req)) { + if (fuse_uring_req_fg_noncritical(req)) { list_add_tail(&req->list, &queue->fuse_req_fg_queue); fuse_uring_flush_queue_fg(queue); } else { @@ -1825,7 +1923,6 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) struct fuse_conn *fc = req->fm->fc; struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; - struct fuse_ring_ent *ent = NULL; queue = fuse_uring_select_queue(ring, true); if (!queue) @@ -1839,33 +1936,26 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - list_add_tail(&req->list, &queue->fuse_req_bg_queue); + if (fuse_uring_req_critical(req)) + list_add_tail(&req->list, &queue->fuse_req_bg_crit_queue); + else + list_add_tail(&req->list, &queue->fuse_req_bg_queue); queue->nr_reqs++; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); spin_lock(&fc->bg_lock); fc->num_background++; if (fc->num_background == fc->max_background) fc->blocked = 1; fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); + spin_unlock(&queue->lock); /* * Due to bg_queue flush limits there might be other bg requests - * in the queue that need to be handled first. Or no further req - * might be available. + * in the queue that need to be handled first, or this one might have + * to wait. Budget freed on other queues might also admit several. */ - req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, - list); - if (ent && req) { - fuse_uring_add_req_to_ring_ent(ent, req); - spin_unlock(&queue->lock); - - fuse_uring_dispatch_ent(ent, true); - } else { - spin_unlock(&queue->lock); - } + fuse_uring_dispatch_pending(queue); return true; } @@ -1874,27 +1964,16 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; bool removed = fuse_remove_pending_req(req, &queue->lock); - struct fuse_ring_ent *ent; - struct fuse_req *next = NULL; if (removed) { /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; - fuse_uring_end_foreground(queue, req); - - /* the freed foreground slot might let a queued request run */ - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); - if (ent) - next = list_first_entry_or_null(&queue->fuse_req_queue, - struct fuse_req, list); - if (next) - fuse_uring_add_req_to_ring_ent(ent, next); + fuse_uring_end_active(queue, req); spin_unlock(&queue->lock); - if (next) - fuse_uring_dispatch_ent(ent, true); + /* the freed non-critical slot might let queued requests run */ + fuse_uring_dispatch_pending(queue); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 4f86f64e801777..1071f155e2b042 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -95,21 +95,25 @@ struct fuse_ring_queue { /* fuse requests waiting for an entry slot */ struct list_head fuse_req_queue; - /* background fuse requests */ + /* non-critical background fuse requests */ struct list_head fuse_req_bg_queue; - /* foreground fuse requests waiting for the foreground limit */ + /* uring_critical background fuse requests */ + struct list_head fuse_req_bg_crit_queue; + + /* non-critical foreground fuse requests waiting for the non-critical limit */ struct list_head fuse_req_fg_queue; /* number of registered ring entries */ unsigned int nr_ents; /* - * foreground requests on fuse_req_queue or in userspace, limited to - * the entries not reserved for background requests with writeback cache; - * see fuse_uring_max_foreground() + * non-critical requests, foreground and background, on fuse_req_queue + * or in userspace. With the writeback cache they are limited to the + * entries that are not critical entries, which are used only by + * uring_critical requests, see fuse_uring_max_noncritical(). */ - unsigned int active_foreground; + unsigned int active_noncritical; /* number of requests queued or in userspace */ unsigned int nr_reqs; @@ -118,6 +122,12 @@ struct fuse_ring_queue { unsigned int active_background; + /* + * uring_critical background requests on fuse_req_queue or in + * userspace, one is always allowed regardless of the bg limits + */ + unsigned int active_bg_critical; + bool stopped; }; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index fee1864d73d47b..496edb9df8d188 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2101,7 +2101,7 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); /* might be called from writeback, like a background request */ - args.no_fg_limit = true; + args.uring_critical = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 1154e4acb3ca05..5168d8cebe3d86 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1012,7 +1012,7 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, - .ap.args.no_fg_limit = true, + .ap.args.uring_critical = true, .ap.num_folios = 1, .ap.folios = &folio, .ap.descs = &desc, @@ -1165,7 +1165,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; - ap->args.no_fg_limit = true; + ap->args.uring_critical = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1595,7 +1595,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; - ap->args.no_fg_limit = true; + ap->args.uring_critical = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; @@ -2923,6 +2923,7 @@ __acquires(fi->lock) args->in_args[1].size = inarg->size; args->force = true; args->nocreds = true; + args->uring_critical = true; err = fuse_simple_background(fm, args, GFP_ATOMIC); if (err == -ENOMEM) { diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index e0fd2418567e5c..846a4625edd670 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -432,7 +432,20 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; - bool no_fg_limit:1; + /* + * Request is on the path invalidation waits for: the page reads in + * fuse_do_readfolio() and fuse_send_readpages(), the page writes in + * fuse_send_write_pages() and fuse_send_writepage(), and + * fuse_flush_times(), which writeback calls. With the writeback cache, + * it must never be held back behind requests that may be blocked in + * userspace on a DLM revoke: foreground or background, it is exempt + * from the io-uring entry limits and may use the critical entries, see + * fuse_uring_req_critical(), and a background one does not wait for + * the background budget, see fuse_simple_background(). Critical + * requests are not limited, so the daemon must never block them on a + * DLM revoke that waits on page invalidation on this node. + */ + bool uring_critical:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -521,7 +534,10 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring - * FR_URING_FG: request is counted in the io-uring queue foreground limit + * FR_URING_NONCRIT: request is counted in the io-uring queue non-critical limit, + * shared by non-critical foreground and background requests + * FR_URING_CRITICAL: critical background request counted in the io-uring + * queue active critical background requests */ enum fuse_req_flag { FR_ISREPLY, @@ -538,7 +554,8 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, - FR_URING_FG, + FR_URING_NONCRIT, + FR_URING_CRITICAL, }; /**