From fc08cf9a2fe51d4726ad54f442693cf86efd6474 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Fri, 9 Oct 2026 00:40:33 +0000 Subject: [PATCH 1/3] fuse: throttle foreground io-uring requests with the writeback cache With the writeback cache, foreground requests can wait in userspace on DLM revokes. A revoke waits on page invalidation, which waits on writeback. If foreground requests held every entry of an io-uring queue, background writeback could not be sent and none of them would complete. When the connection has the writeback cache, each queue now keeps a share of its entries for background requests. Background requests may use a quarter of the entries, but at least 2. Foreground requests may use the rest, but at least 8. A foreground request over the limit waits on the new fuse_req_fg_queue and only moves to fuse_req_queue when a slot frees up. Background requests keep their own path to free entries and are held to their cap, except that each queue may always have one active background request. If a queue has fewer than 10 entries both minimums cannot hold, and the foreground minimum wins. The foreground count is released on completion, on commit errors and on entry teardown. Requests marked no_fg_limit skip the foreground limit. These are the page cache reads and writes (redfs_do_readfolio, redfs_send_readpages and redfs_send_write_pages) and the flush from write_inode. A page fault holds the inode range lock while its read is outstanding, and the revoke that is waiting for that range lock would never finish if the read were queued behind other foreground requests. The flush is sent by writeback itself. Without the writeback cache, dispatch is unchanged. Signed-off-by: Hai Zhong Zhou --- fs/fuse/dev_uring.c | 141 +++++++++++++++++++++++++++++++++++++++--- fs/fuse/dev_uring_i.h | 13 ++++ fs/fuse/dir.c | 2 + fs/fuse/file.c | 3 + fs/fuse/fuse_i.h | 3 + 5 files changed, 155 insertions(+), 7 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 71249d1119533c..8ab950418259db 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -100,6 +100,67 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } +/* + * With the writeback cache, foreground requests may hold at most half of the + * entries. They might wait in userspace on DLM revokes, which wait on page + * invalidation, which waits on writeback - if they held all entries, + * background writeback could not be sent and none of them would complete. + */ +#define FUSE_URING_MIN_FOREGROUND 8U +#define FUSE_URING_MIN_BACKGROUND 2U + +static unsigned int fuse_uring_max_background(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents / 4, FUSE_URING_MIN_BACKGROUND); +} + +static unsigned int fuse_uring_max_foreground(struct fuse_ring_queue *queue) +{ + unsigned int bg = fuse_uring_max_background(queue); + + if (queue->nr_ents <= bg) { + pr_warn_once("fuse-io-uring configuration value is too small, could cause hung issue\n"); + return queue->nr_ents; + } + + return queue->nr_ents - bg; +} + +static bool fuse_uring_req_fg_limited(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !(req->args && req->args->no_fg_limit); +} + +static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) +{ + lockdep_assert_held(&queue->lock); + + while (queue->active_foreground < fuse_uring_max_foreground(queue) && + !list_empty(&queue->fuse_req_fg_queue)) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_fg_queue, + struct fuse_req, list); + set_bit(FR_URING_FG, &req->flags); + queue->active_foreground++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_end_foreground(struct fuse_ring_queue *queue, + struct fuse_req *req) +{ + lockdep_assert_held(&queue->lock); + + if (test_and_clear_bit(FR_URING_FG, &req->flags)) { + queue->active_foreground--; + fuse_uring_flush_queue_fg(queue); + } +} + static void __fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { @@ -112,6 +173,7 @@ static void __fuse_uring_req_end(struct fuse_ring_ent *ent, ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); + fuse_uring_end_foreground(queue, req); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); @@ -171,9 +233,15 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) LIST_HEAD(req_list); spin_lock(&queue->lock); - list_for_each_entry(req, &queue->fuse_req_queue, list) + list_for_each_entry(req, &queue->fuse_req_queue, list) { + clear_bit(FR_PENDING, &req->flags); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; + } + list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); list_splice_init(&queue->fuse_req_queue, &req_list); + list_splice_init(&queue->fuse_req_fg_queue, &req_list); queue->nr_reqs = 0; spin_unlock(&queue->lock); @@ -237,6 +305,7 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) spin_lock(&queue->lock); if (fuse_request_expired(fc, &queue->fuse_req_queue) || fuse_request_expired(fc, &queue->fuse_req_bg_queue) || + fuse_request_expired(fc, &queue->fuse_req_fg_queue) || ent_list_request_expired(fc, &queue->ent_w_req_queue) || ent_list_request_expired(fc, &queue->ent_in_userspace)) { spin_unlock(&queue->lock); @@ -461,6 +530,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); queue->fpq.processing = pq; @@ -524,7 +594,10 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; } + queue->nr_ents--; /* * The entry must not be freed immediately, due to access of direct @@ -1138,6 +1211,8 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -1151,6 +1226,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; + struct fuse_ring_ent *avail; + struct fuse_req *next = NULL; err = -ENOTCONN; if (!ring) @@ -1189,7 +1266,32 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + if (test_bit(FR_BACKGROUND, &req->flags)) { + queue->active_background--; + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); + } + + /* + * The freed foreground or background slot might have moved a + * queued request to fuse_req_queue. This entry is not usable, + * so hand the request to another available entry. + */ + avail = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (avail) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(avail, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(avail, true); + req->out.h.error = err; clear_bit(FR_SENT, &req->flags); fuse_request_end(req); @@ -1434,6 +1536,8 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; + queue->nr_ents++; + fuse_uring_flush_queue_fg(queue); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1670,6 +1774,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; + struct fuse_req *next; int err; err = -EINVAL; @@ -1684,18 +1789,25 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); queue->nr_reqs++; - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else + if (fuse_uring_req_fg_limited(req)) { + list_add_tail(&req->list, &queue->fuse_req_fg_queue); + fuse_uring_flush_queue_fg(queue); + } else { list_add_tail(&req->list, &queue->fuse_req_queue); + } + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + next = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); - if (ent) + if (next) fuse_uring_dispatch_ent(ent, false); return; @@ -1762,12 +1874,27 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; bool removed = fuse_remove_pending_req(req, &queue->lock); + struct fuse_ring_ent *ent; + struct fuse_req *next = NULL; if (removed) { /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + + /* the freed foreground slot might let a queued request run */ + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (ent) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(ent, true); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 667f668ef3277f..4f86f64e801777 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -98,6 +98,19 @@ struct fuse_ring_queue { /* background fuse requests */ struct list_head fuse_req_bg_queue; + /* foreground fuse requests waiting for the foreground limit */ + struct list_head fuse_req_fg_queue; + + /* number of registered ring entries */ + unsigned int nr_ents; + + /* + * foreground requests on fuse_req_queue or in userspace, limited to + * the entries not reserved for background requests with writeback cache; + * see fuse_uring_max_foreground() + */ + unsigned int active_foreground; + /* number of requests queued or in userspace */ unsigned int nr_reqs; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 4553980c1461a8..fee1864d73d47b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2100,6 +2100,8 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) if (fm->fc->setattr_writeback) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); + /* might be called from writeback, like a background request */ + args.no_fg_limit = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 748260f92e7cc7..1154e4acb3ca05 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1012,6 +1012,7 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, + .ap.args.no_fg_limit = true, .ap.num_folios = 1, .ap.folios = &folio, .ap.descs = &desc, @@ -1164,6 +1165,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; + ap->args.no_fg_limit = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1593,6 +1595,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + ap->args.no_fg_limit = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 5c403f0b71d21f..e0fd2418567e5c 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -432,6 +432,7 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + bool no_fg_limit:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -520,6 +521,7 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring + * FR_URING_FG: request is counted in the io-uring queue foreground limit */ enum fuse_req_flag { FR_ISREPLY, @@ -536,6 +538,7 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_URING_FG, }; /** From 6dd966b890721069e4b1b0f97b7c96f87331e696 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Sat, 10 Oct 2026 08:42:16 -0700 Subject: [PATCH 2/3] fuse: upgrade only the granted subrange in the DLM grant cache fuse_dlm_lock_range() flipped the mode of every READ range overlapping a newly recorded WRITE grant in its entirety, not just the part the grant covers. A whole-file READ grant -- read(2) leaves one behind via readahead -- overlapped by any WRITE grant was thus recorded as a whole-file WRITE grant, and fuse_dlm_lock_is_held() then reported WRITE coverage the server never gave. Every write-side check trusts that record: the fault fast path, fuse_get_dlm_lock()'s early exit, and the write path's re-validation all see the range as covered, no lock request reaches the server, and writeback sends WRITEs the server observes under a read lock. Split the overlapping READ range at the grant boundaries and upgrade only the intersection. The splits preserve their mode, so an allocation failure partway through still leaves the tree describing exactly the grants held. Found probing the fault-around fix in the next commit: the probe's first-touch write fault should have requested a WRITE grant for its batch window, but the poisoned cache reported the page already covered and no request reached the server. Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 48 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 573d198b69dccf..c6d51a089736a8 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -177,7 +177,10 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * For overlapping ranges, handle lock compatibility: * - READ locks are compatible with existing READ locks * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * - WRITE locks need to upgrade existing READ locks, but only over the + * intersection with [start, end]: the server granted nothing outside + * it, so the parts of a READ range beyond the grant are split off and + * keep their mode * * Return: 0 on success, negative error code on failure */ @@ -211,6 +214,49 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, /* Check lock compatibility */ if (lock_mode == FUSE_PCACHE_LK_WRITE && lock_mode != range->mode) { + /* + * Upgrade only what the server granted: the grant + * covers [start, end], not the whole of an + * overlapping READ range. Split off the pieces + * outside the grant first -- mode-preserving, so + * an allocation failure below still leaves the + * tree describing exactly the grants held -- and + * upgrade just the intersection. + */ + if (range->start < start) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = range->start; + new_range->end = start - 1; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->start = start; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } + if (range->end > end) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = end + 1; + new_range->end = range->end; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->end = end; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } /* we own the lock but have to update it. */ list_add_tail(&range->list, &to_upgrade); } From 85af4a6a33c7a9518098f4ca4f0b5590fc16c7a2 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Sat, 10 Oct 2026 08:42:17 -0700 Subject: [PATCH 3/3] fuse: drop fault-around for shared-writable mappings under DLM With the writeback cache on, page_mkwrite skipped the DLM lock request entirely, on the assumption that fuse_filemap_fault() had already taken a write grant for any page a shared-writable mapping could dirty. That assumption does not hold: fault-around (filemap_map_pages) PTE-maps any resident folio in its window without calling ->fault for it, so a folio read in under a READ grant -- by read(2) or readahead -- can take its first write fault with no WRITE grant ever requested. The page is dirtied anyway and writeback later sends a WRITE the server sees under a read lock (DFS cases 040, 080, 081, 090 and 200). page_mkwrite cannot take the grant itself: it is called with mmap_lock (or the per-VMA lock) held and has no VM_FAULT_RETRY protocol to drop it, while the buffered write path holds the IO range lock LOCKED while faulting in its source buffer, which takes mmap_lock. With a pending mmap_lock writer queued between the two, blocking in page_mkwrite on the range lock or the DLM round trip recreates the three-party deadlock that 94effa2 ("fuse: do not block the fault path under mmap_lock") removed from the fault path. Instead, give shared-writable VMAs under the writeback cache with DLM a vm_operations_struct without ->map_pages, so the first touch of every page goes through fuse_filemap_fault(), which records a WRITE grant for its batch window before any PTE is installed and already drops mmap_lock to block. page_mkwrite keeps a non-blocking check of the recorded grant -- the same LOCKED fence the fault fast path uses -- and returns VM_FAULT_NOPAGE on a miss. A verified miss can only mean a revoke in flight, whose invalidation clears the PTE, so the re-fault resolves through fuse_filemap_fault(); a trylock-contended miss re-faults with mmap_lock released in between, so the range lock holder can make progress. This also makes fuse_filemap_fault()'s FAULT_FLAG_TRIED pass take the drop-and-retry slow path instead of blocking in place: with every first touch of a shared-writable page now funnelled through ->fault, blocking there on the range lock with mmap_lock held steps into page_mkwrite's old role in the same three-party cycle, and did so instantly under a buffered write sourced from an mmap of the file racing munmap and a write fault. Retries may repeat (4064b98 "mm: allow VM_FAULT_RETRY for multiple times"), so waiting for a contended range must happen with mmap_lock dropped, every time; blocking in place remains only for callers that allow no retries at all. The non-writeback-cache arm keeps the existing single-page fuse_get_page_mkwrite_lock() behaviour. Reported-by: Hai Zhong Zhou Suggested-by: Yong Ze Chen Signed-off-by: Allison Henderson --- fs/fuse/file.c | 162 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 39 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 1154e4acb3ca05..0cc69cace86c71 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3381,6 +3381,36 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l } return err; } + +/* + * Batch size for fuse_fault_read_private_folios(): one fault-around + * window (64k with 4k pages), centred on the faulted page. A + * spanning access (one instruction touching the tail of one page and + * the head of the next -- any unaligned copy does this constantly) + * needs BOTH pages mapped at once to retire, and every server round + * trip gets this client's grant revoked and its page cache dropped + * (the server does not exempt the requester, and the notification + * runs one op behind). A one-page grant-and-populate therefore + * ping-pongs forever: mapping each page costs a round trip whose + * revoke drops the other. A window covering both pages makes the + * neighbour's fault a pure fast-path hit -- recorded grant, resident + * folio, NO server op and no new revoke -- and the spanning access + * completes. Centred, not aligned: an aligned window has edges + * (every 16th page boundary), and a spanning access sitting on an + * edge gets two one-page-covering windows and ping-pongs just the + * same (generic/263); a centred window always covers both of the + * faulted page's neighbours, so no boundary has the problem. + */ +#define FUSE_FAULT_BATCH_PAGES 16 + +/* First page of the populate/grant window centred on @index. */ +static pgoff_t fuse_fault_batch_base(pgoff_t index) +{ + if (index < FUSE_FAULT_BATCH_PAGES / 2) + return 0; + return index - FUSE_FAULT_BATCH_PAGES / 2; +} + /* * Wait for writeback against this page to complete before allowing it * to be marked dirty again, and hence written back again, possibly @@ -3402,9 +3432,60 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct file *file = vmf->vma->vm_file; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = vmf->pgoff << PAGE_SHIFT; - if (!fm->fc->writeback_cache && fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; + if (fm->fc->writeback_cache && fm->fc->dlm) { + /* + * Writenotify routes every clean-to-dirty transition of a + * shared mapping through here, but the WRITE grant itself + * is taken by fuse_filemap_fault(): shared-writable VMAs + * get no ->map_pages under DLM (see fuse_file_mmap()), so + * every PTE in such a VMA was installed by a fault that + * recorded a WRITE grant for its batch window. Only + * verify here, and only without blocking -- nothing may + * sleep on the range lock or the server while mmap_lock + * (or the per-VMA lock) is held: the buffered write path + * holds the range lock LOCKED while faulting in its + * source buffer (which takes mmap_lock), so with a + * pending mmap_lock writer queued between the two, + * blocking here deadlocks three ways -- the cycle + * 94effa2 ("fuse: do not block the fault path under + * mmap_lock") removed from the fault path -- and + * page_mkwrite has no VM_FAULT_RETRY protocol to drop + * the lock and come back with. + * + * Same fence as fuse_filemap_fault()'s fast path: only + * once LOCKED can the recorded grant be trusted. On a + * miss, NOPAGE. A contended trylock re-faults with + * mmap_lock released in between, so the holder can make + * progress; a verified miss means a revoke is in flight + * (only an invalidate removes a recorded grant), whose + * unmap_mapping_range() clears the PTE, so the re-fault + * goes through fuse_filemap_fault(), which re-takes the + * grant with the proper drop-and-retry dance. + */ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + bool held = false; + + if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, + pos + PAGE_SIZE - 1, + FUSE_RANGE_LOCK_WRITE)) { + bool locked = + fuse_range_lock_try_mark_locked(fi, &rlock); + + held = locked && + fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, + FUSE_PAGE_LOCK_WRITE); + if (locked && !held) + pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", + fi->nodeid, vmf->pgoff); + fuse_range_lock_release(fi, &rlock); + } + + if (!held) + return VM_FAULT_NOPAGE; + } else if (fm->fc->dlm) { size_t length = PAGE_SIZE; int err = fuse_get_page_mkwrite_lock(file, pos, length); if (err < 0) { @@ -3448,35 +3529,6 @@ static bool fuse_fault_needs_read(struct inode *inode, pgoff_t index) return !uptodate; } -/* - * Batch size for fuse_fault_read_private_folios(): one fault-around - * window (64k with 4k pages), centred on the faulted page. A - * spanning access (one instruction touching the tail of one page and - * the head of the next -- any unaligned copy does this constantly) - * needs BOTH pages mapped at once to retire, and every server round - * trip gets this client's grant revoked and its page cache dropped - * (the server does not exempt the requester, and the notification - * runs one op behind). A one-page grant-and-populate therefore - * ping-pongs forever: mapping each page costs a round trip whose - * revoke drops the other. A window covering both pages makes the - * neighbour's fault a pure fast-path hit -- recorded grant, resident - * folio, NO server op and no new revoke -- and the spanning access - * completes. Centred, not aligned: an aligned window has edges - * (every 16th page boundary), and a spanning access sitting on an - * edge gets two one-page-covering windows and ping-pongs just the - * same (generic/263); a centred window always covers both of the - * faulted page's neighbours, so no boundary has the problem. - */ -#define FUSE_FAULT_BATCH_PAGES 16 - -/* First page of the populate/grant window centred on @index. */ -static pgoff_t fuse_fault_batch_base(pgoff_t index) -{ - if (index < FUSE_FAULT_BATCH_PAGES / 2) - return 0; - return index - FUSE_FAULT_BATCH_PAGES / 2; -} - /* * Read a batch of pages centred on @index into freshly * allocated folios that are NOT in the page cache. Unlike @@ -3707,9 +3759,19 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) * miss and come back here; each pass leaves a recorded * grant behind, which only a real invalidation takes away * again. + * + * FAULT_FLAG_TRIED does not demote a pass to the blocking + * path below: retries may repeat (4064b98 "mm: allow + * VM_FAULT_RETRY for multiple times"), and blocking on + * the range lock with mmap_lock held closes a three-party + * cycle whenever the lock holder is a buffered write + * faulting in its source buffer (which waits on + * mmap_lock) behind a queued mmap_lock writer. A miss on + * the retried fast path only means the range is contended + * again; waiting for it must happen with mmap_lock + * dropped, every time. */ - if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && - !(vmf->flags & FAULT_FLAG_TRIED)) { + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) return VM_FAULT_RETRY; @@ -3763,11 +3825,12 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) } /* - * Retries exhausted or not allowed: block in place. The - * waits are killable, and a same-task fault-in never - * conflicts with its own IO's range lock, so this cannot - * deadlock on itself; it can still stack behind a pending - * mmap_lock writer the way any blocking fault can. + * Retry not allowed (a caller without + * FAULT_FLAG_ALLOW_RETRY): block in place. The waits are + * killable, and a same-task fault-in never conflicts with + * its own IO's range lock, so this cannot deadlock on + * itself; it can still stack behind a pending mmap_lock + * writer the way any blocking fault can. * * Nothing held once the range lock is LOCKED may wait for * the server (see fuse_cache_read_iter), and @@ -3906,6 +3969,23 @@ static const struct vm_operations_struct fuse_file_vm_ops = { .page_mkwrite = fuse_page_mkwrite, }; +/* + * Shared-writable VMAs under the writeback cache with DLM: no + * ->map_pages. Fault-around installs PTEs for any resident folio in + * its window without calling ->fault, so a folio read in under a READ + * grant (read(2), readahead) could take its first write in + * ->page_mkwrite with no WRITE grant ever requested -- and + * page_mkwrite cannot take one there (no retry protocol; see + * fuse_page_mkwrite()). Without fault-around, every first touch goes + * through fuse_filemap_fault(), which records a WRITE grant for the + * batch window before any PTE is installed. + */ +static const struct vm_operations_struct fuse_file_wb_dlm_vm_ops = { + .close = fuse_vma_close, + .fault = fuse_filemap_fault, + .page_mkwrite = fuse_page_mkwrite, +}; + static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; @@ -3985,7 +4065,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) fuse_link_write_file(file); file_accessed(file); - vma->vm_ops = &fuse_file_vm_ops; + if (fc->writeback_cache && fc->dlm && + (vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + vma->vm_ops = &fuse_file_wb_dlm_vm_ops; + else + vma->vm_ops = &fuse_file_vm_ops; return 0; }