diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 6520500d01bc41..8d87f330a6b036 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -649,10 +649,21 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, return -ENOMEM; __set_bit(FR_BACKGROUND, &req->flags); } else { + /* + * A critical request must not wait for the background budget: + * async readahead holds its folios locked here, and the budget + * might be held by requests blocked in userspace on a DLM revoke + * that waits for those folios. It still counts in the budget, + * and the limits on active background requests still apply. + */ + bool critical = args->uring_critical && fm->fc->writeback_cache; + WARN_ON(args->nocreds); - req = fuse_get_req(&invalid_mnt_idmap, fm, true); + req = fuse_get_req(&invalid_mnt_idmap, fm, !critical); if (IS_ERR(req)) return PTR_ERR(req); + if (critical) + __set_bit(FR_BACKGROUND, &req->flags); } fuse_args_to_req(req, args); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 1d372cede3ee69..c80de864d83183 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -71,6 +71,46 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) return pdu->ent; } +/* + * With the writeback cache, some entries of a queue are critical entries, used + * only by uring_critical requests. Other requests, foreground and background, + * might wait in userspace on DLM revokes, which wait on page invalidation, + * which waits on writeback and page reads - if they held all entries, the + * critical requests could not be sent and none of them would complete. + * + * The critical entries are a quarter of the entries, but at least 2. + * Non-critical requests share the rest, but always keep at least one entry, + * and always leave at least one critical entry when there are 2 or more. + */ +#define FUSE_URING_MIN_CRITICAL 2U + +static unsigned int fuse_uring_nr_critical(struct fuse_ring_queue *queue) +{ + if (queue->nr_ents < 2) + return 0; + + return min(max(queue->nr_ents / 4, FUSE_URING_MIN_CRITICAL), + queue->nr_ents - 1); +} + +static unsigned int fuse_uring_max_noncritical(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents - fuse_uring_nr_critical(queue), 1U); +} + +static bool fuse_uring_req_critical(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + req->args && req->args->uring_critical; +} + +static bool fuse_uring_req_fg_noncritical(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !fuse_uring_req_critical(req); +} + static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) { struct fuse_ring *ring = queue->ring; @@ -84,9 +124,36 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) * This prevents a single queue from consuming all resources and * eliminates the need for remote queue wake-ups when global * limits are met but this queue has no more waiting requests. + * + * Critical bg requests wait on their own list, so they never wait + * behind non-critical bg requests, and are admitted first. They are + * subject to the bg limits, but one critical bg request per queue is + * always allowed. It is not counted in the non-critical limit and can + * use the critical entries, so critical bg requests always progress. + * + * With the writeback cache, non-critical bg requests also share the + * non-critical limit with foreground requests, see + * fuse_uring_max_noncritical(). */ + while ((fc->active_background < fc->max_background || + !queue->active_bg_critical) && + !list_empty(&queue->fuse_req_bg_crit_queue)) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_bg_crit_queue, + struct fuse_req, list); + fc->active_background++; + queue->active_background++; + set_bit(FR_URING_CRITICAL, &req->flags); + queue->active_bg_critical++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } + while ((fc->active_background < fc->max_background || !queue->active_background) && + (!fc->writeback_cache || queue->stopped || + queue->active_noncritical < fuse_uring_max_noncritical(queue)) && (!list_empty(&queue->fuse_req_bg_queue))) { struct fuse_req *req; @@ -94,29 +161,82 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) struct fuse_req, list); fc->active_background++; queue->active_background++; + if (fc->writeback_cache) { + set_bit(FR_URING_NONCRIT, &req->flags); + queue->active_noncritical++; + } + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) +{ + lockdep_assert_held(&queue->lock); + + while (queue->active_noncritical < fuse_uring_max_noncritical(queue) && + !list_empty(&queue->fuse_req_fg_queue)) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_fg_queue, + struct fuse_req, list); + set_bit(FR_URING_NONCRIT, &req->flags); + queue->active_noncritical++; list_move_tail(&req->list, &queue->fuse_req_queue); } } +/* + * Undo the queue counting of an admitted request, and admit waiting requests + * into the freed slots. Requests that were never admitted are not counted. + */ +static void fuse_uring_end_active(struct fuse_ring_queue *queue, + struct fuse_req *req) +{ + struct fuse_conn *fc = queue->ring->fc; + bool noncritical, background; + + lockdep_assert_held(&queue->lock); + + noncritical = test_and_clear_bit(FR_URING_NONCRIT, &req->flags); + if (noncritical) + queue->active_noncritical--; + + background = test_bit(FR_BACKGROUND, &req->flags); + if (background) { + queue->active_background--; + if (test_and_clear_bit(FR_URING_CRITICAL, &req->flags)) + queue->active_bg_critical--; + } + + /* + * The non-critical limit is shared by foreground and bg requests. A + * freed non-critical slot is offered first to the kind of request that + * freed it, so under load neither kind can starve the other. + */ + if (noncritical && !background) + fuse_uring_flush_queue_fg(queue); + if (noncritical || background) { + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); + } + if (noncritical && background) + fuse_uring_flush_queue_fg(queue); +} + static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; lockdep_assert_not_held(&queue->lock); spin_lock(&queue->lock); ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); - if (test_bit(FR_BACKGROUND, &req->flags)) { - queue->active_background--; - spin_lock(&fc->bg_lock); - fuse_uring_flush_queue_bg(queue); - spin_unlock(&fc->bg_lock); - } + fuse_uring_end_active(queue, req); spin_unlock(&queue->lock); if (error) @@ -133,9 +253,15 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) LIST_HEAD(req_list); spin_lock(&queue->lock); - list_for_each_entry(req, &queue->fuse_req_queue, list) + list_for_each_entry(req, &queue->fuse_req_queue, list) { + clear_bit(FR_PENDING, &req->flags); + if (test_and_clear_bit(FR_URING_NONCRIT, &req->flags)) + queue->active_noncritical--; + } + list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); list_splice_init(&queue->fuse_req_queue, &req_list); + list_splice_init(&queue->fuse_req_fg_queue, &req_list); queue->nr_reqs = 0; spin_unlock(&queue->lock); @@ -384,6 +510,8 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_bg_crit_queue); + INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); queue->fpq.processing = pq; @@ -447,7 +575,10 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); + if (test_and_clear_bit(FR_URING_NONCRIT, &req->flags)) + queue->active_noncritical--; } + queue->nr_ents--; /* * The entry must not be freed immediately, due to access of direct @@ -1023,6 +1154,40 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, fuse_uring_req_end(ent, req, err); } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + +/* + * Hand each request on fuse_req_queue to an available entry. Freeing a slot + * or background budget can admit more than one request at once, and every one + * of them needs an entry: a request left on fuse_req_queue next to an idle + * entry waits for the next event on this queue, which never comes when the + * requests in userspace are all blocked on a DLM revoke. + * + * Must be called without the queue lock. The entries are sent in task context, + * as the caller might hold the io_uring submission lock. + */ +static void fuse_uring_dispatch_pending(struct fuse_ring_queue *queue) +{ + struct fuse_ring_ent *ent; + struct fuse_req *req; + + for (;;) { + spin_lock(&queue->lock); + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + req = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (req) + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + + if (!req) + return; + + fuse_uring_dispatch_ent(ent, true); + } +} + /* * Get the next fuse req and send it */ @@ -1032,11 +1197,15 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, { int err; struct fuse_req *req; + bool more; retry: spin_lock(&queue->lock); fuse_uring_ent_avail(ent, queue); req = fuse_uring_ent_assign_req(ent); + /* the request that ended might have admitted more than this one */ + more = !list_empty(&queue->fuse_req_queue) && + !list_empty(&queue->ent_avail_queue); spin_unlock(&queue->lock); if (req) { @@ -1044,6 +1213,9 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, if (err) goto retry; } + + if (more) + fuse_uring_dispatch_pending(queue); } static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) @@ -1090,6 +1262,17 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped)) return err; + /* + * With a single entry no entry is critical, so a non-critical request + * blocked in userspace on a DLM revoke can hold the only entry, see + * fuse_uring_nr_critical(). Checked on commit rather than on register, + * as a queue has a single entry while its entries are being registered, + * and libfuse registers all of them before it commits a reply. + */ + if (fc->writeback_cache && READ_ONCE(queue->nr_ents) < 2) + pr_warn_once("fuse-io-uring: qid=%u has a single entry, page I/O might hang behind a request blocked in userspace; use an io-uring queue depth of 2 or more\n", + qid); + spin_lock(&queue->lock); /* Find a request based on the unique ID of the fuse request * This should get revised, as it needs a hash calculation and list @@ -1112,7 +1295,18 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + queue->nr_reqs--; + fuse_uring_end_active(queue, req); + spin_unlock(&queue->lock); + + /* + * The freed non-critical or background slot might have moved + * queued requests to fuse_req_queue. This entry is not usable, + * so hand them to the available entries. + */ + fuse_uring_dispatch_pending(queue); + req->out.h.error = err; clear_bit(FR_SENT, &req->flags); fuse_request_end(req); @@ -1357,6 +1551,12 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; + queue->nr_ents++; + /* the new entry might raise the non-critical limit */ + fuse_uring_flush_queue_fg(queue); + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1593,6 +1793,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; + struct fuse_req *next; int err; err = -EINVAL; @@ -1607,18 +1808,25 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); queue->nr_reqs++; - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else + if (fuse_uring_req_fg_noncritical(req)) { + list_add_tail(&req->list, &queue->fuse_req_fg_queue); + fuse_uring_flush_queue_fg(queue); + } else { list_add_tail(&req->list, &queue->fuse_req_queue); + } + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + next = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); - if (ent) + if (next) fuse_uring_dispatch_ent(ent, false); return; @@ -1636,7 +1844,6 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) struct fuse_conn *fc = req->fm->fc; struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; - struct fuse_ring_ent *ent = NULL; queue = fuse_uring_select_queue(ring, true); if (!queue) @@ -1650,33 +1857,26 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - list_add_tail(&req->list, &queue->fuse_req_bg_queue); + if (fuse_uring_req_critical(req)) + list_add_tail(&req->list, &queue->fuse_req_bg_crit_queue); + else + list_add_tail(&req->list, &queue->fuse_req_bg_queue); queue->nr_reqs++; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); spin_lock(&fc->bg_lock); fc->num_background++; if (fc->num_background == fc->max_background) fc->blocked = 1; fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); + spin_unlock(&queue->lock); /* * Due to bg_queue flush limits there might be other bg requests - * in the queue that need to be handled first. Or no further req - * might be available. + * in the queue that need to be handled first, or this one might have + * to wait. Budget freed on other queues might also admit several. */ - req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, - list); - if (ent && req) { - fuse_uring_add_req_to_ring_ent(ent, req); - spin_unlock(&queue->lock); - - fuse_uring_dispatch_ent(ent, true); - } else { - spin_unlock(&queue->lock); - } + fuse_uring_dispatch_pending(queue); return true; } @@ -1690,7 +1890,11 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; + fuse_uring_end_active(queue, req); spin_unlock(&queue->lock); + + /* the freed non-critical slot might let queued requests run */ + fuse_uring_dispatch_pending(queue); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 778924ea342c76..da7e8a17301ccc 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -95,9 +95,26 @@ struct fuse_ring_queue { /* fuse requests waiting for an entry slot */ struct list_head fuse_req_queue; - /* background fuse requests */ + /* non-critical background fuse requests */ struct list_head fuse_req_bg_queue; + /* uring_critical background fuse requests */ + struct list_head fuse_req_bg_crit_queue; + + /* non-critical foreground fuse requests waiting for the non-critical limit */ + struct list_head fuse_req_fg_queue; + + /* number of registered ring entries */ + unsigned int nr_ents; + + /* + * non-critical requests, foreground and background, on fuse_req_queue + * or in userspace. With the writeback cache they are limited to the + * entries that are not critical entries, which are used only by + * uring_critical requests, see fuse_uring_max_noncritical(). + */ + unsigned int active_noncritical; + /* number of requests queued or in userspace */ unsigned int nr_reqs; @@ -105,6 +122,12 @@ struct fuse_ring_queue { unsigned int active_background; + /* + * uring_critical background requests on fuse_req_queue or in + * userspace, one is always allowed regardless of the bg limits + */ + unsigned int active_bg_critical; + bool stopped; }; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index a58e586754ef58..6c12580c1a7e49 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2119,6 +2119,8 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) if (fm->fc->setattr_writeback) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); + /* might be called from writeback, like a background request */ + args.uring_critical = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index fffef155ceb586..b10aa4ae3021fa 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1002,6 +1002,7 @@ static int fuse_do_readpage(struct file *file, struct page *page) struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, + .ap.args.uring_critical = true, .ap.num_pages = 1, .ap.pages = &page, .ap.descs = &desc, @@ -1112,6 +1113,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; + ap->args.uring_critical = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1558,6 +1560,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + ap->args.uring_critical = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; @@ -2779,6 +2782,7 @@ __acquires(fi->lock) args->in_args[1].size = inarg->size; args->force = true; args->nocreds = true; + args->uring_critical = true; err = fuse_simple_background(fm, args, GFP_ATOMIC); if (err == -ENOMEM) { @@ -3375,6 +3379,36 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l } return err; } + +/* + * Batch size for fuse_fault_read_private_folios(): one fault-around + * window (64k with 4k pages), centred on the faulted page. A + * spanning access (one instruction touching the tail of one page and + * the head of the next -- any unaligned copy does this constantly) + * needs BOTH pages mapped at once to retire, and every server round + * trip gets this client's grant revoked and its page cache dropped + * (the server does not exempt the requester, and the notification + * runs one op behind). A one-page grant-and-populate therefore + * ping-pongs forever: mapping each page costs a round trip whose + * revoke drops the other. A window covering both pages makes the + * neighbour's fault a pure fast-path hit -- recorded grant, resident + * folio, NO server op and no new revoke -- and the spanning access + * completes. Centred, not aligned: an aligned window has edges + * (every 16th page boundary), and a spanning access sitting on an + * edge gets two one-page-covering windows and ping-pongs just the + * same (generic/263); a centred window always covers both of the + * faulted page's neighbours, so no boundary has the problem. + */ +#define FUSE_FAULT_BATCH_PAGES 16 + +/* First page of the populate/grant window centred on @index. */ +static pgoff_t fuse_fault_batch_base(pgoff_t index) +{ + if (index < FUSE_FAULT_BATCH_PAGES / 2) + return 0; + return index - FUSE_FAULT_BATCH_PAGES / 2; +} + /* * Wait for writeback against this page to complete before allowing it * to be marked dirty again, and hence written back again, possibly @@ -3396,9 +3430,60 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct file *file = vmf->vma->vm_file; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = vmf->pgoff << PAGE_SHIFT; - if (!fm->fc->writeback_cache && fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; + if (fm->fc->writeback_cache && fm->fc->dlm) { + /* + * Writenotify routes every clean-to-dirty transition of a + * shared mapping through here, but the WRITE grant itself + * is taken by fuse_filemap_fault(): shared-writable VMAs + * get no ->map_pages under DLM (see fuse_file_mmap()), so + * every PTE in such a VMA was installed by a fault that + * recorded a WRITE grant for its batch window. Only + * verify here, and only without blocking -- nothing may + * sleep on the range lock or the server while mmap_lock + * (or the per-VMA lock) is held: the buffered write path + * holds the range lock LOCKED while faulting in its + * source buffer (which takes mmap_lock), so with a + * pending mmap_lock writer queued between the two, + * blocking here deadlocks three ways -- the cycle + * 94effa2 ("fuse: do not block the fault path under + * mmap_lock") removed from the fault path -- and + * page_mkwrite has no VM_FAULT_RETRY protocol to drop + * the lock and come back with. + * + * Same fence as fuse_filemap_fault()'s fast path: only + * once LOCKED can the recorded grant be trusted. On a + * miss, NOPAGE. A contended trylock re-faults with + * mmap_lock released in between, so the holder can make + * progress; a verified miss means a revoke is in flight + * (only an invalidate removes a recorded grant), whose + * unmap_mapping_range() clears the PTE, so the re-fault + * goes through fuse_filemap_fault(), which re-takes the + * grant with the proper drop-and-retry dance. + */ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + bool held = false; + + if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, + pos + PAGE_SIZE - 1, + FUSE_RANGE_LOCK_WRITE)) { + bool locked = + fuse_range_lock_try_mark_locked(fi, &rlock); + + held = locked && + fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, + FUSE_PAGE_LOCK_WRITE); + if (locked && !held) + pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", + fi->nodeid, vmf->pgoff); + fuse_range_lock_release(fi, &rlock); + } + + if (!held) + return VM_FAULT_NOPAGE; + } else if (fm->fc->dlm) { size_t length = PAGE_SIZE; int err = fuse_get_page_mkwrite_lock(file, pos, length); if (err < 0) { @@ -3442,35 +3527,6 @@ static bool fuse_fault_needs_read(struct inode *inode, pgoff_t index) return !uptodate; } -/* - * Batch size for fuse_fault_read_private_folios(): one fault-around - * window (64k with 4k pages), centred on the faulted page. A - * spanning access (one instruction touching the tail of one page and - * the head of the next -- any unaligned copy does this constantly) - * needs BOTH pages mapped at once to retire, and every server round - * trip gets this client's grant revoked and its page cache dropped - * (the server does not exempt the requester, and the notification - * runs one op behind). A one-page grant-and-populate therefore - * ping-pongs forever: mapping each page costs a round trip whose - * revoke drops the other. A window covering both pages makes the - * neighbour's fault a pure fast-path hit -- recorded grant, resident - * folio, NO server op and no new revoke -- and the spanning access - * completes. Centred, not aligned: an aligned window has edges - * (every 16th page boundary), and a spanning access sitting on an - * edge gets two one-page-covering windows and ping-pongs just the - * same (generic/263); a centred window always covers both of the - * faulted page's neighbours, so no boundary has the problem. - */ -#define FUSE_FAULT_BATCH_PAGES 16 - -/* First page of the populate/grant window centred on @index. */ -static pgoff_t fuse_fault_batch_base(pgoff_t index) -{ - if (index < FUSE_FAULT_BATCH_PAGES / 2) - return 0; - return index - FUSE_FAULT_BATCH_PAGES / 2; -} - /* * Read a batch of pages centred on @index into freshly * allocated folios that are NOT in the page cache. Unlike @@ -3703,9 +3759,19 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) * miss and come back here; each pass leaves a recorded * grant behind, which only a real invalidation takes away * again. + * + * FAULT_FLAG_TRIED does not demote a pass to the blocking + * path below: retries may repeat (4064b98 "mm: allow + * VM_FAULT_RETRY for multiple times"), and blocking on + * the range lock with mmap_lock held closes a three-party + * cycle whenever the lock holder is a buffered write + * faulting in its source buffer (which waits on + * mmap_lock) behind a queued mmap_lock writer. A miss on + * the retried fast path only means the range is contended + * again; waiting for it must happen with mmap_lock + * dropped, every time. */ - if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && - !(vmf->flags & FAULT_FLAG_TRIED)) { + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) return VM_FAULT_RETRY; @@ -3759,11 +3825,12 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) } /* - * Retries exhausted or not allowed: block in place. The - * waits are killable, and a same-task fault-in never - * conflicts with its own IO's range lock, so this cannot - * deadlock on itself; it can still stack behind a pending - * mmap_lock writer the way any blocking fault can. + * Retry not allowed (a caller without + * FAULT_FLAG_ALLOW_RETRY): block in place. The waits are + * killable, and a same-task fault-in never conflicts with + * its own IO's range lock, so this cannot deadlock on + * itself; it can still stack behind a pending mmap_lock + * writer the way any blocking fault can. * * Nothing held once the range lock is LOCKED may wait for * the server (see fuse_cache_read_iter), and @@ -3902,6 +3969,23 @@ static const struct vm_operations_struct fuse_file_vm_ops = { .page_mkwrite = fuse_page_mkwrite, }; +/* + * Shared-writable VMAs under the writeback cache with DLM: no + * ->map_pages. Fault-around installs PTEs for any resident folio in + * its window without calling ->fault, so a folio read in under a READ + * grant (read(2), readahead) could take its first write in + * ->page_mkwrite with no WRITE grant ever requested -- and + * page_mkwrite cannot take one there (no retry protocol; see + * fuse_page_mkwrite()). Without fault-around, every first touch goes + * through fuse_filemap_fault(), which records a WRITE grant for the + * batch window before any PTE is installed. + */ +static const struct vm_operations_struct fuse_file_wb_dlm_vm_ops = { + .close = fuse_vma_close, + .fault = fuse_filemap_fault, + .page_mkwrite = fuse_page_mkwrite, +}; + static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; @@ -3983,7 +4067,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) } file_accessed(file); - vma->vm_ops = &fuse_file_vm_ops; + if (fc->writeback_cache && fc->dlm && + (vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + vma->vm_ops = &fuse_file_wb_dlm_vm_ops; + else + vma->vm_ops = &fuse_file_vm_ops; return 0; } diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 9284880b2e96c6..e6080f312cc047 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -176,7 +176,10 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * For overlapping ranges, handle lock compatibility: * - READ locks are compatible with existing READ locks * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * - WRITE locks need to upgrade existing READ locks, but only over the + * intersection with [start, end]: the server granted nothing outside + * it, so the parts of a READ range beyond the grant are split off and + * keep their mode * * Return: 0 on success, negative error code on failure */ @@ -210,6 +213,49 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, /* Check lock compatibility */ if (lock_mode == FUSE_PCACHE_LK_WRITE && lock_mode != range->mode) { + /* + * Upgrade only what the server granted: the grant + * covers [start, end], not the whole of an + * overlapping READ range. Split off the pieces + * outside the grant first -- mode-preserving, so + * an allocation failure below still leaves the + * tree describing exactly the grants held -- and + * upgrade just the intersection. + */ + if (range->start < start) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = range->start; + new_range->end = start - 1; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->start = start; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } + if (range->end > end) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = end + 1; + new_range->end = range->end; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->end = end; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } /* we own the lock but have to update it. */ list_add_tail(&range->list, &to_upgrade); } @@ -620,7 +666,8 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); } -/* Context for a fuse_get_dlm_lock() request, embedding struct +/* + * Context for a fuse_get_dlm_lock() request, embedding struct * fuse_args as required by the request API. */ struct fuse_dlm_lock_args { @@ -652,7 +699,7 @@ struct fuse_dlm_lock_args { * there instead if still necessary, right before it touches the page * cache. * - * Does not re-validate the granted range against what was requested -- + * Does not re-validate the granted range against what was requested; * fuse_get_dlm_lock() still does that itself after being woken. */ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, @@ -669,7 +716,7 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, * range is treated as covered) here: either way fuse_get_dlm_lock() * goes on to consider the range usable. Any other error fails the * IO instead, and the caller releases the still-INIT rlock directly - * without ever touching the page cache under it -- leave it alone. + * without ever touching the page cache under it, so leave it alone. */ if (error && error != -ENOSYS) return; @@ -687,7 +734,7 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, * moved to READY as part of processing a reply that leaves the range * covered, before this function's caller is even woken up, or * directly to LOCKED when the range is already covered without a - * round trip -- see fuse_get_dlm_lock_complete() and the declaration + * round trip, see fuse_get_dlm_lock_complete() and the declaration * in fuse_dlm_cache.h. In the later case, the grant is re-validated * one LOCKED (fencing further revokes) before being trusted; if a * concurrent invalidate is found to have revoked it, @rlock is put @@ -720,7 +767,8 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (!length) return 0; - /* note that this can be run from different processes + /* + * note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care * of any races in lock requests. @@ -729,13 +777,13 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, * never disagree about what counts as covered. * * rlock is still INIT here, invisible to invalidation. Move - * it to LOCKED -- which blocks until any invalidate that is + * it to LOCKED, which blocks until any invalidate that is * concurrently draining an overlapping range (one that got * past its own conflict check while we were INIT) has fully - * revoked the grant and released -- then re-check: only once + * revoked the grant and released, then re-check: only once * LOCKED, which fences any further revoke, can is_held() be * trusted. If the grant was revoked, undo back to INIT (never - * blocks) and continue to request a fresh grant below. + * blocks) and fall through to request a fresh grant below. */ if (rlock) { err = fuse_range_lock_mark_locked(fi, rlock); @@ -750,7 +798,8 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned + /* + * note that the offset and length don't have to be page aligned * here but since we only get here on writeback caching we will * send out page aligned requests */ @@ -827,7 +876,7 @@ static inline uint64_t fuse_range_lock_last(struct fuse_range_lock *lock) INTERVAL_TREE_DEFINE(struct fuse_range_lock, rb, uint64_t, __subtree_end, fuse_range_lock_start, fuse_range_lock_last, static, - fuse_range_it); + fuse_range_it); /** * fuse_range_lock_tree_init - Initialize the range lock manager @@ -852,14 +901,14 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) * @lock itself) and either @lock or that range is a WRITE lock (READ * ranges may overlap each other freely). When @locked_only is set, a * range still in INIT state (reserved, not yet touching the page cache) - * is not considered, but a range already in READY or LOCKED state is -- + * is not considered, but a range already in READY or LOCKED state is, * see fuse_range_lock_acquire_locked(). * * A range held by @lock's own task never conflicts: the only way that * happens is the task faulting in its own user buffer mid-IO, where * fuse_filemap_fault() takes a range lock on the faulting page while * the read/write path already holds one covering it. Waiting here - * would then deadlock on ourselves, and skipping is safe -- the outer + * would then deadlock on ourselves, and skipping is safe: the outer * hold already gives invalidation an overlapping range to conflict * with for as long as both are held. * @@ -869,7 +918,7 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) */ static bool fuse_range_conflicts(struct fuse_range_lock_tree *tree, struct fuse_range_lock *lock, - bool locked_only) + bool locked_only) { struct fuse_range_lock *cur; @@ -892,7 +941,7 @@ static bool fuse_range_conflicts(struct fuse_range_lock_tree *tree, * @lock: The range lock to try to acquire * * Conflict tested against every existing range regardless of state, same - * as a plain exclusive acquire -- this is what keeps two local IOs on an + * as a plain exclusive acquire. This is what keeps two local IOs on an * overlapping range serialized against each other even while both are * still reserving (INIT), not yet touching the page cache. * @@ -987,7 +1036,7 @@ static bool fuse_range_try_mark_locked(struct fuse_range_lock_tree *tree, * reboot clears. * * Return: 0 with the range reserved, or -ERESTARTSYS if a fatal signal - * arrived first -- the lock was never inserted then and must not be + * arrived first. The lock was never inserted then and must not be * passed to fuse_range_lock_release(). */ int fuse_range_lock_acquire_init(struct fuse_inode *inode, @@ -1019,7 +1068,7 @@ int fuse_range_lock_acquire_init(struct fuse_inode *inode, * callers that must not block, e.g. the fault path under mmap_lock. * * Return: true with the range reserved, false if a conflicting range is - * currently held -- the lock was not inserted then and must not be + * currently held. The lock was not inserted then and must not be * passed to fuse_range_lock_release(). */ bool fuse_range_lock_try_acquire_init(struct fuse_inode *inode, @@ -1070,7 +1119,7 @@ void fuse_range_lock_mark_ready(struct fuse_inode *inode, * Killable, like fuse_range_lock_acquire_init(). * * Return: 0 once LOCKED, or -ERESTARTSYS if a fatal signal arrived - * first -- @lock then keeps its previous state (INIT or READY) and is + * first. @lock then keeps its previous state (INIT or READY) and is * still held, so the caller must still release it. */ int fuse_range_lock_mark_locked(struct fuse_inode *inode, @@ -1091,7 +1140,7 @@ int fuse_range_lock_mark_locked(struct fuse_inode *inode, * Same semantics as fuse_range_lock_mark_locked() but never waits. * * Return: true if @lock is now LOCKED, false if a conflicting READY or - * LOCKED range is held -- @lock then keeps its previous state and is + * LOCKED range is held. @lock then keeps its previous state and is * still held, so the caller must still release it. */ bool fuse_range_lock_try_mark_locked(struct fuse_inode *inode, @@ -1138,8 +1187,8 @@ void fuse_range_lock_mark_init(struct fuse_inode *inode, */ void fuse_range_lock_acquire_locked(struct fuse_inode *inode, struct fuse_range_lock *lock, - uint64_t start, uint64_t end, - enum fuse_range_lock_mode mode) + uint64_t start, uint64_t end, + enum fuse_range_lock_mode mode) { struct fuse_range_lock_tree *tree = &inode->io_range_lock; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 532ef7aff6e06a..ea98502532ebb0 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -422,6 +422,20 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + /* + * Request is on the path invalidation waits for: the page reads in + * fuse_do_readfolio() and fuse_send_readpages(), the page writes in + * fuse_send_write_pages() and fuse_send_writepage(), and + * fuse_flush_times(), which writeback calls. With the writeback cache, + * it must never be held back behind requests that may be blocked in + * userspace on a DLM revoke: foreground or background, it is exempt + * from the io-uring entry limits and may use the critical entries, see + * fuse_uring_req_critical(), and a background one does not wait for + * the background budget, see fuse_simple_background(). Critical + * requests are not limited, so the daemon must never block them on a + * DLM revoke that waits on page invalidation on this node. + */ + bool uring_critical:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -510,6 +524,10 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring + * FR_URING_NONCRIT: request is counted in the io-uring queue non-critical limit, + * shared by non-critical foreground and background requests + * FR_URING_CRITICAL: critical background request counted in the io-uring + * queue active critical background requests */ enum fuse_req_flag { FR_ISREPLY, @@ -526,6 +544,8 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_URING_NONCRIT, + FR_URING_CRITICAL, }; /**