diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 71249d1119533c..8ab950418259db 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -100,6 +100,67 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } +/* + * With the writeback cache, foreground requests may hold at most half of the + * entries. They might wait in userspace on DLM revokes, which wait on page + * invalidation, which waits on writeback - if they held all entries, + * background writeback could not be sent and none of them would complete. + */ +#define FUSE_URING_MIN_FOREGROUND 8U +#define FUSE_URING_MIN_BACKGROUND 2U + +static unsigned int fuse_uring_max_background(struct fuse_ring_queue *queue) +{ + return max(queue->nr_ents / 4, FUSE_URING_MIN_BACKGROUND); +} + +static unsigned int fuse_uring_max_foreground(struct fuse_ring_queue *queue) +{ + unsigned int bg = fuse_uring_max_background(queue); + + if (queue->nr_ents <= bg) { + pr_warn_once("fuse-io-uring configuration value is too small, could cause hung issue\n"); + return queue->nr_ents; + } + + return queue->nr_ents - bg; +} + +static bool fuse_uring_req_fg_limited(struct fuse_req *req) +{ + return req->fm->fc->writeback_cache && + !test_bit(FR_BACKGROUND, &req->flags) && + !(req->args && req->args->no_fg_limit); +} + +static void fuse_uring_flush_queue_fg(struct fuse_ring_queue *queue) +{ + lockdep_assert_held(&queue->lock); + + while (queue->active_foreground < fuse_uring_max_foreground(queue) && + !list_empty(&queue->fuse_req_fg_queue)) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_fg_queue, + struct fuse_req, list); + set_bit(FR_URING_FG, &req->flags); + queue->active_foreground++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_end_foreground(struct fuse_ring_queue *queue, + struct fuse_req *req) +{ + lockdep_assert_held(&queue->lock); + + if (test_and_clear_bit(FR_URING_FG, &req->flags)) { + queue->active_foreground--; + fuse_uring_flush_queue_fg(queue); + } +} + static void __fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, int error) { @@ -112,6 +173,7 @@ static void __fuse_uring_req_end(struct fuse_ring_ent *ent, ent->fuse_req = NULL; queue->nr_reqs--; list_del_init(&req->list); + fuse_uring_end_foreground(queue, req); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); @@ -171,9 +233,15 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) LIST_HEAD(req_list); spin_lock(&queue->lock); - list_for_each_entry(req, &queue->fuse_req_queue, list) + list_for_each_entry(req, &queue->fuse_req_queue, list) { + clear_bit(FR_PENDING, &req->flags); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; + } + list_for_each_entry(req, &queue->fuse_req_fg_queue, list) clear_bit(FR_PENDING, &req->flags); list_splice_init(&queue->fuse_req_queue, &req_list); + list_splice_init(&queue->fuse_req_fg_queue, &req_list); queue->nr_reqs = 0; spin_unlock(&queue->lock); @@ -237,6 +305,7 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) spin_lock(&queue->lock); if (fuse_request_expired(fc, &queue->fuse_req_queue) || fuse_request_expired(fc, &queue->fuse_req_bg_queue) || + fuse_request_expired(fc, &queue->fuse_req_fg_queue) || ent_list_request_expired(fc, &queue->ent_w_req_queue) || ent_list_request_expired(fc, &queue->ent_in_userspace)) { spin_unlock(&queue->lock); @@ -461,6 +530,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, INIT_LIST_HEAD(&queue->ent_in_userspace); INIT_LIST_HEAD(&queue->fuse_req_queue); INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->fuse_req_fg_queue); INIT_LIST_HEAD(&queue->ent_released); queue->fpq.processing = pq; @@ -524,7 +594,10 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags if (req) { /* remove entry from queue->fpq->processing */ list_del_init(&req->list); + if (test_and_clear_bit(FR_URING_FG, &req->flags)) + queue->active_foreground--; } + queue->nr_ents--; /* * The entry must not be freed immediately, due to access of direct @@ -1138,6 +1211,8 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg); + /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -1151,6 +1226,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; + struct fuse_ring_ent *avail; + struct fuse_req *next = NULL; err = -ENOTCONN; if (!ring) @@ -1189,7 +1266,32 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (err != 0) { pr_info_ratelimited("qid=%d commit_id %llu state %d", queue->qid, commit_id, ent->state); + queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + if (test_bit(FR_BACKGROUND, &req->flags)) { + queue->active_background--; + spin_lock(&fc->bg_lock); + fuse_uring_flush_queue_bg(queue); + spin_unlock(&fc->bg_lock); + } + + /* + * The freed foreground or background slot might have moved a + * queued request to fuse_req_queue. This entry is not usable, + * so hand the request to another available entry. + */ + avail = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (avail) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(avail, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(avail, true); + req->out.h.error = err; clear_bit(FR_SENT, &req->flags); fuse_request_end(req); @@ -1434,6 +1536,8 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, spin_lock(&queue->lock); ent->cmd = cmd; + queue->nr_ents++; + fuse_uring_flush_queue_fg(queue); spin_unlock(&queue->lock); /* Marks the ring entry as ready */ @@ -1670,6 +1774,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) struct fuse_ring *ring = fc->ring; struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; + struct fuse_req *next; int err; err = -EINVAL; @@ -1684,18 +1789,25 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; - ent = list_first_entry_or_null(&queue->ent_avail_queue, - struct fuse_ring_ent, list); queue->nr_reqs++; - if (ent) - fuse_uring_add_req_to_ring_ent(ent, req); - else + if (fuse_uring_req_fg_limited(req)) { + list_add_tail(&req->list, &queue->fuse_req_fg_queue); + fuse_uring_flush_queue_fg(queue); + } else { list_add_tail(&req->list, &queue->fuse_req_queue); + } + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + next = ent ? list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list) : NULL; + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); - if (ent) + if (next) fuse_uring_dispatch_ent(ent, false); return; @@ -1762,12 +1874,27 @@ bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; bool removed = fuse_remove_pending_req(req, &queue->lock); + struct fuse_ring_ent *ent; + struct fuse_req *next = NULL; if (removed) { /* Update counters after successful removal */ spin_lock(&queue->lock); queue->nr_reqs--; + fuse_uring_end_foreground(queue, req); + + /* the freed foreground slot might let a queued request run */ + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (ent) + next = list_first_entry_or_null(&queue->fuse_req_queue, + struct fuse_req, list); + if (next) + fuse_uring_add_req_to_ring_ent(ent, next); spin_unlock(&queue->lock); + + if (next) + fuse_uring_dispatch_ent(ent, true); } return removed; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 667f668ef3277f..4f86f64e801777 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -98,6 +98,19 @@ struct fuse_ring_queue { /* background fuse requests */ struct list_head fuse_req_bg_queue; + /* foreground fuse requests waiting for the foreground limit */ + struct list_head fuse_req_fg_queue; + + /* number of registered ring entries */ + unsigned int nr_ents; + + /* + * foreground requests on fuse_req_queue or in userspace, limited to + * the entries not reserved for background requests with writeback cache; + * see fuse_uring_max_foreground() + */ + unsigned int active_foreground; + /* number of requests queued or in userspace */ unsigned int nr_reqs; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 4553980c1461a8..fee1864d73d47b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2100,6 +2100,8 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) if (fm->fc->setattr_writeback) inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); + /* might be called from writeback, like a background request */ + args.no_fg_limit = true; return fuse_simple_request(fm, &args); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 748260f92e7cc7..0cc69cace86c71 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1012,6 +1012,7 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, struct fuse_io_args ia = { .ap.args.page_zeroing = true, .ap.args.out_pages = true, + .ap.args.no_fg_limit = true, .ap.num_folios = 1, .ap.folios = &folio, .ap.descs = &desc, @@ -1164,6 +1165,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, ap->args.out_pages = true; ap->args.page_zeroing = true; ap->args.page_replace = true; + ap->args.no_fg_limit = true; /* Don't overflow end offset */ if (pos + (count - 1) == LLONG_MAX) { @@ -1593,6 +1595,7 @@ static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + ap->args.no_fg_limit = true; err = fuse_simple_request(fm, &ap->args); if (!err && ia->write.out.size > count) err = -EIO; @@ -3378,6 +3381,36 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l } return err; } + +/* + * Batch size for fuse_fault_read_private_folios(): one fault-around + * window (64k with 4k pages), centred on the faulted page. A + * spanning access (one instruction touching the tail of one page and + * the head of the next -- any unaligned copy does this constantly) + * needs BOTH pages mapped at once to retire, and every server round + * trip gets this client's grant revoked and its page cache dropped + * (the server does not exempt the requester, and the notification + * runs one op behind). A one-page grant-and-populate therefore + * ping-pongs forever: mapping each page costs a round trip whose + * revoke drops the other. A window covering both pages makes the + * neighbour's fault a pure fast-path hit -- recorded grant, resident + * folio, NO server op and no new revoke -- and the spanning access + * completes. Centred, not aligned: an aligned window has edges + * (every 16th page boundary), and a spanning access sitting on an + * edge gets two one-page-covering windows and ping-pongs just the + * same (generic/263); a centred window always covers both of the + * faulted page's neighbours, so no boundary has the problem. + */ +#define FUSE_FAULT_BATCH_PAGES 16 + +/* First page of the populate/grant window centred on @index. */ +static pgoff_t fuse_fault_batch_base(pgoff_t index) +{ + if (index < FUSE_FAULT_BATCH_PAGES / 2) + return 0; + return index - FUSE_FAULT_BATCH_PAGES / 2; +} + /* * Wait for writeback against this page to complete before allowing it * to be marked dirty again, and hence written back again, possibly @@ -3399,9 +3432,60 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct file *file = vmf->vma->vm_file; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = vmf->pgoff << PAGE_SHIFT; - if (!fm->fc->writeback_cache && fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; + if (fm->fc->writeback_cache && fm->fc->dlm) { + /* + * Writenotify routes every clean-to-dirty transition of a + * shared mapping through here, but the WRITE grant itself + * is taken by fuse_filemap_fault(): shared-writable VMAs + * get no ->map_pages under DLM (see fuse_file_mmap()), so + * every PTE in such a VMA was installed by a fault that + * recorded a WRITE grant for its batch window. Only + * verify here, and only without blocking -- nothing may + * sleep on the range lock or the server while mmap_lock + * (or the per-VMA lock) is held: the buffered write path + * holds the range lock LOCKED while faulting in its + * source buffer (which takes mmap_lock), so with a + * pending mmap_lock writer queued between the two, + * blocking here deadlocks three ways -- the cycle + * 94effa2 ("fuse: do not block the fault path under + * mmap_lock") removed from the fault path -- and + * page_mkwrite has no VM_FAULT_RETRY protocol to drop + * the lock and come back with. + * + * Same fence as fuse_filemap_fault()'s fast path: only + * once LOCKED can the recorded grant be trusted. On a + * miss, NOPAGE. A contended trylock re-faults with + * mmap_lock released in between, so the holder can make + * progress; a verified miss means a revoke is in flight + * (only an invalidate removes a recorded grant), whose + * unmap_mapping_range() clears the PTE, so the re-fault + * goes through fuse_filemap_fault(), which re-takes the + * grant with the proper drop-and-retry dance. + */ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_range_lock rlock; + bool held = false; + + if (fuse_range_lock_try_acquire_init(fi, &rlock, pos, + pos + PAGE_SIZE - 1, + FUSE_RANGE_LOCK_WRITE)) { + bool locked = + fuse_range_lock_try_mark_locked(fi, &rlock); + + held = locked && + fuse_dlm_lock_is_held(fi, pos, PAGE_SIZE, + FUSE_PAGE_LOCK_WRITE); + if (locked && !held) + pr_warn_ratelimited("fuse: mkwrite with no WRITE grant, nodeid %llu idx %lu\n", + fi->nodeid, vmf->pgoff); + fuse_range_lock_release(fi, &rlock); + } + + if (!held) + return VM_FAULT_NOPAGE; + } else if (fm->fc->dlm) { size_t length = PAGE_SIZE; int err = fuse_get_page_mkwrite_lock(file, pos, length); if (err < 0) { @@ -3445,35 +3529,6 @@ static bool fuse_fault_needs_read(struct inode *inode, pgoff_t index) return !uptodate; } -/* - * Batch size for fuse_fault_read_private_folios(): one fault-around - * window (64k with 4k pages), centred on the faulted page. A - * spanning access (one instruction touching the tail of one page and - * the head of the next -- any unaligned copy does this constantly) - * needs BOTH pages mapped at once to retire, and every server round - * trip gets this client's grant revoked and its page cache dropped - * (the server does not exempt the requester, and the notification - * runs one op behind). A one-page grant-and-populate therefore - * ping-pongs forever: mapping each page costs a round trip whose - * revoke drops the other. A window covering both pages makes the - * neighbour's fault a pure fast-path hit -- recorded grant, resident - * folio, NO server op and no new revoke -- and the spanning access - * completes. Centred, not aligned: an aligned window has edges - * (every 16th page boundary), and a spanning access sitting on an - * edge gets two one-page-covering windows and ping-pongs just the - * same (generic/263); a centred window always covers both of the - * faulted page's neighbours, so no boundary has the problem. - */ -#define FUSE_FAULT_BATCH_PAGES 16 - -/* First page of the populate/grant window centred on @index. */ -static pgoff_t fuse_fault_batch_base(pgoff_t index) -{ - if (index < FUSE_FAULT_BATCH_PAGES / 2) - return 0; - return index - FUSE_FAULT_BATCH_PAGES / 2; -} - /* * Read a batch of pages centred on @index into freshly * allocated folios that are NOT in the page cache. Unlike @@ -3704,9 +3759,19 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) * miss and come back here; each pass leaves a recorded * grant behind, which only a real invalidation takes away * again. + * + * FAULT_FLAG_TRIED does not demote a pass to the blocking + * path below: retries may repeat (4064b98 "mm: allow + * VM_FAULT_RETRY for multiple times"), and blocking on + * the range lock with mmap_lock held closes a three-party + * cycle whenever the lock holder is a buffered write + * faulting in its source buffer (which waits on + * mmap_lock) behind a queued mmap_lock writer. A miss on + * the retried fast path only means the range is contended + * again; waiting for it must happen with mmap_lock + * dropped, every time. */ - if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && - !(vmf->flags & FAULT_FLAG_TRIED)) { + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) return VM_FAULT_RETRY; @@ -3760,11 +3825,12 @@ static vm_fault_t fuse_filemap_fault(struct vm_fault *vmf) } /* - * Retries exhausted or not allowed: block in place. The - * waits are killable, and a same-task fault-in never - * conflicts with its own IO's range lock, so this cannot - * deadlock on itself; it can still stack behind a pending - * mmap_lock writer the way any blocking fault can. + * Retry not allowed (a caller without + * FAULT_FLAG_ALLOW_RETRY): block in place. The waits are + * killable, and a same-task fault-in never conflicts with + * its own IO's range lock, so this cannot deadlock on + * itself; it can still stack behind a pending mmap_lock + * writer the way any blocking fault can. * * Nothing held once the range lock is LOCKED may wait for * the server (see fuse_cache_read_iter), and @@ -3903,6 +3969,23 @@ static const struct vm_operations_struct fuse_file_vm_ops = { .page_mkwrite = fuse_page_mkwrite, }; +/* + * Shared-writable VMAs under the writeback cache with DLM: no + * ->map_pages. Fault-around installs PTEs for any resident folio in + * its window without calling ->fault, so a folio read in under a READ + * grant (read(2), readahead) could take its first write in + * ->page_mkwrite with no WRITE grant ever requested -- and + * page_mkwrite cannot take one there (no retry protocol; see + * fuse_page_mkwrite()). Without fault-around, every first touch goes + * through fuse_filemap_fault(), which records a WRITE grant for the + * batch window before any PTE is installed. + */ +static const struct vm_operations_struct fuse_file_wb_dlm_vm_ops = { + .close = fuse_vma_close, + .fault = fuse_filemap_fault, + .page_mkwrite = fuse_page_mkwrite, +}; + static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) { struct fuse_file *ff = file->private_data; @@ -3982,7 +4065,11 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) fuse_link_write_file(file); file_accessed(file); - vma->vm_ops = &fuse_file_vm_ops; + if (fc->writeback_cache && fc->dlm && + (vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + vma->vm_ops = &fuse_file_wb_dlm_vm_ops; + else + vma->vm_ops = &fuse_file_vm_ops; return 0; } diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 573d198b69dccf..c6d51a089736a8 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -177,7 +177,10 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * For overlapping ranges, handle lock compatibility: * - READ locks are compatible with existing READ locks * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * - WRITE locks need to upgrade existing READ locks, but only over the + * intersection with [start, end]: the server granted nothing outside + * it, so the parts of a READ range beyond the grant are split off and + * keep their mode * * Return: 0 on success, negative error code on failure */ @@ -211,6 +214,49 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, /* Check lock compatibility */ if (lock_mode == FUSE_PCACHE_LK_WRITE && lock_mode != range->mode) { + /* + * Upgrade only what the server granted: the grant + * covers [start, end], not the whole of an + * overlapping READ range. Split off the pieces + * outside the grant first -- mode-preserving, so + * an allocation failure below still leaves the + * tree describing exactly the grants held -- and + * upgrade just the intersection. + */ + if (range->start < start) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = range->start; + new_range->end = start - 1; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->start = start; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } + if (range->end > end) { + new_range = kmalloc(sizeof(*new_range), + GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + new_range->start = end + 1; + new_range->end = range->end; + new_range->mode = range->mode; + INIT_LIST_HEAD(&new_range->list); + fuse_page_it_remove(range, &cache->ranges); + range->end = end; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, + &cache->ranges); + } /* we own the lock but have to update it. */ list_add_tail(&range->list, &to_upgrade); } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 5c403f0b71d21f..e0fd2418567e5c 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -432,6 +432,7 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + bool no_fg_limit:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -520,6 +521,7 @@ struct fuse_io_priv { * FR_PRIVATE: request is on private list * FR_ASYNC: request is asynchronous * FR_URING: request is handled through fuse-io-uring + * FR_URING_FG: request is counted in the io-uring queue foreground limit */ enum fuse_req_flag { FR_ISREPLY, @@ -536,6 +538,7 @@ enum fuse_req_flag { FR_PRIVATE, FR_ASYNC, FR_URING, + FR_URING_FG, }; /**