diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index fee1864d73d47b..fecab44c0b340b 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -138,6 +138,26 @@ static void fuse_dir_changed(struct inode *dir) inode_maybe_inc_iversion(dir, false); } +/* + * Note in the parent directory that one of its entries was invalidated. + * + * This bumps the same change counter as fuse_dir_changed(), which doubles as a + * generation for the parent's entry cache: fuse_dentry_revalidate() samples it + * before sending its LOOKUP and refuses to re-arm the entry timeout if it moved + * meanwhile. + * + * Only for invalidations coming from the server - a plain + * fuse_invalidate_entry_cache() (a negative lookup, say) must not bump the + * counter, or it would needlessly drop the readdir cache as well. + */ +void fuse_entry_invalidated(struct dentry *entry) +{ + /* d_lock keeps d_parent stable; a live child pins its parent inode */ + spin_lock(&entry->d_lock); + inode_maybe_inc_iversion(d_inode(entry->d_parent), false); + spin_unlock(&entry->d_lock); +} + /* * Mark the attributes as stale due to an atime change. Avoid the invalidate if * atime is not used. @@ -286,7 +306,7 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, struct fuse_lookupx_out ext_out; struct fuse_statx sx; struct fuse_forget_link *forget; - u64 attr_version; + u64 attr_version, dir_version; uint32_t lookupx_flags = FUSE_LOOKUPX_FOR_REVALIDATE; /* For negative dentries, always do a fresh lookup */ @@ -308,6 +328,19 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, if (S_ISDIR(inode->i_mode)) lookupx_flags |= FUSE_LOOKUPX_TARGET_WAS_DIR; + /* + * Unlike ->lookup() and the create paths, revalidate runs + * without the parent's i_rwsem, so it does not exclude + * fuse_reverse_inval_entry(). An invalidation for this name + * can therefore land while the LOOKUP is in flight, and + * re-arming the entry timeout from the reply below would + * silently undo it - permanently so for FUSE_EXPIRE_ONLY and + * for fuse_prune_aliases(), which leave the dentry hashed. + * + * Sample the parent's entry-cache generation here and compare + * it once the reply is in. + */ + dir_version = inode_query_iversion(dir); ret = fuse_do_lookupx(fm, get_node_id(dir), name, &ext_out, lookupx_flags); @@ -338,7 +371,9 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, fuse_change_attributes(inode, &ext_out.entry.attr, &sx, ATTR_TIMEOUT(&ext_out.entry), attr_version); - fuse_change_entry_timeout(entry, &ext_out.entry); + + if (inode_peek_iversion(dir) == dir_version) + fuse_change_entry_timeout(entry, &ext_out.entry); } else if (inode) { fi = get_fuse_inode(inode); if (flags & LOOKUP_RCU) { @@ -703,6 +738,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, struct fuse_inode *fi; struct fuse_file *ff; int epoch, err; + u64 evict_ctr; bool trunc = flags & O_TRUNC; /* Userspace expects S_IFREG in create mode */ @@ -753,6 +789,8 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, if (err) goto out_free_ff; + evict_ctr = fuse_get_evict_ctr(fm->fc); + err = fuse_simple_idmap_request(idmap, fm, &args); free_ext_value(&args); if (err) @@ -767,7 +805,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, ff->nodeid = outentry.nodeid; ff->open_flags = outopenp->open_flags; inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation, - &outentry.attr, ATTR_TIMEOUT(&outentry), 0, 0); + &outentry.attr, ATTR_TIMEOUT(&outentry), 0, evict_ctr); if (!inode) { flags &= ~(O_CREAT | O_EXCL | O_TRUNC); fuse_sync_release(NULL, ff, flags); @@ -866,6 +904,7 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun struct dentry *d; struct fuse_forget_link *forget; int epoch, err; + u64 evict_ctr; if (fuse_is_bad(dir)) return ERR_PTR(-EIO); @@ -888,6 +927,8 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun goto out_put_forget_req; } + evict_ctr = fuse_get_evict_ctr(fm->fc); + err = fuse_simple_idmap_request(idmap, fm, args); free_ext_value(args); if (err) @@ -901,7 +942,7 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun goto out_put_forget_req; inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, - &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0); + &outarg.attr, ATTR_TIMEOUT(&outarg), 0, evict_ctr); if (!inode) { fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1); return ERR_PTR(-ENOMEM); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 573d198b69dccf..95a2f7a9002fb6 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -10,7 +10,6 @@ #include #include - /* A range of pages with a lock */ struct fuse_dlm_range { /* Interval tree node */ @@ -43,8 +42,8 @@ static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) } INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); + fuse_dlm_range_start, fuse_dlm_range_last, static, + fuse_page_it); /** * fuse_page_cache_init - Initialize a page cache lock manager @@ -379,7 +378,7 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, * Return: 0 on success, negative error code on failure */ int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) + uint64_t start, uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *next; @@ -621,8 +620,10 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); } -/* Context for a fuse_get_dlm_lock() request, embedding struct - * fuse_args as required by the request API. */ +/* + * Context for a fuse_get_dlm_lock() request, embedding struct + * fuse_args as required by the request API. + */ struct fuse_dlm_lock_args { struct fuse_args args; struct fuse_inode *fi; @@ -652,7 +653,7 @@ struct fuse_dlm_lock_args { * there instead if still necessary, right before it touches the page * cache. * - * Does not re-validate the granted range against what was requested -- + * Does not re-validate the granted range against what was requested; * fuse_get_dlm_lock() still does that itself after being woken. */ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, @@ -669,7 +670,7 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, * range is treated as covered) here: either way fuse_get_dlm_lock() * goes on to consider the range usable. Any other error fails the * IO instead, and the caller releases the still-INIT rlock directly - * without ever touching the page cache under it -- leave it alone. + * without ever touching the page cache under it, so leave it alone. */ if (error && error != -ENOSYS) return; @@ -687,7 +688,7 @@ static void fuse_get_dlm_lock_complete(struct fuse_mount *fm, * moved to READY as part of processing a reply that leaves the range * covered, before this function's caller is even woken up, or * directly to LOCKED when the range is already covered without a - * round trip -- see fuse_get_dlm_lock_complete() and the declaration + * round trip, see fuse_get_dlm_lock_complete() and the declaration * in fuse_dlm_cache.h. In the later case, the grant is re-validated * one LOCKED (fencing further revokes) before being trusted; if a * concurrent invalidate is found to have revoked it, @rlock is put @@ -720,38 +721,42 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (!length) return 0; - /* note that this can be run from different processes + /* + * note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care * of any races in lock requests. * The early exit uses the same helper the callers re-validate * with, so this check and a later fuse_dlm_lock_is_held() can * never disagree about what counts as covered. - * - * rlock is still INIT here, invisible to invalidation. Move - * it to LOCKED -- which blocks until any invalidate that is - * concurrently draining an overlapping range (one that got - * past its own conflict check while we were INIT) has fully - * revoked the grant and released -- then re-check: only once - * LOCKED, which fences any further revoke, can is_held() be - * trusted. If the grant was revoked, undo back to INIT (never - * blocks) and fall through to request a fresh grant below. */ - if (rlock) { - err = fuse_range_lock_mark_locked(fi, rlock); - if (err) - return err; - } + * + * rlock is still INIT here, invisible to invalidation. Move + * it to LOCKED, which blocks until any invalidate that is + * concurrently draining an overlapping range (one that got + * past its own conflict check while we were INIT) has fully + * revoked the grant and released, then re-check: only once + * LOCKED, which fences any further revoke, can is_held() be + * trusted. If the grant was revoked, undo back to INIT (never + * blocks) and fall through to request a fresh grant below. + */ + if (rlock) { + err = fuse_range_lock_mark_locked(fi, rlock); + if (err) + return err; + } if (fuse_dlm_lock_is_held(fi, offset, length, mode)) return 0; /* we already have this area locked */ - if (rlock) - fuse_range_lock_mark_init(fi, rlock); + if (rlock) + fuse_range_lock_mark_init(fi, rlock); memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned + /* + * note that the offset and length don't have to be page aligned * here but since we only get here on writeback caching we will - * send out page aligned requests */ + * send out page aligned requests + */ inarg.start = offset & PAGE_MASK; inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? @@ -824,7 +829,7 @@ static inline uint64_t fuse_range_lock_last(struct fuse_range_lock *lock) } INTERVAL_TREE_DEFINE(struct fuse_range_lock, rb, uint64_t, __subtree_end, - fuse_range_lock_start, fuse_range_lock_last, static, + fuse_range_lock_start, fuse_range_lock_last, static, fuse_range_it); /** @@ -850,14 +855,14 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) * @lock itself) and either @lock or that range is a WRITE lock (READ * ranges may overlap each other freely). When @locked_only is set, a * range still in INIT state (reserved, not yet touching the page cache) - * is not considered, but a range already in READY or LOCKED state is -- + * is not considered, but a range already in READY or LOCKED state is, * see fuse_range_lock_acquire_locked(). * * A range held by @lock's own task never conflicts: the only way that * happens is the task faulting in its own user buffer mid-IO, where * fuse_filemap_fault() takes a range lock on the faulting page while * the read/write path already holds one covering it. Waiting here - * would then deadlock on ourselves, and skipping is safe -- the outer + * would then deadlock on ourselves, and skipping is safe: the outer * hold already gives invalidation an overlapping range to conflict * with for as long as both are held. * @@ -866,7 +871,7 @@ void fuse_range_lock_tree_init(struct fuse_inode *inode) * Return: true if @lock's range conflicts with an existing held range. */ static bool fuse_range_conflicts(struct fuse_range_lock_tree *tree, - struct fuse_range_lock *lock, + struct fuse_range_lock *lock, bool locked_only) { struct fuse_range_lock *cur; @@ -890,14 +895,14 @@ static bool fuse_range_conflicts(struct fuse_range_lock_tree *tree, * @lock: The range lock to try to acquire * * Conflict tested against every existing range regardless of state, same - * as a plain exclusive acquire -- this is what keeps two local IOs on an + * as a plain exclusive acquire. This is what keeps two local IOs on an * overlapping range serialized against each other even while both are * still reserving (INIT), not yet touching the page cache. * * Return: true if @lock was inserted, false if the caller must wait. */ static bool fuse_range_try_lock_init(struct fuse_range_lock_tree *tree, - struct fuse_range_lock *lock) + struct fuse_range_lock *lock) { bool conflict; @@ -927,7 +932,7 @@ static bool fuse_range_try_lock_init(struct fuse_range_lock_tree *tree, * Return: true if @lock was inserted, false if the caller must wait. */ static bool fuse_range_try_lock_locked(struct fuse_range_lock_tree *tree, - struct fuse_range_lock *lock) + struct fuse_range_lock *lock) { bool conflict; @@ -953,7 +958,7 @@ static bool fuse_range_try_lock_locked(struct fuse_range_lock_tree *tree, * a conflicting READY or LOCKED range to be released. */ static bool fuse_range_try_mark_locked(struct fuse_range_lock_tree *tree, - struct fuse_range_lock *lock) + struct fuse_range_lock *lock) { bool conflict; @@ -985,7 +990,7 @@ static bool fuse_range_try_mark_locked(struct fuse_range_lock_tree *tree, * reboot clears. * * Return: 0 with the range reserved, or -ERESTARTSYS if a fatal signal - * arrived first -- the lock was never inserted then and must not be + * arrived first. The lock was never inserted then and must not be * passed to fuse_range_lock_release(). */ int fuse_range_lock_acquire_init(struct fuse_inode *inode, @@ -1017,7 +1022,7 @@ int fuse_range_lock_acquire_init(struct fuse_inode *inode, * callers that must not block, e.g. the fault path under mmap_lock. * * Return: true with the range reserved, false if a conflicting range is - * currently held -- the lock was not inserted then and must not be + * currently held. The lock was not inserted then and must not be * passed to fuse_range_lock_release(). */ bool fuse_range_lock_try_acquire_init(struct fuse_inode *inode, @@ -1046,7 +1051,7 @@ bool fuse_range_lock_try_acquire_init(struct fuse_inode *inode, * nothing for this call to wait for and no waiter to wake. */ void fuse_range_lock_mark_ready(struct fuse_inode *inode, - struct fuse_range_lock *lock) + struct fuse_range_lock *lock) { struct fuse_range_lock_tree *tree = &inode->io_range_lock; @@ -1068,7 +1073,7 @@ void fuse_range_lock_mark_ready(struct fuse_inode *inode, * Killable, like fuse_range_lock_acquire_init(). * * Return: 0 once LOCKED, or -ERESTARTSYS if a fatal signal arrived - * first -- @lock then keeps its previous state (INIT or READY) and is + * first. @lock then keeps its previous state (INIT or READY) and is * still held, so the caller must still release it. */ int fuse_range_lock_mark_locked(struct fuse_inode *inode, @@ -1089,7 +1094,7 @@ int fuse_range_lock_mark_locked(struct fuse_inode *inode, * Same semantics as fuse_range_lock_mark_locked() but never waits. * * Return: true if @lock is now LOCKED, false if a conflicting READY or - * LOCKED range is held -- @lock then keeps its previous state and is + * LOCKED range is held. @lock then keeps its previous state and is * still held, so the caller must still release it. */ bool fuse_range_lock_try_mark_locked(struct fuse_inode *inode, @@ -1110,7 +1115,7 @@ bool fuse_range_lock_try_mark_locked(struct fuse_inode *inode, * @lock's (until now READY or LOCKED) range and can now proceed around it. */ void fuse_range_lock_mark_init(struct fuse_inode *inode, - struct fuse_range_lock *lock) + struct fuse_range_lock *lock) { struct fuse_range_lock_tree *tree = &inode->io_range_lock; @@ -1135,7 +1140,7 @@ void fuse_range_lock_mark_init(struct fuse_inode *inode, * The range is rounded out to whole pages; see fuse_dlm_cache.h. */ void fuse_range_lock_acquire_locked(struct fuse_inode *inode, - struct fuse_range_lock *lock, + struct fuse_range_lock *lock, uint64_t start, uint64_t end, enum fuse_range_lock_mode mode) { @@ -1156,7 +1161,7 @@ void fuse_range_lock_acquire_locked(struct fuse_inode *inode, * fuse_range_lock_acquire_init() or fuse_range_lock_acquire_locked() */ void fuse_range_lock_release(struct fuse_inode *inode, - struct fuse_range_lock *lock) + struct fuse_range_lock *lock) { struct fuse_range_lock_tree *tree = &inode->io_range_lock; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index e0fd2418567e5c..e776b3d82afac0 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1081,9 +1081,6 @@ struct fuse_conn { /** Version counter for attribute changes */ atomic64_t attr_version; - /** Waitqueue for attr_version initialization */ - wait_queue_head_t attr_version_waitq; - /** Version counter for evict inode */ atomic64_t evict_ctr; @@ -1476,6 +1473,8 @@ void fuse_invalidate_attr_mask(struct inode *inode, u32 mask); void fuse_invalidate_entry_cache(struct dentry *entry); +void fuse_entry_invalidated(struct dentry *entry); + void fuse_invalidate_atime(struct inode *inode); u64 fuse_time_to_jiffies(u64 sec, u32 nsec); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 35e9a16db24d7f..d453f30abbaa6f 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -245,6 +245,26 @@ static ino_t fuse_squash_ino(u64 ino64) return ino; } +/* + * May attributes returned by a request be marked valid, i.e. cleared + * from inval_mask? + * + * Not if they come from a fuse_iget() call whose request raced with an + * evict or a reverse invalidation - either would have invalidated the + * result if the inode's attr_version would've been preserved. + * + * !evict_ctr -> this is not fuse_iget() + * fi->attr_version != 0 -> this is not a new inode + * evict_ctr == fuse_get_evict_ctr() -> no evicts or missed + * invalidations during the request + */ +static bool fuse_may_validate_attrs(struct fuse_conn *fc, + struct fuse_inode *fi, u64 evict_ctr) +{ + return !evict_ctr || fi->attr_version || + evict_ctr == fuse_get_evict_ctr(fc); +} + /* * Handle statx-specific attribute updates with partial attribute support. */ @@ -260,18 +280,8 @@ static void fuse_change_attributes_common_sx(struct inode *inode, lockdep_assert_held(&fi->lock); - /* - * Clear returned basic stats from invalid mask. - * - * Don't do this if this is coming from a fuse_iget() call and there - * might have been a racing evict which would've invalidated the result - * if the attr_version would've been preserved. - * - * !evict_ctr -> this is create - * fi->attr_version != 0 -> this is not a new inode - * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request - */ - if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) + /* Clear returned basic stats from invalid mask */ + if (fuse_may_validate_attrs(fc, fi, evict_ctr)) set_mask_bits(&fi->inval_mask, returned_attrs, 0); fi->attr_version = atomic64_inc_return(&fc->attr_version); @@ -354,11 +364,10 @@ static void fuse_change_attributes_common_sx(struct inode *inode, } } - /* Common fields for both statx and getattr */ - if (attr->blksize != 0) - inode->i_blkbits = ilog2(attr->blksize); + if (attr->blksize) + fi->cached_i_blkbits = ilog2(attr->blksize); else - inode->i_blkbits = inode->i_sb->s_blocksize_bits; + fi->cached_i_blkbits = fc->blkbits; /* * Don't set the sticky bit in i_mode, unless we want the VFS @@ -398,22 +407,11 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, evict_ctr); } - /* - * Clear basic stats from invalid mask. - * - * Don't do this if this is coming from a fuse_iget() call and there - * might have been a racing evict which would've invalidated the result - * if the attr_version would've been preserved. - * - * !evict_ctr -> this is create - * fi->attr_version != 0 -> this is not a new inode - * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request - */ - if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) + /* Clear basic stats from invalid mask */ + if (fuse_may_validate_attrs(fc, fi, evict_ctr)) set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); fi->attr_version = atomic64_inc_return(&fc->attr_version); - wake_up_all(&fc->attr_version_waitq); fi->i_time = attr_valid; fi->i_perm_time = attr_valid; @@ -864,6 +862,7 @@ static void fuse_prune_aliases(struct inode *inode) spin_lock(&inode->i_lock); hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { + fuse_entry_invalidated(dentry); fuse_invalidate_entry_cache(dentry); } spin_unlock(&inode->i_lock); @@ -880,6 +879,7 @@ static void fuse_invalidate_inode_entry(struct inode *inode) dentry = d_find_alias(inode); if (dentry) { d_invalidate(dentry); + fuse_entry_invalidated(dentry); fuse_invalidate_entry_cache(dentry); dput(dentry); } @@ -891,6 +891,7 @@ static void fuse_invalidate_inode_entry(struct inode *inode) if (!d_unhashed(dentry)) __d_drop(dentry); spin_unlock(&dentry->d_lock); + fuse_entry_invalidated(dentry); fuse_invalidate_entry_cache(dentry); } spin_unlock(&inode->i_lock); @@ -957,18 +958,26 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, uint64_t lock_end; inode = fuse_ilookup(fc, nodeid, NULL); - if (!inode) - return -ENOENT; + if (!inode) { + /* + * Not in the icache, but a LOOKUP/READDIRPLUS reply carrying + * pre-invalidation attributes may be about to create it. Bump + * evict_ctr so that such a reply leaves the new inode's + * attributes invalid (see fuse_change_attributes_common()), + * then look up again in case the inode was hashed while we + * were bumping. ilookup5() and iget5_locked() serialize on + * the inode hash lock, so if the second lookup still misses, + * the creator inserted after it and is guaranteed to observe + * the bumped counter. + */ + atomic64_inc(&fc->evict_ctr); + inode = fuse_ilookup(fc, nodeid, NULL); + if (!inode) + return -ENOENT; + } fi = get_fuse_inode(inode); - spin_lock(&fi->lock); - while (fi->attr_version == 0) { - spin_unlock(&fi->lock); - wait_event(fc->attr_version_waitq, READ_ONCE(fi->attr_version) != 0); - spin_lock(&fi->lock); - } - fi->attr_version = atomic64_inc_return(&fc->attr_version); spin_unlock(&fi->lock); @@ -1470,7 +1479,6 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, atomic_set(&fc->dev_count, 1); atomic_set(&fc->epoch, 1); init_waitqueue_head(&fc->blocked_waitq); - init_waitqueue_head(&fc->attr_version_waitq); fuse_iqueue_init(&fc->iq, fiq_ops, fiq_priv); INIT_LIST_HEAD(&fc->bg_queue); INIT_LIST_HEAD(&fc->entry);