From: Méven Car IORING_OP_UTIMENSAT always went to io-wq. On tmpfs the setattr only changes the inode in memory, so the handoff to a worker cost more than the work: 2000 files took 7 ms with utimensat(2) and 20 ms through the ring. Try the request inline first. The lookup uses LOOKUP_CACHED, and the freeze protection and the inode lock are only tried, the former with the new mnt_want_write_trylock(). A filesystem whose ->setattr() never sleeps for a change of times alone says so with FS_SETATTR_TIMES_NOWAIT, which tmpfs sets. Any other filesystem, an uncached path, a contended lock or a delegation returns -EAGAIN, and io_uring sends the request to io-wq as before. Signed-off-by: Méven Car --- fs/internal.h | 5 ++++- fs/namespace.c | 20 ++++++++++++++++++++ fs/utimes.c | 45 +++++++++++++++++++++++++++++++++++---------- include/linux/fs.h | 1 + io_uring/fs.c | 22 ++++++++++++++-------- mm/shmem.c | 3 ++- 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/fs/internal.h b/fs/internal.h index 985b2bc5895e..2370faec54f5 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -64,7 +64,9 @@ int filename_symlinkat(struct filename *from, int newdfd, struct filename *to); int filename_linkat(int olddfd, struct filename *old, int newdfd, struct filename *new, int flags); int filename_utimensat(int dfd, struct filename *name, - struct timespec64 *times, int flags); + struct timespec64 *times, int flags, bool nowait); +int __vfs_utimes(const struct path *path, struct timespec64 *times, + bool nowait); int vfs_tmpfile(struct mnt_idmap *idmap, const struct path *parentpath, struct file *file, umode_t mode); @@ -86,6 +88,7 @@ extern int sb_prepare_remount_readonly(struct super_block *); extern void __init mnt_init(void); int mnt_get_write_access_file(struct file *file); +int mnt_want_write_trylock(struct vfsmount *m); void mnt_put_write_access_file(struct file *file); extern void dissolve_on_fput(struct vfsmount *); diff --git a/fs/namespace.c b/fs/namespace.c index ae5dc64f8b45..f4aadd4e1958 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -499,6 +499,26 @@ int mnt_want_write(struct vfsmount *m) } EXPORT_SYMBOL_GPL(mnt_want_write); +/** + * mnt_want_write_trylock - get write access to a mount without waiting + * @m: the mount on which to take a write + * + * Like mnt_want_write(), but returns -EAGAIN instead of waiting when the + * filesystem is frozen. When the write operation is finished, + * mnt_drop_write() must be called. + */ +int mnt_want_write_trylock(struct vfsmount *m) +{ + int ret; + + if (!sb_start_write_trylock(m->mnt_sb)) + return -EAGAIN; + ret = mnt_get_write_access(m); + if (ret) + sb_end_write(m->mnt_sb); + return ret; +} + /** * mnt_get_write_access_file - get write access to a file's mount * @file: the file who's mount on which to take a write diff --git a/fs/utimes.c b/fs/utimes.c index 8072f05ac8dc..a0d67b8e96a1 100644 --- a/fs/utimes.c +++ b/fs/utimes.c @@ -18,7 +18,9 @@ static bool nsec_valid(long nsec) return nsec >= 0 && nsec <= 999999999; } -int vfs_utimes(const struct path *path, struct timespec64 *times) +/* With @nowait, returns -EAGAIN instead of sleeping */ +int __vfs_utimes(const struct path *path, struct timespec64 *times, + bool nowait) { int error; struct iattr newattrs; @@ -34,9 +36,17 @@ int vfs_utimes(const struct path *path, struct timespec64 *times) times = NULL; } - error = mnt_want_write(path->mnt); - if (error) - goto out; + if (nowait) { + if (!(inode->i_sb->s_type->fs_flags & FS_SETATTR_TIMES_NOWAIT)) + return -EAGAIN; + error = mnt_want_write_trylock(path->mnt); + if (error) + goto out; + } else { + error = mnt_want_write(path->mnt); + if (error) + goto out; + } newattrs.ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_ATIME; if (times) { @@ -63,24 +73,35 @@ int vfs_utimes(const struct path *path, struct timespec64 *times) newattrs.ia_valid |= ATTR_TOUCH; } retry_deleg: - inode_lock(inode); + if (!nowait) { + inode_lock(inode); + } else if (!inode_trylock(inode)) { + error = -EAGAIN; + goto drop; + } + /* Without a delegated_inode, a delegation gives -EWOULDBLOCK */ error = notify_change(mnt_idmap(path->mnt), path->dentry, &newattrs, - &delegated_inode); + nowait ? NULL : &delegated_inode); inode_unlock(inode); if (is_delegated(&delegated_inode)) { error = break_deleg_wait(&delegated_inode); if (!error) goto retry_deleg; } - +drop: mnt_drop_write(path->mnt); out: return error; } + +int vfs_utimes(const struct path *path, struct timespec64 *times) +{ + return __vfs_utimes(path, times, false); +} EXPORT_SYMBOL_GPL(vfs_utimes); int filename_utimensat(int dfd, struct filename *name, - struct timespec64 *times, int flags) + struct timespec64 *times, int flags, bool nowait) { struct path path; int lookup_flags = 0, error; @@ -90,13 +111,17 @@ int filename_utimensat(int dfd, struct filename *name, if (!(flags & AT_SYMLINK_NOFOLLOW)) lookup_flags |= LOOKUP_FOLLOW; + if (nowait) + lookup_flags |= LOOKUP_CACHED; retry: error = filename_lookup(dfd, name, lookup_flags, &path, NULL); if (error) return error; - error = vfs_utimes(&path, times); + error = __vfs_utimes(&path, times, nowait); path_put(&path); + if (nowait && error == -ESTALE) + return -EAGAIN; if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; goto retry; @@ -108,7 +133,7 @@ static int do_utimes_path(int dfd, const char __user *filename, struct timespec64 *times, int flags) { CLASS(filename_uflags, name)(filename, flags); - return filename_utimensat(dfd, name, times, flags); + return filename_utimensat(dfd, name, times, flags, false); } static int do_utimes_fd(int fd, struct timespec64 *times, int flags) diff --git a/include/linux/fs.h b/include/linux/fs.h index f9d1e05e8ae6..d1800fba15d8 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2296,6 +2296,7 @@ struct file_system_type { #define FS_POWER_FREEZE 256 /* Always freeze on suspend/hibernate */ #define FS_USERNS_MOUNT_RESTRICTED 512 /* Restrict mount in userns if not already visible */ #define FS_USERNS_DELEGATABLE 1024 /* Can be mounted inside userns from outside */ +#define FS_SETATTR_TIMES_NOWAIT 2048 /* ->setattr() of times alone never sleeps */ #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ int (*init_fs_context)(struct fs_context *); const struct fs_parameter_spec *parameters; diff --git a/io_uring/fs.c b/io_uring/fs.c index 71164bddd958..664ea918ae3b 100644 --- a/io_uring/fs.c +++ b/io_uring/fs.c @@ -358,34 +358,40 @@ int io_utimensat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (ut->flags) return -EINVAL; ut->flags |= IO_UTIMENSAT_NO_PATH; - goto out; + return 0; } err = delayed_getname_uflags(&ut->filename, fname, ut->flags); if (unlikely(err)) return err; req->flags |= REQ_F_NEED_CLEANUP; -out: - req->flags |= REQ_F_FORCE_ASYNC; return 0; } int io_utimensat(struct io_kiocb *req, unsigned int issue_flags) { struct io_utimensat *ut = io_kiocb_to_cmd(req, struct io_utimensat); + bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; int ret = 0; - WARN_ON_ONCE((issue_flags & IO_URING_F_NONBLOCK) && - !(ut->flags & IO_UTIMENSAT_NOOP)); - if (ut->flags & IO_UTIMENSAT_NO_PATH) { CLASS(fd, f)(ut->dfd); ret = -EBADF; if (!fd_empty(f)) - ret = vfs_utimes(&fd_file(f)->f_path, ut->times); + ret = __vfs_utimes(&fd_file(f)->f_path, ut->times, + force_nonblock); + if (ret == -EAGAIN && force_nonblock) + return -EAGAIN; } else if (!(ut->flags & IO_UTIMENSAT_NOOP)) { CLASS(filename_complete_delayed, name)(&ut->filename); - ret = filename_utimensat(ut->dfd, name, ut->times, ut->flags); + ret = filename_utimensat(ut->dfd, name, ut->times, ut->flags, + force_nonblock); + if (ret == -EAGAIN && force_nonblock) { + ret = putname_to_delayed(&ut->filename, + no_free_ptr(name)); + if (likely(!ret)) + return -EAGAIN; + } } req->flags &= ~REQ_F_NEED_CLEANUP; diff --git a/mm/shmem.c b/mm/shmem.c index 897fa2b61346..405b39fc5d0e 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -5310,7 +5310,8 @@ static struct file_system_type shmem_fs_type = { .parameters = shmem_fs_parameters, #endif .kill_sb = kill_anon_super, - .fs_flags = FS_USERNS_MOUNT | FS_ALLOW_IDMAP | FS_MGTIME, + .fs_flags = FS_USERNS_MOUNT | FS_ALLOW_IDMAP | FS_MGTIME | + FS_SETATTR_TIMES_NOWAIT, }; #if defined(CONFIG_SYSFS) && defined(CONFIG_TMPFS) -- 2.56.0