Add an API for fetching the registered buffer associated with a io_uring cmd. This is useful for callers who need access to the buffer but do not have prior knowledge of the buffer's user address or length. Signed-off-by: Joanne Koong --- include/linux/io_uring/cmd.h | 3 +++ io_uring/rsrc.c | 14 ++++++++++++++ io_uring/rsrc.h | 2 ++ io_uring/uring_cmd.c | 13 +++++++++++++ 4 files changed, 32 insertions(+) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index 7509025b4071..8c11d9a92733 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -43,6 +43,9 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, struct io_uring_cmd *ioucmd, unsigned int issue_flags); +int io_uring_cmd_import_fixed_full(int rw, struct iov_iter *iter, + struct io_uring_cmd *ioucmd, + unsigned int issue_flags); int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, const struct iovec __user *uvec, size_t uvec_segs, diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index d787c16dc1c3..2c3d8489ae52 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1147,6 +1147,20 @@ int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter, return io_import_fixed(ddir, iter, node->buf, buf_addr, len); } +int io_import_reg_buf_full(struct io_kiocb *req, struct iov_iter *iter, + int ddir, unsigned issue_flags) +{ + struct io_rsrc_node *node; + struct io_mapped_ubuf *imu; + + node = io_find_buf_node(req, issue_flags); + if (!node) + return -EFAULT; + + imu = node->buf; + return io_import_fixed(ddir, iter, imu, imu->ubuf, imu->len); +} + /* Lock two rings at once. The rings must be different! */ static void lock_two_rings(struct io_ring_ctx *ctx1, struct io_ring_ctx *ctx2) { diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index a3ca6ba66596..4e01eb0f277e 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -64,6 +64,8 @@ struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req, int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter, u64 buf_addr, size_t len, int ddir, unsigned issue_flags); +int io_import_reg_buf_full(struct io_kiocb *req, struct iov_iter *iter, + int ddir, unsigned issue_flags); int io_import_reg_vec(int ddir, struct iov_iter *iter, struct io_kiocb *req, struct iou_vec *vec, unsigned nr_iovs, unsigned issue_flags); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index d1e3ba62ee8e..07730ced9449 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -292,6 +292,19 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, } EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed); +int io_uring_cmd_import_fixed_full(int rw, struct iov_iter *iter, + struct io_uring_cmd *ioucmd, + unsigned int issue_flags) +{ + struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); + + if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED))) + return -EINVAL; + + return io_import_reg_buf_full(req, iter, rw, issue_flags); +} +EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_full); + int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, const struct iovec __user *uvec, size_t uvec_segs, -- 2.47.3 Simplify the logic for getting the next fuse request. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 78 ++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 50 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index f6b12aebb8bb..415924b346c0 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -710,34 +710,6 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, return err; } -/* - * Write data to the ring buffer and send the request to userspace, - * userspace will read it - * This is comparable with classical read(/dev/fuse) - */ -static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent, - struct fuse_req *req, - unsigned int issue_flags) -{ - struct fuse_ring_queue *queue = ent->queue; - int err; - struct io_uring_cmd *cmd; - - err = fuse_uring_prepare_send(ent, req); - if (err) - return err; - - spin_lock(&queue->lock); - cmd = ent->cmd; - ent->cmd = NULL; - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - spin_unlock(&queue->lock); - - io_uring_cmd_done(cmd, 0, issue_flags); - return 0; -} - /* * Make a ring entry available for fuse_req assignment */ @@ -834,11 +806,13 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, } /* - * Get the next fuse req and send it + * Get the next fuse req. + * + * Returns true if the next fuse request has been assigned to the ent. + * Else, there is no next fuse request and this returns false. */ -static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, - struct fuse_ring_queue *queue, - unsigned int issue_flags) +static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent, + struct fuse_ring_queue *queue) { int err; struct fuse_req *req; @@ -850,10 +824,12 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent, spin_unlock(&queue->lock); if (req) { - err = fuse_uring_send_next_to_ring(ent, req, issue_flags); + err = fuse_uring_prepare_send(ent, req); if (err) goto retry; } + + return req != NULL; } static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) @@ -871,6 +847,20 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) return 0; } +static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, + ssize_t ret, unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + ent->state = FRRS_USERSPACE; + list_move_tail(&ent->list, &queue->ent_in_userspace); + ent->cmd = NULL; + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, ret, issue_flags); +} + /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, struct fuse_conn *fc) @@ -942,7 +932,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * and fetching is done in one step vs legacy fuse, which has separated * read (fetch request) and write (commit result). */ - fuse_uring_next_fuse_req(ent, queue, issue_flags); + if (fuse_uring_get_next_fuse_req(ent, queue)) + fuse_uring_send(ent, cmd, 0, issue_flags); return 0; } @@ -1190,20 +1181,6 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return -EIOCBQUEUED; } -static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, - ssize_t ret, unsigned int issue_flags) -{ - struct fuse_ring_queue *queue = ent->queue; - - spin_lock(&queue->lock); - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - ent->cmd = NULL; - spin_unlock(&queue->lock); - - io_uring_cmd_done(cmd, ret, issue_flags); -} - /* * This prepares and sends the ring request in fuse-uring task context. * User buffers are not mapped yet - the application does not have permission @@ -1219,8 +1196,9 @@ static void fuse_uring_send_in_task(struct io_uring_cmd *cmd, if (!(issue_flags & IO_URING_F_TASK_DEAD)) { err = fuse_uring_prepare_send(ent, ent->fuse_req); if (err) { - fuse_uring_next_fuse_req(ent, queue, issue_flags); - return; + if (!fuse_uring_get_next_fuse_req(ent, queue)) + return; + err = 0; } } else { err = -ECANCELED; -- 2.47.3 Move header copying to ring logic into a new copy_header_to_ring() function. This consolidates error handling. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 415924b346c0..e94af90d4d46 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -574,6 +574,17 @@ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, return err; } +static int copy_header_to_ring(void __user *ring, const void *header, + size_t header_size) +{ + if (copy_to_user(ring, header, header_size)) { + pr_info_ratelimited("Copying header to ring failed.\n"); + return -EFAULT; + } + + return 0; +} + static int fuse_uring_copy_from_ring(struct fuse_ring *ring, struct fuse_req *req, struct fuse_ring_ent *ent) @@ -634,13 +645,11 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, * Some op code have that as zero size. */ if (args->in_args[0].size > 0) { - err = copy_to_user(&ent->headers->op_in, in_args->value, - in_args->size); - if (err) { - pr_info_ratelimited( - "Copying the header failed.\n"); - return -EFAULT; - } + err = copy_header_to_ring(&ent->headers->op_in, + in_args->value, + in_args->size); + if (err) + return err; } in_args++; num_args--; @@ -655,9 +664,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; - err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out, - sizeof(ent_in_out)); - return err ? -EFAULT : 0; + return copy_header_to_ring(&ent->headers->ring_ent_in_out, &ent_in_out, + sizeof(ent_in_out)); } static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, @@ -686,14 +694,8 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - err = copy_to_user(&ent->headers->in_out, &req->in.h, - sizeof(req->in.h)); - if (err) { - err = -EFAULT; - return err; - } - - return 0; + return copy_header_to_ring(&ent->headers->in_out, &req->in.h, + sizeof(req->in.h)); } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, -- 2.47.3 Move header copying from ring logic into a new copy_header_from_ring() function. This consolidates error handling. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index e94af90d4d46..faa7217e85c4 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -585,6 +585,17 @@ static int copy_header_to_ring(void __user *ring, const void *header, return 0; } +static int copy_header_from_ring(void *header, const void __user *ring, + size_t header_size) +{ + if (copy_from_user(header, ring, header_size)) { + pr_info_ratelimited("Copying header from ring failed.\n"); + return -EFAULT; + } + + return 0; +} + static int fuse_uring_copy_from_ring(struct fuse_ring *ring, struct fuse_req *req, struct fuse_ring_ent *ent) @@ -595,10 +606,10 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, int err; struct fuse_uring_ent_in_out ring_in_out; - err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out, - sizeof(ring_in_out)); + err = copy_header_from_ring(&ring_in_out, &ent->headers->ring_ent_in_out, + sizeof(ring_in_out)); if (err) - return -EFAULT; + return err; err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, &iter); @@ -789,10 +800,10 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, struct fuse_conn *fc = ring->fc; ssize_t err = 0; - err = copy_from_user(&req->out.h, &ent->headers->in_out, - sizeof(req->out.h)); + err = copy_header_from_ring(&req->out.h, &ent->headers->in_out, + sizeof(req->out.h)); if (err) { - req->out.h.error = -EFAULT; + req->out.h.error = err; goto out; } -- 2.47.3 Use enum types to identify which part of the header needs to be copied. This improves the interface and will simplify both kernel-space and user-space header addresses when fixed buffer support is added. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 55 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index faa7217e85c4..d96368e93e8d 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -31,6 +31,12 @@ struct fuse_uring_pdu { static const struct fuse_iqueue_ops fuse_io_uring_ops; +enum fuse_uring_header_type { + FUSE_URING_HEADER_IN_OUT, + FUSE_URING_HEADER_OP, + FUSE_URING_HEADER_RING_ENT, +}; + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -574,9 +580,31 @@ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, return err; } -static int copy_header_to_ring(void __user *ring, const void *header, - size_t header_size) +static void __user *get_user_ring_header(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type) +{ + switch (type) { + case FUSE_URING_HEADER_IN_OUT: + return &ent->headers->in_out; + case FUSE_URING_HEADER_OP: + return &ent->headers->op_in; + case FUSE_URING_HEADER_RING_ENT: + return &ent->headers->ring_ent_in_out; + } + + WARN_ON_ONCE(1); + return NULL; +} + +static int copy_header_to_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, + const void *header, size_t header_size) { + void __user *ring = get_user_ring_header(ent, type); + + if (!ring) + return -EINVAL; + if (copy_to_user(ring, header, header_size)) { pr_info_ratelimited("Copying header to ring failed.\n"); return -EFAULT; @@ -585,9 +613,15 @@ static int copy_header_to_ring(void __user *ring, const void *header, return 0; } -static int copy_header_from_ring(void *header, const void __user *ring, - size_t header_size) +static int copy_header_from_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, + void *header, size_t header_size) { + const void __user *ring = get_user_ring_header(ent, type); + + if (!ring) + return -EINVAL; + if (copy_from_user(header, ring, header_size)) { pr_info_ratelimited("Copying header from ring failed.\n"); return -EFAULT; @@ -606,8 +640,8 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, int err; struct fuse_uring_ent_in_out ring_in_out; - err = copy_header_from_ring(&ring_in_out, &ent->headers->ring_ent_in_out, - sizeof(ring_in_out)); + err = copy_header_from_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ring_in_out, sizeof(ring_in_out)); if (err) return err; @@ -656,7 +690,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, * Some op code have that as zero size. */ if (args->in_args[0].size > 0) { - err = copy_header_to_ring(&ent->headers->op_in, + err = copy_header_to_ring(ent, FUSE_URING_HEADER_OP, in_args->value, in_args->size); if (err) @@ -675,7 +709,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; - return copy_header_to_ring(&ent->headers->ring_ent_in_out, &ent_in_out, + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ent_in_out, sizeof(ent_in_out)); } @@ -705,7 +740,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - return copy_header_to_ring(&ent->headers->in_out, &req->in.h, + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h, sizeof(req->in.h)); } @@ -800,7 +835,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, struct fuse_conn *fc = ring->fc; ssize_t err = 0; - err = copy_header_from_ring(&req->out.h, &ent->headers->in_out, + err = copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, sizeof(req->out.h)); if (err) { req->out.h.error = err; -- 2.47.3 Rename the headers and payload fields to user_headers and user_payload. This makes it explicit that these pointers reference userspace addresses and prepares for upcoming fixed buffer support, where there will be separate fields for kernel-space pointers to the payload and headers. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 17 ++++++++--------- fs/fuse/dev_uring_i.h | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index d96368e93e8d..c814b571494f 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -585,11 +585,11 @@ static void __user *get_user_ring_header(struct fuse_ring_ent *ent, { switch (type) { case FUSE_URING_HEADER_IN_OUT: - return &ent->headers->in_out; + return &ent->user_headers->in_out; case FUSE_URING_HEADER_OP: - return &ent->headers->op_in; + return &ent->user_headers->op_in; case FUSE_URING_HEADER_RING_ENT: - return &ent->headers->ring_ent_in_out; + return &ent->user_headers->ring_ent_in_out; } WARN_ON_ONCE(1); @@ -645,7 +645,7 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, if (err) return err; - err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, + err = import_ubuf(ITER_SOURCE, ent->user_payload, ring->max_payload_sz, &iter); if (err) return err; @@ -674,7 +674,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter); + err = import_ubuf(ITER_DEST, ent->user_payload, ring->max_payload_sz, &iter); if (err) { pr_info_ratelimited("fuse: Import of user buffer failed\n"); return err; @@ -710,8 +710,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, ent_in_out.payload_sz = cs.ring.copied_sz; return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, - &ent_in_out, - sizeof(ent_in_out)); + &ent_in_out, sizeof(ent_in_out)); } static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, @@ -1104,8 +1103,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, INIT_LIST_HEAD(&ent->list); ent->queue = queue; - ent->headers = iov[0].iov_base; - ent->payload = iov[1].iov_base; + ent->user_headers = iov[0].iov_base; + ent->user_payload = iov[1].iov_base; atomic_inc(&ring->queue_refs); return ent; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 51a563922ce1..381fd0b8156a 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -39,8 +39,8 @@ enum fuse_ring_req_state { /** A fuse ring entry, part of the ring queue */ struct fuse_ring_ent { /* userspace buffer */ - struct fuse_uring_req_header __user *headers; - void __user *payload; + struct fuse_uring_req_header __user *user_headers; + void __user *user_payload; /* the ring queue that owns the request */ struct fuse_ring_queue *queue; -- 2.47.3 Add a new helper function setup_fuse_copy_state() to contain the logic for setting up the copy state for payload copying. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index c814b571494f..c6b22b14b354 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -630,6 +630,28 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, return 0; } +static int setup_fuse_copy_state(struct fuse_ring *ring, struct fuse_req *req, + struct fuse_ring_ent* ent, int rw, + struct iov_iter *iter, + struct fuse_copy_state *cs) +{ + int err; + + err = import_ubuf(rw, ent->user_payload, ring->max_payload_sz, + iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } + + fuse_copy_init(cs, rw == ITER_DEST, iter); + + cs->is_uring = true; + cs->req = req; + + return 0; +} + static int fuse_uring_copy_from_ring(struct fuse_ring *ring, struct fuse_req *req, struct fuse_ring_ent *ent) @@ -645,15 +667,10 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, if (err) return err; - err = import_ubuf(ITER_SOURCE, ent->user_payload, ring->max_payload_sz, - &iter); + err = setup_fuse_copy_state(ring, req, ent, ITER_SOURCE, &iter, &cs); if (err) return err; - fuse_copy_init(&cs, false, &iter); - cs.is_uring = true; - cs.req = req; - return fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); } @@ -674,15 +691,9 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = import_ubuf(ITER_DEST, ent->user_payload, ring->max_payload_sz, &iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); + err = setup_fuse_copy_state(ring, req, ent, ITER_DEST, &iter, &cs); + if (err) return err; - } - - fuse_copy_init(&cs, true, &iter); - cs.is_uring = true; - cs.req = req; if (num_args > 0) { /* -- 2.47.3 Add support for io-uring registered buffers for fuse daemons communicating through the io-uring interface. Daemons may register buffers ahead of time, which will eliminate the overhead of pinning/unpinning user pages and translating virtual addresses for every server-kernel interaction. To support page-aligned payloads, the buffer is structured such that the payload is at the front of the buffer and the fuse_uring_req_header is offset from the end of the buffer. To be backwards compatible, fuse uring still needs to support non-registered buffers as well. Signed-off-by: Joanne Koong --- fs/fuse/dev_uring.c | 200 +++++++++++++++++++++++++++++++++--------- fs/fuse/dev_uring_i.h | 27 +++++- 2 files changed, 183 insertions(+), 44 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index c6b22b14b354..f501bc81f331 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -580,6 +580,22 @@ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, return err; } +static void *get_kernel_ring_header(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type) +{ + switch (type) { + case FUSE_URING_HEADER_IN_OUT: + return &ent->headers->in_out; + case FUSE_URING_HEADER_OP: + return &ent->headers->op_in; + case FUSE_URING_HEADER_RING_ENT: + return &ent->headers->ring_ent_in_out; + } + + WARN_ON_ONCE(1); + return NULL; +} + static void __user *get_user_ring_header(struct fuse_ring_ent *ent, enum fuse_uring_header_type type) { @@ -600,16 +616,22 @@ static int copy_header_to_ring(struct fuse_ring_ent *ent, enum fuse_uring_header_type type, const void *header, size_t header_size) { - void __user *ring = get_user_ring_header(ent, type); + if (ent->fixed_buffer) { + void *ring = get_kernel_ring_header(ent, type); - if (!ring) - return -EINVAL; + if (!ring) + return -EINVAL; + memcpy(ring, header, header_size); + } else { + void __user *ring = get_user_ring_header(ent, type); - if (copy_to_user(ring, header, header_size)) { - pr_info_ratelimited("Copying header to ring failed.\n"); - return -EFAULT; + if (!ring) + return -EINVAL; + if (copy_to_user(ring, header, header_size)) { + pr_info_ratelimited("Copying header to ring failed.\n"); + return -EFAULT; + } } - return 0; } @@ -617,14 +639,21 @@ static int copy_header_from_ring(struct fuse_ring_ent *ent, enum fuse_uring_header_type type, void *header, size_t header_size) { - const void __user *ring = get_user_ring_header(ent, type); + if (ent->fixed_buffer) { + const void *ring = get_kernel_ring_header(ent, type); - if (!ring) - return -EINVAL; + if (!ring) + return -EINVAL; + memcpy(header, ring, header_size); + } else { + const void __user *ring = get_user_ring_header(ent, type); - if (copy_from_user(header, ring, header_size)) { - pr_info_ratelimited("Copying header from ring failed.\n"); - return -EFAULT; + if (!ring) + return -EINVAL; + if (copy_from_user(header, ring, header_size)) { + pr_info_ratelimited("Copying header from ring failed.\n"); + return -EFAULT; + } } return 0; @@ -637,11 +666,15 @@ static int setup_fuse_copy_state(struct fuse_ring *ring, struct fuse_req *req, { int err; - err = import_ubuf(rw, ent->user_payload, ring->max_payload_sz, - iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); - return err; + if (ent->fixed_buffer) { + *iter = ent->payload_iter; + } else { + err = import_ubuf(rw, ent->user_payload, ring->max_payload_sz, + iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } } fuse_copy_init(cs, rw == ITER_DEST, iter); @@ -754,6 +787,62 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, sizeof(req->in.h)); } +/* + * Prepare fixed buffer for access. Sets up the payload iter and kmaps the + * header. + * + * Callers must call fuse_uring_unmap_buffer() in the same scope to release the + * header mapping. + * + * For non-fixed buffers, this is a no-op. + */ +static int fuse_uring_map_buffer(struct fuse_ring_ent *ent) +{ + size_t header_size = sizeof(struct fuse_uring_req_header); + struct iov_iter iter; + struct page *header_page; + size_t count, start; + ssize_t copied; + int err; + + if (!ent->fixed_buffer) + return 0; + + err = io_uring_cmd_import_fixed_full(ITER_DEST, &iter, ent->cmd, 0); + if (err) + return err; + + count = iov_iter_count(&iter); + if (count < header_size || count & (PAGE_SIZE - 1)) + return -EINVAL; + + /* Adjust the payload iter to protect the header from any overwrites */ + ent->payload_iter = iter; + iov_iter_truncate(&ent->payload_iter, count - header_size); + + /* Set up the headers */ + iov_iter_advance(&iter, count - header_size); + copied = iov_iter_get_pages2(&iter, &header_page, header_size, 1, &start); + if (copied < header_size) + return -EFAULT; + ent->headers = kmap_local_page(header_page) + start; + + /* + * We can release the acquired reference on the header page immediately + * since the page is pinned and io_uring_cmd_import_fixed_full() + * prevents it from being unpinned while we are using it. + */ + put_page(header_page); + + return 0; +} + +static void fuse_uring_unmap_buffer(struct fuse_ring_ent *ent) +{ + if (ent->fixed_buffer) + kunmap_local(ent->headers); +} + static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, struct fuse_req *req) { @@ -932,6 +1021,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, unsigned int qid = READ_ONCE(cmd_req->qid); struct fuse_pqueue *fpq; struct fuse_req *req; + bool next_req; err = -ENOTCONN; if (!ring) @@ -982,6 +1072,13 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, /* without the queue lock, as other locks are taken */ fuse_uring_prepare_cancel(cmd, issue_flags, ent); + + err = fuse_uring_map_buffer(ent); + if (err) { + fuse_uring_req_end(ent, req, err); + return err; + } + fuse_uring_commit(ent, req, issue_flags); /* @@ -990,7 +1087,9 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, * and fetching is done in one step vs legacy fuse, which has separated * read (fetch request) and write (commit result). */ - if (fuse_uring_get_next_fuse_req(ent, queue)) + next_req = fuse_uring_get_next_fuse_req(ent, queue); + fuse_uring_unmap_buffer(ent); + if (next_req) fuse_uring_send(ent, cmd, 0, issue_flags); return 0; } @@ -1086,39 +1185,49 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, struct iovec iov[FUSE_URING_IOV_SEGS]; int err; + err = -ENOMEM; + ent = kzalloc(sizeof(*ent), GFP_KERNEL_ACCOUNT); + if (!ent) + return ERR_PTR(err); + + INIT_LIST_HEAD(&ent->list); + + ent->queue = queue; + + if (READ_ONCE(cmd->sqe->uring_cmd_flags) & IORING_URING_CMD_FIXED) { + ent->fixed_buffer = true; + atomic_inc(&ring->queue_refs); + return ent; + } + err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); if (err) { pr_info_ratelimited("Failed to get iovec from sqe, err=%d\n", err); - return ERR_PTR(err); + goto error; } err = -EINVAL; if (iov[0].iov_len < sizeof(struct fuse_uring_req_header)) { pr_info_ratelimited("Invalid header len %zu\n", iov[0].iov_len); - return ERR_PTR(err); + goto error; } payload_size = iov[1].iov_len; if (payload_size < ring->max_payload_sz) { pr_info_ratelimited("Invalid req payload len %zu\n", payload_size); - return ERR_PTR(err); + goto error; } - - err = -ENOMEM; - ent = kzalloc(sizeof(*ent), GFP_KERNEL_ACCOUNT); - if (!ent) - return ERR_PTR(err); - - INIT_LIST_HEAD(&ent->list); - - ent->queue = queue; ent->user_headers = iov[0].iov_base; ent->user_payload = iov[1].iov_base; atomic_inc(&ring->queue_refs); return ent; + +error: + kfree(ent); + return ERR_PTR(err); } /* @@ -1249,20 +1358,29 @@ static void fuse_uring_send_in_task(struct io_uring_cmd *cmd, { struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); struct fuse_ring_queue *queue = ent->queue; + bool send_ent = true; int err; - if (!(issue_flags & IO_URING_F_TASK_DEAD)) { - err = fuse_uring_prepare_send(ent, ent->fuse_req); - if (err) { - if (!fuse_uring_get_next_fuse_req(ent, queue)) - return; - err = 0; - } - } else { - err = -ECANCELED; + if (issue_flags & IO_URING_F_TASK_DEAD) { + fuse_uring_send(ent, cmd, -ECANCELED, issue_flags); + return; + } + + err = fuse_uring_map_buffer(ent); + if (err) { + fuse_uring_req_end(ent, ent->fuse_req, err); + return; + } + + err = fuse_uring_prepare_send(ent, ent->fuse_req); + if (err) { + send_ent = fuse_uring_get_next_fuse_req(ent, queue); + err = 0; } + fuse_uring_unmap_buffer(ent); - fuse_uring_send(ent, cmd, err, issue_flags); + if (send_ent) + fuse_uring_send(ent, cmd, err, issue_flags); } static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 381fd0b8156a..fe14acccd6a6 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -7,6 +7,7 @@ #ifndef _FS_FUSE_DEV_URING_I_H #define _FS_FUSE_DEV_URING_I_H +#include #include "fuse_i.h" #ifdef CONFIG_FUSE_IO_URING @@ -38,9 +39,29 @@ enum fuse_ring_req_state { /** A fuse ring entry, part of the ring queue */ struct fuse_ring_ent { - /* userspace buffer */ - struct fuse_uring_req_header __user *user_headers; - void __user *user_payload; + /* + * If true, the buffer was pre-registered by the daemon and the + * pages backing it are pinned in kernel memory. The fixed buffer layout + * is: [payload][header at end]. Use payload_iter and headers for + * copying to/from the ring. + * + * Otherwise, use user_headers and user_payload which point to userspace + * addresses representing the ring memory. + */ + bool fixed_buffer; + + union { + /* fixed_buffer == false */ + struct { + struct fuse_uring_req_header __user *user_headers; + void __user *user_payload; + }; + /* fixed_buffer == true */ + struct { + struct fuse_uring_req_header *headers; + struct iov_iter payload_iter; + }; + }; /* the ring queue that owns the request */ struct fuse_ring_queue *queue; -- 2.47.3