Joanne has submitted a patch for adding a zero-copy solution. We have also done some research and testing before, and the test data shows improved performance. This patch is submitted for discussion. libfuse section: https://github.com/lreeze123/libfuse/tree/zero-copy Signed-off-by: Xiaobing Li --- fs/fuse/dev_uring.c | 44 +++++++++++++++++++++++++++++++------------ fs/fuse/dev_uring_i.h | 4 ++++ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index f6b12aebb8bb..23790ae78853 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -584,15 +584,20 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, int err; struct fuse_uring_ent_in_out ring_in_out; - err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out, - sizeof(ring_in_out)); - if (err) - return -EFAULT; + if (ent->zero_copy) { + iter = ent->payload_iter; + ring_in_out.payload_sz = ent->cmd->sqe->len; + } else { + err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out, + sizeof(ring_in_out)); + if (err) + return -EFAULT; - err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, - &iter); - if (err) - return err; + err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, + &iter); + if (err) + return err; + } fuse_copy_init(&cs, false, &iter); cs.is_uring = true; @@ -618,10 +623,14 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); - return err; + if (ent->zero_copy) { + iter = ent->payload_iter; + } else { + err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } } fuse_copy_init(&cs, true, &iter); @@ -1068,6 +1077,17 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->headers = iov[0].iov_base; ent->payload = iov[1].iov_base; + if (READ_ONCE(cmd->sqe->uring_cmd_flags) & IORING_URING_CMD_FIXED) { + ent->zero_copy = true; + err = io_uring_cmd_import_fixed((u64)ent->payload, payload_size, ITER_DEST, + &ent->payload_iter, cmd, 0); + + if (err) { + kfree(ent); + return ERR_PTR(err); + } + } + atomic_inc(&ring->queue_refs); return ent; } diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 51a563922ce1..cb5de6e7a262 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -38,6 +38,10 @@ enum fuse_ring_req_state { /** A fuse ring entry, part of the ring queue */ struct fuse_ring_ent { + bool zero_copy; + + struct iov_iter payload_iter; + /* userspace buffer */ struct fuse_uring_req_header __user *headers; void __user *payload; -- 2.34.1