Allow kernel-managed buffers to be selected. This requires modifying the io_br_sel struct to separate the fields for address and val, since a kernel address cannot be distinguished from a negative val when error checking. Auto-commit any selected kernel-managed buffer. Signed-off-by: Joanne Koong --- include/linux/io_uring_types.h | 8 ++++---- io_uring/kbuf.c | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index e1adb0d20a0a..36fac08db636 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -93,13 +93,13 @@ struct io_mapped_region { */ struct io_br_sel { struct io_buffer_list *buf_list; - /* - * Some selection parts return the user address, others return an error. - */ union { + /* for classic/ring provided buffers */ void __user *addr; - ssize_t val; + /* for kernel-managed buffers */ + void *kaddr; }; + ssize_t val; }; diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 65102aaadd15..c98cecb56b8c 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -155,7 +155,8 @@ static int io_provided_buffers_select(struct io_kiocb *req, size_t *len, return 1; } -static bool io_should_commit(struct io_kiocb *req, unsigned int issue_flags) +static bool io_should_commit(struct io_kiocb *req, struct io_buffer_list *bl, + unsigned int issue_flags) { /* * If we came in unlocked, we have no choice but to consume the @@ -170,7 +171,11 @@ static bool io_should_commit(struct io_kiocb *req, unsigned int issue_flags) if (issue_flags & IO_URING_F_UNLOCKED) return true; - /* uring_cmd commits kbuf upfront, no need to auto-commit */ + /* kernel-managed buffers are auto-committed */ + if (bl->flags & IOBL_KERNEL_MANAGED) + return true; + + /* multishot uring_cmd commits kbuf upfront, no need to auto-commit */ if (!io_file_can_poll(req) && req->opcode != IORING_OP_URING_CMD) return true; return false; @@ -201,8 +206,12 @@ static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->buf_index = READ_ONCE(buf->bid); sel.buf_list = bl; sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr)); + if (bl->flags & IOBL_KERNEL_MANAGED) + sel.kaddr = (void *)buf->addr; + else + sel.addr = u64_to_user_ptr(READ_ONCE(buf->addr)); - if (io_should_commit(req, issue_flags)) { + if (io_should_commit(req, bl, issue_flags)) { io_kbuf_commit(req, sel.buf_list, *len, 1); sel.buf_list = NULL; } -- 2.47.3