From: Anuj Gupta Let callers of io_uring_cmd_import_fixed() opt in to dma-buf registered buffers with IO_URING_CMD_IMPORT_DMABUF, and attach the current map on import. The map has to be dropped at hardware completion rather than from task_work. Add io_uring_cmd_hw_done() for drivers to call from end_io, which drops the map and queues task_work, and warn if task_work is queued with a map still attached. Signed-off-by: Anuj Gupta --- include/linux/io_uring/cmd.h | 9 +++++++++ io_uring/uring_cmd.c | 28 ++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index 23fb66a1e2c1..d4e17c2f59fd 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -12,6 +12,9 @@ /* io_uring_cmd is being issued again */ #define IORING_URING_CMD_REISSUE (1U << 31) +/* io_uring_cmd_import_fixed() flags */ +#define IO_URING_CMD_IMPORT_DMABUF (1U << 0) + struct io_uring_cmd { struct file *file; const struct io_uring_sqe *sqe; @@ -86,6 +89,8 @@ void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags); void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, io_req_tw_func_t task_work_cb, unsigned flags); +void io_uring_cmd_hw_done(struct io_uring_cmd *ioucmd, + io_req_tw_func_t task_work_cb); /* * Note: the caller should never hard code @issue_flags and only use the @@ -145,6 +150,10 @@ static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, io_req_tw_func_t task_work_cb, unsigned flags) { } +static inline void io_uring_cmd_hw_done(struct io_uring_cmd *ioucmd, + io_req_tw_func_t task_work_cb) +{ +} static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, unsigned int issue_flags) { diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index c2bc9dd242ec..c89f154190b1 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -10,6 +10,7 @@ #include "io_uring.h" #include "alloc_cache.h" +#include "dma-buf.h" #include "rsrc.h" #include "kbuf.h" #include "uring_cmd.h" @@ -28,6 +29,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); struct io_async_cmd *ac = req->async_data; + io_detach_dmabuf_map(req); if (issue_flags & IO_URING_F_UNLOCKED) return; @@ -132,12 +134,26 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT)) return; + /* the dma-buf map must be dropped first, see io_uring_cmd_hw_done() */ + WARN_ON_ONCE(req->dmabuf_map); req->io_task_work.func = task_work_cb; __io_req_task_work_add(req, flags); } EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task); +/* + * Queue completion in task context after dropping references that must be + * released at hardware completion rather than from task_work. + */ +void io_uring_cmd_hw_done(struct io_uring_cmd *ioucmd, + io_req_tw_func_t task_work_cb) +{ + io_detach_dmabuf_map(cmd_to_io_kiocb(ioucmd)); + __io_uring_cmd_do_in_task(ioucmd, task_work_cb, IOU_F_TWQ_LAZY_WAKE); +} +EXPORT_SYMBOL_GPL(io_uring_cmd_hw_done); + /* * Called by consumers of io_uring_cmd, if they originally returned * -EIOCBQUEUED upon receiving the command. @@ -257,6 +273,7 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) ret = file->f_op->uring_cmd(ioucmd, issue_flags); if (ret == -EAGAIN) { + io_detach_dmabuf_map(req); ioucmd->flags |= IORING_URING_CMD_REISSUE; return ret; } @@ -276,13 +293,20 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, unsigned int import_flags) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); + unsigned int flags = 0; + int ret; if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED))) return -EINVAL; - if (WARN_ON_ONCE(import_flags)) + if (WARN_ON_ONCE(import_flags & ~IO_URING_CMD_IMPORT_DMABUF)) return -EINVAL; + if (import_flags & IO_URING_CMD_IMPORT_DMABUF) + flags |= IO_REGBUF_IMPORT_ALLOW_DMABUF; - return io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags, 0); + ret = io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags, flags); + if (ret) + return ret; + return io_attach_dmabuf_map(req, iter, issue_flags); } EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed); -- 2.43.0