io_init_req() currently calls io_init_drain() and consumes IO_RING_F_DRAIN_NEXT before request initialization and BPF filter checks have completed. If io_init_req(), io_uring_run_bpf_filters(), or a later request in the same link fails, io_submit_fail_init() completes the request or link via io_req_defer_failed(), bypassing io_drain_req(). This can leave IO_RING_F_DRAIN_ACTIVE or IO_RING_F_DRAIN_NEXT set on the ring with no deferred drain request and ctx->drain_req == NULL: 1. P0 (e.g. POLL_ADD) is in flight 2. Invalid IOSQE_IO_DRAIN SQE sets IO_RING_F_DRAIN_ACTIVE, then fails with -EINVAL and frees immediately 3. Subsequent non-drain SQE sees IO_RING_F_DRAIN_ACTIVE, is forced through io_drain_req(), and blocks in ->defer_list behind P0 Similarly, a failed link containing an IOSQE_IO_DRAIN SQE leaks IO_RING_F_DRAIN_ACTIVE and IO_RING_F_DRAIN_NEXT onto the next unrelated request, or an -ENOMEM failure allocating io_defer_entry in io_drain_req() fails the request with IO_RING_F_DRAIN_ACTIVE still set. Do not touch IO_RING_F_DRAIN_ACTIVE or IO_RING_F_DRAIN_NEXT in io_init_req(). During link assembly, only propagate REQ_F_IO_DRAIN to a non-failing link head, and move drain flag activation into io_drain_req() after struct io_defer_entry allocation succeeds. Also clear state->link.head before flushing an unterminated link in io_submit_state_end(). Fixes: 8fb7aee05591 ("io_uring: drain based on allocates reqs") Cc: stable@vger.kernel.org Signed-off-by: Zhiqi Zhang --- io_uring/io_uring.c | 100 +++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 5db57bf675f0..4e8b93a584b4 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1471,8 +1471,9 @@ static __cold void io_drain_req(struct io_kiocb *req) __must_hold(&ctx->uring_lock) { struct io_ring_ctx *ctx = req->ctx; - bool drain = req->flags & IOSQE_IO_DRAIN; struct io_defer_entry *de; + bool drain_next = false; + bool drain; de = kmalloc_obj(*de, GFP_KERNEL_ACCOUNT); if (!de) { @@ -1480,6 +1481,33 @@ static __cold void io_drain_req(struct io_kiocb *req) return; } + /* + * If a request in the middle or end of a link needs draining, drain + * the link head (propagated at link assembly) and the next request/link + * after the current link. + */ + if (unlikely(req->link)) { + struct io_kiocb *cur; + + io_for_each_link(cur, req->link) { + if (cur->flags & REQ_F_IO_DRAIN) { + drain_next = true; + break; + } + } + } + if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT)) { + ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT; + ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE; + req->flags |= REQ_F_IO_DRAIN; + } + if (unlikely(req->flags & REQ_F_IO_DRAIN)) + ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE; + if (unlikely(drain_next)) + ctx->int_flags |= IO_RING_F_DRAIN_NEXT; + req->flags |= REQ_F_FORCE_ASYNC; + drain = req->flags & REQ_F_IO_DRAIN; + io_prep_async_link(req); trace_io_uring_defer(req); de->req = req; @@ -1801,6 +1829,13 @@ static inline void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags) io_queue_async(req, issue_flags, ret); } +static inline bool io_drain_needed(const struct io_ring_ctx *ctx, + const struct io_kiocb *req) +{ + return (ctx->int_flags & (IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT)) || + (req->flags & REQ_F_IO_DRAIN); +} + static void io_queue_sqe_fallback(struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) { @@ -1815,7 +1850,7 @@ static void io_queue_sqe_fallback(struct io_kiocb *req) } else { /* can't fail with IO_URING_F_INLINE */ io_req_sqe_copy(req, IO_URING_F_INLINE); - if (unlikely(req->ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) + if (unlikely(io_drain_needed(req->ctx, req))) io_drain_req(req); else io_queue_iowq(req); @@ -1847,24 +1882,6 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx, return true; } -static void io_init_drain(struct io_ring_ctx *ctx) -{ - struct io_kiocb *head = ctx->submit_state.link.head; - - ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE; - if (head) { - /* - * If we need to drain a request in the middle of a link, drain - * the head request and the next request/link after the current - * link. Considering sequential execution of links, - * REQ_F_IO_DRAIN will be maintained for every request of our - * link. - */ - head->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC; - ctx->int_flags |= IO_RING_F_DRAIN_NEXT; - } -} - static __cold int io_init_fail_req(struct io_kiocb *req, int err) { /* ensure per-opcode data is cleared if we fail before prep */ @@ -1937,25 +1954,13 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, } if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS) ctx->int_flags |= IO_RING_F_DRAIN_DISABLED; - if (sqe_flags & IOSQE_IO_DRAIN) { - if (ctx->int_flags & IO_RING_F_DRAIN_DISABLED) - return io_init_fail_req(req, -EOPNOTSUPP); - io_init_drain(ctx); - } - } - if (unlikely(ctx->int_flags & (IO_RING_F_OP_RESTRICTED | IO_RING_F_DRAIN_ACTIVE | IO_RING_F_DRAIN_NEXT))) { - if (!io_check_restriction(ctx, req, sqe_flags)) - return io_init_fail_req(req, -EACCES); - /* knock it to the slow queue path, will be drained there */ - if (ctx->int_flags & IO_RING_F_DRAIN_ACTIVE) - req->flags |= REQ_F_FORCE_ASYNC; - /* if there is no link, we're at "next" request and need to drain */ - if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_NEXT) && !ctx->submit_state.link.head) { - ctx->int_flags &= ~IO_RING_F_DRAIN_NEXT; - ctx->int_flags |= IO_RING_F_DRAIN_ACTIVE; - req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC; - } + if ((sqe_flags & IOSQE_IO_DRAIN) && + (ctx->int_flags & IO_RING_F_DRAIN_DISABLED)) + return io_init_fail_req(req, -EOPNOTSUPP); } + if (unlikely(ctx->int_flags & IO_RING_F_OP_RESTRICTED) && + !io_check_restriction(ctx, req, sqe_flags)) + return io_init_fail_req(req, -EACCES); if (!def->ioprio && sqe->ioprio) return io_init_fail_req(req, -EINVAL); @@ -2065,17 +2070,22 @@ static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, io_req_sqe_copy(req, IO_URING_F_INLINE); link->last->link = req; link->last = req; + if (unlikely((req->flags & REQ_F_IO_DRAIN) && + !(link->head->flags & REQ_F_FAIL))) + link->head->flags |= REQ_F_IO_DRAIN; if (req->flags & IO_REQ_LINK_FLAGS) return 0; /* last request of the link, flush it */ req = link->head; link->head = NULL; - if (req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)) + if ((req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)) || + io_drain_needed(ctx, req)) goto fallback; - } else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS | - REQ_F_FORCE_ASYNC | REQ_F_FAIL))) { + } else if (unlikely((req->flags & (IO_REQ_LINK_FLAGS | REQ_F_FORCE_ASYNC | + REQ_F_FAIL)) || + io_drain_needed(ctx, req))) { if (req->flags & IO_REQ_LINK_FLAGS) { link->head = req; link->last = req; @@ -2097,8 +2107,12 @@ static void io_submit_state_end(struct io_ring_ctx *ctx) { struct io_submit_state *state = &ctx->submit_state; - if (unlikely(state->link.head)) - io_queue_sqe_fallback(state->link.head); + if (unlikely(state->link.head)) { + struct io_kiocb *head = state->link.head; + + state->link.head = NULL; + io_queue_sqe_fallback(head); + } /* flush only after queuing links as they can generate completions */ io_submit_flush_completions(ctx); if (state->plug_started) base-commit: 666539eecdf4db623f5e3c3e55cf9f2dfad24aa4 -- 2.43.0