A bsg uring_cmd driver needs storage for completion data which outlives its block request when the request is freed in end-I/O. scsi_bsg needs to preserve sense data until task work can copy it to userspace. Allocate a pool when a bsg queue with a uring_cmd handler is registered, and free it when the bsg device is released. Each pool has BSG_DEFAULT_CMDS slots of SCSI_SENSE_BUFFERSIZE bytes, for the only in-tree uring_cmd consumer, scsi_bsg. Transport bsg devices and kernels without io_uring allocate no pool. A bitmap and the buffers share one allocation. The get helper returns NULL when the pool is empty so the driver can return -EAGAIN without allocating or waiting for a buffer in the I/O path. Slots are shared by all openers of a device. One opener can retain all 64 slots, for example with unreaped CHECK CONDITION completions on a DEFER_TASKRUN ring, causing other openers to receive -EAGAIN too. bsg nodes are normally restricted to privileged users, and sharing avoids allocating a pool on every open. SG_SET_COMMAND_Q does not resize it. A checked-out buffer must be returned before completing or failing its command. io_uring retains the file reference through command completion, and the open cdev keeps the bsg device alive, so the pool cannot be freed while a buffer is checked out. Protect the bitmap with irqsave locking because buffers can be returned from end-I/O as well as process context. The next patch makes scsi_bsg use the pool to release completed requests before task work. No caller checks out buffers in this patch alone. Cc: stable@vger.kernel.org Co-developed-by: Yang Xiuwei Signed-off-by: Yang Xiuwei Signed-off-by: Yingjie Wang --- Changes in v5: - Replace the per-file pool with Yang's per-device pool: inline bitmap, one allocation, irqsave locking. Return quietly on invalid get/put arguments and trim comments. The subject changed accordingly. - Add Co-developed-by and retain Yang's Signed-off-by from his posted patch. This patch is a prerequisite for the following fix; it has no Fixes tag of its own. block/bsg.c | 109 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/bsg.h | 3 ++ 2 files changed, 112 insertions(+) diff --git a/block/bsg.c b/block/bsg.c index b296fa373292..73ec02ee8c23 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include +#include #include #define BSG_DESCRIPTION "Block layer SCSI generic (bsg) driver" @@ -30,6 +32,7 @@ struct bsg_device { unsigned int reserved_size; bsg_sg_io_fn *sg_io_fn; bsg_uring_cmd_fn *uring_cmd_fn; + struct bsg_completion_pool *completion_pool; }; static inline struct bsg_device *to_bsg_device(struct inode *inode) @@ -40,6 +43,101 @@ static inline struct bsg_device *to_bsg_device(struct inode *inode) #define BSG_DEFAULT_CMDS 64 #define BSG_MAX_DEVS (1 << MINORBITS) +/* + * Per-device completion buffers for uring_cmd. These can retain completion + * data after the request is freed. scsi_bsg is the only user, so each + * buffer holds SCSI_SENSE_BUFFERSIZE bytes. + */ +struct bsg_completion_pool { + spinlock_t lock; /* protects checked_out, including in end-I/O */ + DECLARE_BITMAP(checked_out, BSG_DEFAULT_CMDS); + u8 bufs[]; +}; + +static struct bsg_completion_pool *bsg_completion_pool_alloc(void) +{ + struct bsg_completion_pool *pool; + + pool = kzalloc(struct_size(pool, bufs, + BSG_DEFAULT_CMDS * SCSI_SENSE_BUFFERSIZE), + GFP_KERNEL); + if (!pool) + return NULL; + spin_lock_init(&pool->lock); + return pool; +} + +static void bsg_completion_pool_free(struct bsg_completion_pool *pool) +{ + if (!pool) + return; + WARN_ON_ONCE(!bitmap_empty(pool->checked_out, BSG_DEFAULT_CMDS)); + kfree(pool); +} + +/** + * bsg_uring_cmd_get_completion_buf - check out a per-device completion buffer + * @ioucmd: the uring command being issued + * @len: required buffer size + * + * The caller must return the buffer before completing or failing the command. + * + * Return: the buffer, or NULL if there is no pool, @len exceeds the buffer + * size, or the pool is exhausted. + */ +void *bsg_uring_cmd_get_completion_buf(struct io_uring_cmd *ioucmd, size_t len) +{ + struct bsg_device *bd = to_bsg_device(file_inode(ioucmd->file)); + struct bsg_completion_pool *pool = bd->completion_pool; + unsigned long flags; + unsigned int index; + void *buf = NULL; + + if (!pool || len > SCSI_SENSE_BUFFERSIZE) + return NULL; + + spin_lock_irqsave(&pool->lock, flags); + index = find_first_zero_bit(pool->checked_out, BSG_DEFAULT_CMDS); + if (index < BSG_DEFAULT_CMDS) { + __set_bit(index, pool->checked_out); + buf = pool->bufs + index * SCSI_SENSE_BUFFERSIZE; + } + spin_unlock_irqrestore(&pool->lock, flags); + return buf; +} +EXPORT_SYMBOL_GPL(bsg_uring_cmd_get_completion_buf); + +/** + * bsg_uring_cmd_put_completion_buf - return a per-device completion buffer + * @ioucmd: the uring command that checked out the buffer + * @buf: buffer previously returned by bsg_uring_cmd_get_completion_buf() + * + * Return each buffer exactly once, from submission error handling, + * end-I/O or task work. A second return while the buffer is free + * triggers a warning. + */ +void bsg_uring_cmd_put_completion_buf(struct io_uring_cmd *ioucmd, void *buf) +{ + struct bsg_device *bd = to_bsg_device(file_inode(ioucmd->file)); + struct bsg_completion_pool *pool = bd->completion_pool; + unsigned long flags; + unsigned long offset; + unsigned int index; + + if (!pool || !buf) + return; + offset = (unsigned long)buf - (unsigned long)pool->bufs; + if (offset >= BSG_DEFAULT_CMDS * SCSI_SENSE_BUFFERSIZE || + offset % SCSI_SENSE_BUFFERSIZE) + return; + index = offset / SCSI_SENSE_BUFFERSIZE; + + spin_lock_irqsave(&pool->lock, flags); + WARN_ON_ONCE(!__test_and_clear_bit(index, pool->checked_out)); + spin_unlock_irqrestore(&pool->lock, flags); +} +EXPORT_SYMBOL_GPL(bsg_uring_cmd_put_completion_buf); + static DEFINE_IDA(bsg_minor_ida); static const struct class bsg_class; static int bsg_major; @@ -203,6 +301,7 @@ static void bsg_device_release(struct device *dev) { struct bsg_device *bd = container_of(dev, struct bsg_device, device); + bsg_completion_pool_free(bd->completion_pool); ida_free(&bsg_minor_ida, MINOR(bd->device.devt)); kfree(bd); } @@ -234,10 +333,20 @@ struct bsg_device *bsg_register_queue(struct request_queue *q, bd->sg_io_fn = sg_io_fn; bd->uring_cmd_fn = uring_cmd_fn; + /* Transport bsg devices have no uring_cmd handler and need no pool. */ + if (IS_ENABLED(CONFIG_IO_URING) && uring_cmd_fn) { + bd->completion_pool = bsg_completion_pool_alloc(); + if (!bd->completion_pool) { + kfree(bd); + return ERR_PTR(-ENOMEM); + } + } + ret = ida_alloc_max(&bsg_minor_ida, BSG_MAX_DEVS - 1, GFP_KERNEL); if (ret < 0) { if (ret == -ENOSPC) dev_err(parent, "bsg: too many bsg devices\n"); + bsg_completion_pool_free(bd->completion_pool); kfree(bd); return ERR_PTR(ret); } diff --git a/include/linux/bsg.h b/include/linux/bsg.h index 162730bfc2d8..f5560be80c20 100644 --- a/include/linux/bsg.h +++ b/include/linux/bsg.h @@ -20,4 +20,7 @@ struct bsg_device *bsg_register_queue(struct request_queue *q, bsg_sg_io_fn *sg_io_fn, bsg_uring_cmd_fn *uring_cmd_fn); void bsg_unregister_queue(struct bsg_device *bcd); +void *bsg_uring_cmd_get_completion_buf(struct io_uring_cmd *ioucmd, size_t len); +void bsg_uring_cmd_put_completion_buf(struct io_uring_cmd *ioucmd, void *buf); + #endif /* _LINUX_BSG_H */ -- 2.43.0