target_core_uring fails every read and write, as nothing can serve them. A command can arrive before the server is ready for it: at startup, while all tags of a queue are busy, or while the server restarts. Allocate a struct tcmur_cmd for each read and write, fill its descriptor and put it on the pending list of the queue of the submitting CPU. The commands wait there until a later patch lets a server fetch them. Signed-off-by: Qiliang Yuan --- drivers/target/target_core_uring.c | 81 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/drivers/target/target_core_uring.c b/drivers/target/target_core_uring.c index 904ce2a613618..90e443ecb944c 100644 --- a/drivers/target/target_core_uring.c +++ b/drivers/target/target_core_uring.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -29,12 +30,32 @@ struct tcmur_dev; +/** + * struct tcmur_cmd - One command for the server + * @node: Entry in tcmur_queue.pending or in a local list + * @se_cmd: Target command this belongs to + * @sgl: Data pages of the command + * @sgl_nents: Number of entries in @sgl + * @desc: What the server is asked to do + */ +struct tcmur_cmd { + struct list_head node; + struct se_cmd *se_cmd; + struct scatterlist *sgl; + u32 sgl_nents; + struct tcmur_io_desc desc; +}; + /** * struct tcmur_queue - One command queue + * @lock: Protect the list + * @pending: Commands waiting for a tag * @descs: Command descriptors, indexed by tag and mapped by the server * @udev: Device the queue belongs to */ struct tcmur_queue { + spinlock_t lock; + struct list_head pending; struct tcmur_io_desc *descs; struct tcmur_dev *udev; } ____cacheline_aligned_in_smp; @@ -54,6 +75,7 @@ struct tcmur_dev { struct tcmur_queue *queues; }; +static struct kmem_cache *tcmur_cmd_cache; static DEFINE_IDA(tcmur_ida); static dev_t tcmur_devt; @@ -66,12 +88,56 @@ static struct tcmur_dev *TCMUR_DEV(struct se_device *dev) return container_of(dev, struct tcmur_dev, se_dev); } +static struct tcmur_queue *tcmur_cmd_queue(struct tcmur_dev *udev, + struct se_cmd *se_cmd) +{ + return &udev->queues[(unsigned int)se_cmd->cpuid % udev->nr_queues]; +} + +static void tcmur_queue_cmd(struct tcmur_dev *udev, struct tcmur_cmd *tcmd) +{ + struct tcmur_queue *q = tcmur_cmd_queue(udev, tcmd->se_cmd); + + spin_lock(&q->lock); + list_add_tail(&tcmd->node, &q->pending); + spin_unlock(&q->lock); +} + +static sense_reason_t tcmur_submit(struct se_cmd *se_cmd, u32 op, u64 offset, + u64 len, struct scatterlist *sgl, + u32 sgl_nents) +{ + struct tcmur_dev *udev = TCMUR_DEV(se_cmd->se_dev); + struct tcmur_cmd *tcmd; + + tcmd = kmem_cache_zalloc(tcmur_cmd_cache, GFP_NOIO); + if (!tcmd) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + + tcmd->se_cmd = se_cmd; + tcmd->sgl = sgl; + tcmd->sgl_nents = sgl_nents; + tcmd->desc.op = op; + tcmd->desc.offset = offset; + tcmd->desc.len = len; + tcmd->desc.data_len = se_cmd->data_length; + if (op == TCMUR_OP_WRITE && (se_cmd->se_cmd_flags & SCF_FUA)) + tcmd->desc.flags |= TCMUR_IO_F_FUA; + + tcmur_queue_cmd(udev, tcmd); + return 0; +} + static sense_reason_t tcmur_execute_rw(struct se_cmd *se_cmd, struct scatterlist *sgl, u32 sgl_nents, enum dma_data_direction dir) { - /* Fail the commands until a server can pick them up */ - return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + u32 block_size = se_cmd->se_dev->dev_attrib.block_size; + + return tcmur_submit(se_cmd, dir == DMA_FROM_DEVICE ? TCMUR_OP_READ : + TCMUR_OP_WRITE, + se_cmd->t_task_lba * block_size, + se_cmd->data_length, sgl, sgl_nents); } static size_t tcmur_desc_area_size(struct tcmur_dev *udev) @@ -175,6 +241,8 @@ static int tcmur_alloc_queues(struct tcmur_dev *udev) for (i = 0; i < udev->nr_queues; i++) { q = &udev->queues[i]; + spin_lock_init(&q->lock); + INIT_LIST_HEAD(&q->pending); q->udev = udev; q->descs = vmalloc_user(tcmur_desc_area_size(udev)); if (!q->descs) { @@ -414,9 +482,13 @@ static int __init tcmur_module_init(void) { int ret; + tcmur_cmd_cache = KMEM_CACHE(tcmur_cmd, 0); + if (!tcmur_cmd_cache) + return -ENOMEM; + ret = alloc_chrdev_region(&tcmur_devt, 0, TCMUR_MINORS, "tcm-uring"); if (ret) - return ret; + goto destroy_cache; ret = class_register(&tcmur_class); if (ret) @@ -432,6 +504,8 @@ static int __init tcmur_module_init(void) class_unregister(&tcmur_class); unregister_chrdev: unregister_chrdev_region(tcmur_devt, TCMUR_MINORS); +destroy_cache: + kmem_cache_destroy(tcmur_cmd_cache); return ret; } @@ -442,6 +516,7 @@ static void __exit tcmur_module_exit(void) unregister_chrdev_region(tcmur_devt, TCMUR_MINORS); /* Wait for the devices that tcmur_cdev_release() frees with RCU */ rcu_barrier(); + kmem_cache_destroy(tcmur_cmd_cache); } module_init(tcmur_module_init); -- 2.43.0