The server of a target_core_uring device fetches commands and commits their results. It has no access to the data of reads and writes. Let the server pwrite() the data of a read command and pread() the data of a write command on the character device, at a position that encodes the queue, the tag and the offset in the command. Copy straight between the pages of the command and the buffer of the server, so that the data is copied once. Hold a reference on the command during the copy, so that it does not complete underneath. Exporting a 2 GiB device through tcm_loop to fio (io_uring, direct I/O, iodepth 32 for 4k and 8 for 1M, 6 seconds per job) on a 32 CPU VM, with TCM/USER and a single-threaded handler, then with this backstore, 32 queues and one server thread per queue. Both servers keep the data in memory. A single job is slower; the gain comes with concurrency and with large I/O. Three alternating runs (median, min-max in parentheses): workload TCM/USER (MB/s) uring (MB/s) change 4k randread, 1 job 426 (409-463) 354 (348-378) -17% 4k randread, 4 jobs 1165 (1072-1253) 1466 (1451-1518) +26% 4k randread, 16 jobs 920 (916-1162) 6500 (6173-6769) +607% 4k randwrite, 16 jobs 793 (711-886) 6070 (5966-6216) +665% 1M read, 4 jobs 6019 (5857-6047) 47687 (32175-49464) +692% 1M write, 4 jobs 2791 (2649-2907) 28295 (27152-29050) +914% Signed-off-by: Qiliang Yuan --- drivers/target/target_core_uring.c | 89 +++++++++++++++++++++++++++++++++- include/uapi/linux/target_core_uring.h | 15 ++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/drivers/target/target_core_uring.c b/drivers/target/target_core_uring.c index 770bb665fbe00..3a6f8d25d28e5 100644 --- a/drivers/target/target_core_uring.c +++ b/drivers/target/target_core_uring.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -43,7 +44,8 @@ struct tcmur_dev; * @sgl: Data pages of the command * @sgl_nents: Number of entries in @sgl * @desc: What the server is asked to do - * @ref: Held by the tag or the pending list + * @ref: Held by the tag or the pending list and by data copies in + * progress * @status: SAM status to complete @se_cmd with */ struct tcmur_cmd { @@ -232,6 +234,26 @@ static sense_reason_t tcmur_execute_rw(struct se_cmd *se_cmd, se_cmd->data_length, sgl, sgl_nents); } +/* Take a reference on the command the server works on in (@q_id, @tag) */ +static struct tcmur_cmd *tcmur_get_cmd(struct tcmur_dev *udev, u16 q_id, + u16 tag) +{ + struct tcmur_cmd *tcmd; + struct tcmur_queue *q; + + if (q_id >= udev->nr_queues || tag >= udev->queue_depth) + return NULL; + + q = &udev->queues[q_id]; + spin_lock(&q->lock); + tcmd = q->ios[tag].cmd; + if (tcmd) + refcount_inc(&tcmd->ref); + spin_unlock(&q->lock); + + return tcmd; +} + static void tcmur_cancel_fetch(struct io_uring_cmd *ucmd, unsigned int issue_flags) { @@ -363,6 +385,69 @@ static int tcmur_ch_uring_cmd(struct io_uring_cmd *ucmd, return tcmur_ch_uring_cmd_local(ucmd, issue_flags); } +/* Copy between the data pages of a command and the buffer of the server */ +static ssize_t tcmur_ch_copy(struct kiocb *iocb, struct iov_iter *iter, + bool to_cmd) +{ + struct tcmur_dev *udev = iocb->ki_filp->private_data; + u32 off = iocb->ki_pos & TCMUR_BUF_OFF_MASK; + u16 tag = iocb->ki_pos >> TCMUR_BUF_TAG_SHIFT; + u16 q_id = iocb->ki_pos >> TCMUR_BUF_QID_SHIFT; + size_t count, n, copied; + struct tcmur_cmd *tcmd; + struct scatterlist *sg; + ssize_t done = 0; + int i; + + tcmd = tcmur_get_cmd(udev, q_id, tag); + if (!tcmd) + return -EINVAL; + + if (to_cmd != (tcmd->desc.op == TCMUR_OP_READ) || + off > tcmd->desc.data_len) { + done = -EINVAL; + goto put_cmd; + } + + count = min(iov_iter_count(iter), tcmd->desc.data_len - off); + for_each_sg(tcmd->sgl, sg, tcmd->sgl_nents, i) { + if (!count) + break; + if (off >= sg->length) { + off -= sg->length; + continue; + } + + n = min(sg->length - off, count); + if (to_cmd) + copied = copy_page_from_iter(sg_page(sg), + sg->offset + off, n, iter); + else + copied = copy_page_to_iter(sg_page(sg), + sg->offset + off, n, iter); + done += copied; + count -= copied; + if (copied != n) + break; + off = 0; + } + if (!done && count) + done = -EFAULT; +put_cmd: + tcmur_cmd_put(tcmd); + return done; +} + +static ssize_t tcmur_ch_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + return tcmur_ch_copy(iocb, to, false); +} + +static ssize_t tcmur_ch_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + return tcmur_ch_copy(iocb, from, true); +} + static size_t tcmur_desc_area_size(struct tcmur_dev *udev) { return PAGE_ALIGN(udev->queue_depth * sizeof(struct tcmur_io_desc)); @@ -433,6 +518,8 @@ static const struct file_operations tcmur_ch_fops = { .owner = THIS_MODULE, .open = tcmur_ch_open, .release = tcmur_ch_release, + .read_iter = tcmur_ch_read_iter, + .write_iter = tcmur_ch_write_iter, .uring_cmd = tcmur_ch_uring_cmd, .mmap = tcmur_ch_mmap, .unlocked_ioctl = tcmur_ch_ioctl, diff --git a/include/uapi/linux/target_core_uring.h b/include/uapi/linux/target_core_uring.h index a662a23fca9ab..8b95be853753c 100644 --- a/include/uapi/linux/target_core_uring.h +++ b/include/uapi/linux/target_core_uring.h @@ -96,4 +96,19 @@ struct tcmur_io_desc { #define TCMUR_DESC_AREA_SIZE \ (TCMUR_MAX_QUEUE_DEPTH * sizeof(struct tcmur_io_desc)) +/* + * The data of a command is read (TCMUR_OP_WRITE) or written + * (TCMUR_OP_READ) with pread()/pwrite() on the file, at the position + * returned by tcmur_buf_pos(). + */ +#define TCMUR_BUF_TAG_SHIFT 32 +#define TCMUR_BUF_QID_SHIFT 48 +#define TCMUR_BUF_OFF_MASK ((1ULL << TCMUR_BUF_TAG_SHIFT) - 1) + +static inline __u64 tcmur_buf_pos(__u16 q_id, __u16 tag, __u32 off) +{ + return ((__u64)q_id << TCMUR_BUF_QID_SHIFT) | + ((__u64)tag << TCMUR_BUF_TAG_SHIFT) | off; +} + #endif /* _UAPI_LINUX_TARGET_CORE_URING_H */ -- 2.43.0