Prepare for using blk_mq_all_tag_iter() in the SCSI core. Cc: Jens Axboe Cc: Christoph Hellwig Cc: Ming Lei Cc: John Garry Signed-off-by: Bart Van Assche --- block/blk-mq-tag.c | 1 + block/blk-mq.h | 2 -- include/linux/blk-mq.h | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index d880c50629d6..1d56ee8722c5 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -419,6 +419,7 @@ void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, { __blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS); } +EXPORT_SYMBOL(blk_mq_all_tag_iter); /** * blk_mq_tagset_busy_iter - iterate over all started requests in a tag set diff --git a/block/blk-mq.h b/block/blk-mq.h index affb2e14b56e..944668f34856 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -179,8 +179,6 @@ void blk_mq_tag_update_sched_shared_tags(struct request_queue *q); void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool); void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_tag_iter_fn *fn, void *priv); -void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, - void *priv); static inline struct sbq_wait_state *bt_wait_ptr(struct sbitmap_queue *bt, struct blk_mq_hw_ctx *hctx) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 2a5a828f19a0..8ed09783f289 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -921,6 +921,8 @@ void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); void blk_mq_run_hw_queues(struct request_queue *q, bool async); void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs); +void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, + void *priv); void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, busy_tag_iter_fn *fn, void *priv); void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset); Use scsi_device_busy() instead of open-coding it. This patch prepares for skipping the SCSI device budget map initialization in certain cases. Cc: Jens Axboe Cc: Christoph Hellwig Cc: Ming Lei Cc: John Garry Signed-off-by: Bart Van Assche --- drivers/ufs/core/ufshcd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index e2157128e3bf..e03e555cc148 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -1287,13 +1287,13 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, */ static u32 ufshcd_pending_cmds(struct ufs_hba *hba) { - const struct scsi_device *sdev; + struct scsi_device *sdev; unsigned long flags; u32 pending = 0; spin_lock_irqsave(hba->host->host_lock, flags); __shost_for_each_device(sdev, hba->host) - pending += sbitmap_weight(&sdev->budget_map); + pending += scsi_device_busy(sdev); spin_unlock_irqrestore(hba->host->host_lock, flags); return pending; The SCSI core uses the budget map to enforce the cmd_per_lun limit. That limit cannot be exceeded if host->cmd_per_lun >= host->can_queue and if the host tag set is shared across all hardware queues. Since scsi_mq_get_budget() shows up in all CPU profiles for fast SCSI devices, do not allocate a budget map if cmd_per_lun >= can_queue and if the host tag set is shared across all hardware queues. On my UFS 4 test setup this patch improves IOPS by 1% and reduces the time spent in scsi_mq_get_budget() from 0.22% to 0.01%. Cc: Jens Axboe Cc: Christoph Hellwig Cc: Ming Lei Cc: John Garry Signed-off-by: Bart Van Assche --- drivers/scsi/scsi.c | 7 ++++- drivers/scsi/scsi_lib.c | 60 +++++++++++++++++++++++++++++++++----- drivers/scsi/scsi_scan.c | 11 ++++++- include/scsi/scsi_device.h | 5 +--- 4 files changed, 70 insertions(+), 13 deletions(-) diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c index 9a0f467264b3..06066b694d8a 100644 --- a/drivers/scsi/scsi.c +++ b/drivers/scsi/scsi.c @@ -216,6 +216,8 @@ int scsi_device_max_queue_depth(struct scsi_device *sdev) */ int scsi_change_queue_depth(struct scsi_device *sdev, int depth) { + struct Scsi_Host *shost = sdev->host; + depth = min_t(int, depth, scsi_device_max_queue_depth(sdev)); if (depth > 0) { @@ -226,7 +228,10 @@ int scsi_change_queue_depth(struct scsi_device *sdev, int depth) if (sdev->request_queue) blk_set_queue_depth(sdev->request_queue, depth); - sbitmap_resize(&sdev->budget_map, sdev->queue_depth); + if (shost->host_tagset && depth >= shost->can_queue) + sbitmap_free(&sdev->budget_map); + else + sbitmap_resize(&sdev->budget_map, sdev->queue_depth); return sdev->queue_depth; } diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 0c65ecfedfbd..c546514d1049 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -396,7 +396,8 @@ void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd) if (starget->can_queue > 0) atomic_dec(&starget->target_busy); - sbitmap_put(&sdev->budget_map, cmd->budget_token); + if (sdev->budget_map.map) + sbitmap_put(&sdev->budget_map, cmd->budget_token); cmd->budget_token = -1; } @@ -445,6 +446,47 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev) spin_unlock_irqrestore(shost->host_lock, flags); } +struct sdev_in_flight_data { + const struct scsi_device *sdev; + int count; +}; + +static bool scsi_device_check_in_flight(struct request *rq, void *data) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct sdev_in_flight_data *sifd = data; + + if (cmd->device == sifd->sdev) + sifd->count++; + + return true; +} + +/** + * scsi_device_busy() - Number of commands allocated for a SCSI device + * @sdev: SCSI device. + * + * Note: There is a subtle difference between this function and + * scsi_host_busy(). scsi_host_busy() counts the number of commands that have + * been started. This function counts the number of commands that have been + * allocated. At least the UFS driver depends on this function counting commands + * that have already been allocated but that have not yet been started. + */ +int scsi_device_busy(const struct scsi_device *sdev) +{ + struct sdev_in_flight_data sifd = { .sdev = sdev }; + struct blk_mq_tag_set *set = &sdev->host->tag_set; + + if (sdev->budget_map.map) + return sbitmap_weight(&sdev->budget_map); + if (WARN_ON_ONCE(!set->shared_tags)) + return 0; + blk_mq_all_tag_iter(set->shared_tags, scsi_device_check_in_flight, + &sifd); + return sifd.count; +} +EXPORT_SYMBOL(scsi_device_busy); + static inline bool scsi_device_is_busy(struct scsi_device *sdev) { if (scsi_device_busy(sdev) >= sdev->queue_depth) @@ -1358,11 +1400,13 @@ scsi_device_state_check(struct scsi_device *sdev, struct request *req) static inline int scsi_dev_queue_ready(struct request_queue *q, struct scsi_device *sdev) { - int token; + int token = INT_MAX; - token = sbitmap_get(&sdev->budget_map); - if (token < 0) - return -1; + if (sdev->budget_map.map) { + token = sbitmap_get(&sdev->budget_map); + if (token < 0) + return -1; + } if (!atomic_read(&sdev->device_blocked)) return token; @@ -1373,7 +1417,8 @@ static inline int scsi_dev_queue_ready(struct request_queue *q, */ if (scsi_device_busy(sdev) > 1 || atomic_dec_return(&sdev->device_blocked) > 0) { - sbitmap_put(&sdev->budget_map, token); + if (sdev->budget_map.map) + sbitmap_put(&sdev->budget_map, token); return -1; } @@ -1749,7 +1794,8 @@ static void scsi_mq_put_budget(struct request_queue *q, int budget_token) { struct scsi_device *sdev = q->queuedata; - sbitmap_put(&sdev->budget_map, budget_token); + if (sdev->budget_map.map) + sbitmap_put(&sdev->budget_map, budget_token); } /* diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 3c6e089e80c3..6f2d0bf0e3ec 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -218,6 +218,7 @@ static void scsi_unlock_floptical(struct scsi_device *sdev, static int scsi_realloc_sdev_budget_map(struct scsi_device *sdev, unsigned int depth) { + struct Scsi_Host *shost = sdev->host; int new_shift = sbitmap_calculate_shift(depth); bool need_alloc = !sdev->budget_map.map; bool need_free = false; @@ -225,6 +226,13 @@ static int scsi_realloc_sdev_budget_map(struct scsi_device *sdev, int ret; struct sbitmap sb_backup; + if (shost->host_tagset && depth >= shost->can_queue) { + memflags = blk_mq_freeze_queue(sdev->request_queue); + sbitmap_free(&sb_backup); + blk_mq_unfreeze_queue(sdev->request_queue, memflags); + return 0; + } + depth = min_t(unsigned int, depth, scsi_device_max_queue_depth(sdev)); /* @@ -1112,7 +1120,8 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result, scsi_cdl_check(sdev); sdev->max_queue_depth = sdev->queue_depth; - WARN_ON_ONCE(sdev->max_queue_depth > sdev->budget_map.depth); + WARN_ON_ONCE(sdev->budget_map.map && + sdev->max_queue_depth > sdev->budget_map.depth); sdev->sdev_bflags = *bflags; /* diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 6d6500148c4b..3c7a95fa9b67 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -687,10 +687,7 @@ static inline int scsi_device_supports_vpd(struct scsi_device *sdev) return 0; } -static inline int scsi_device_busy(struct scsi_device *sdev) -{ - return sbitmap_weight(&sdev->budget_map); -} +int scsi_device_busy(const struct scsi_device *sdev); /* Macros to access the UNIT ATTENTION counters */ #define scsi_get_ua_new_media_ctr(sdev) \