From: Keith Busch Private slices of the sbitmap can strand an under utilized queue's tags. Allow a driver to request some percentage of tags to be reserved into a common shared pool that anyone may allocate from. The default 0 means the entire tag set is divided up among its users; 100 means no one has an exclusive window and can therefore allocate from the entire tag space. Shared tags being outside the private zone makes it so we can't do a sbitmap range weight to find its active commands, so the debugfs count may no longer accurate. Count the hctx's actual tags via __blk_mq_hctx_tag_busy_iter() to provide accuracy over performance for debugfs. Signed-off-by: Keith Busch --- block/blk-mq-tag.c | 94 +++++++++++++++++++++++++++++++----------- block/blk-mq.c | 3 ++ include/linux/blk-mq.h | 5 +++ 3 files changed, 79 insertions(+), 23 deletions(-) diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 58cf480df9c69..aa7dbceb60dcd 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -144,54 +144,79 @@ static inline bool blk_mq_tag_is_windowed(struct blk_mq_alloc_data *data) (data->hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED); } +struct blk_mq_tag_win { + unsigned int priv_min; + unsigned int priv_max; + unsigned int shared_min; + unsigned int shared_max; +}; + static bool blk_mq_tag_active_window(struct blk_mq_hw_ctx *hctx, - unsigned int depth, unsigned int *min, - unsigned int *max) + unsigned int depth, + struct blk_mq_tag_win *w) { unsigned int users = READ_ONCE(hctx->tags->active_queues); int slot = READ_ONCE(*blk_mq_tag_win_slot(hctx)); - unsigned int p, rem, pos; + unsigned int fair, priv, pct, pos; + + if (users <= 1 || depth <= 1) + return false; - if (users <= 1 || slot < 0 || depth <= 1) + fair = depth / users; + if (!fair) return false; - p = depth / users; - if (!p) + pct = 100 - min(hctx->queue->tag_set->shared_pct, 100u); + priv = fair * pct / 100; + if (!priv) return false; /* * Slots are allocated from a bitmap and freed out of order, so a slot * index can exceed the active-user count once lower slots are freed. * Convert it to a position based on how many active slots precede it. + * If a slot could not be assigned, the window is confined to the + * shared range. */ - pos = bitmap_weight(hctx->tags->active_slots, slot); - rem = depth - p * users; - *min = pos * p; - *max = pos * p + p + rem; - if (*max > depth) - *max = depth; + pos = slot >= 0 ? bitmap_weight(hctx->tags->active_slots, slot) : users; + if (pos < users) { + w->priv_min = pos * priv; + w->priv_max = w->priv_min + priv; + } else { + w->priv_min = w->priv_max = 0; + } + w->shared_min = min(users * priv, depth); + w->shared_max = depth; + + if (w->priv_min == w->priv_max && w->shared_min == w->shared_max) + return false; return true; } -int blk_mq_get_tag_window(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt) +static int __blk_mq_get_tag_window(struct sbitmap_queue *bt, + struct blk_mq_tag_win *w) { - unsigned int min, max; + if (w->priv_max > w->priv_min) { + int tag = sbitmap_queue_get_range(bt, w->priv_min, w->priv_max); - if (blk_mq_tag_active_window(hctx, bt->sb.depth, &min, &max)) - return sbitmap_queue_get_range(bt, min, max); - return __sbitmap_queue_get(bt); + if (tag >= 0) + return tag; + } + + if (w->shared_max > w->shared_min) + return sbitmap_queue_get_range(bt, w->shared_min, w->shared_max); + return BLK_MQ_NO_TAG; } -unsigned int blk_mq_hctx_active(struct blk_mq_hw_ctx *hctx) +int blk_mq_get_tag_window(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt) { - struct sbitmap *sb = &hctx->tags->bitmap_tags.sb; - unsigned int min, max; + struct blk_mq_tag_win w; - if (!blk_mq_tag_active_window(hctx, sb->depth, &min, &max)) - return sbitmap_weight(sb); - return sbitmap_weight_range(sb, min, max - min); + if (blk_mq_tag_active_window(hctx, bt->sb.depth, &w)) + return __blk_mq_get_tag_window(bt, &w); + return __sbitmap_queue_get(bt); } static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, @@ -604,6 +629,29 @@ static void __blk_mq_hctx_tag_busy_iter(struct blk_mq_hw_ctx *hctx, bt_for_each(hctx, hctx->queue, &tags->bitmap_tags, fn, priv, false); } +static bool blk_mq_count_active(struct request *rq, void *priv) +{ + (*(unsigned int *)priv)++; + return true; +} + +unsigned int blk_mq_hctx_active(struct blk_mq_hw_ctx *hctx) +{ + struct request_queue *q = hctx->queue; + unsigned int count = 0; + int srcu_idx; + + if (!percpu_ref_tryget(&q->q_usage_counter)) + return 0; + + srcu_idx = srcu_read_lock(&q->tag_set->tags_srcu); + __blk_mq_hctx_tag_busy_iter(hctx, blk_mq_count_active, &count); + srcu_read_unlock(&q->tag_set->tags_srcu, srcu_idx); + + blk_queue_exit(q); + return count; +} + /** * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag * @q: Request queue to examine. diff --git a/block/blk-mq.c b/block/blk-mq.c index 56ab6ac5ec696..cc56baee74fe8 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4845,6 +4845,9 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) set->queue_depth = BLK_MQ_MAX_DEPTH; } + if (set->shared_pct > 100) + set->shared_pct = 100; + if (!set->nr_maps) set->nr_maps = 1; else if (set->nr_maps > HCTX_MAX_TYPES) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 22cc09d5ef320..59847ac7d319c 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -515,6 +515,10 @@ enum hctx_type { * @numa_node: NUMA node the storage adapter has been connected to. * @timeout: Request processing timeout in jiffies. * @flags: Zero or more BLK_MQ_F_* flags. + * @shared_pct: Percent of tags dedicated to non-exclusive use. 0 means + * the entire tagset is divided among the users for exclusive + * per-user access, and 100 means the entire tag space is + * available to all queue contexts that share it. * @driver_data: Pointer to data owned by the block driver that created this * tag set. * @tags: Tag sets. One tag set per hardware queue. Has @nr_hw_queues @@ -544,6 +548,7 @@ struct blk_mq_tag_set { int numa_node; unsigned int timeout; unsigned int flags; + unsigned int shared_pct; void *driver_data; struct blk_mq_tags **tags; -- 2.52.0