If DS is supported then SCALE can go up to 39. Compute a scale max that is compatible for the entire invs list. Replace the has_range_inv in the invs list with a 0 range_inv_scale_max to mean range invalidation is unavailable. Reviewed-by: Nicolin Chen Tested-by: Nicolin Chen Signed-off-by: Jason Gunthorpe --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 4 ++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 18 +++++++++++++----- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 +++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index 81ceaf97b88c07..6f324ba0730bea 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -647,7 +647,7 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, { KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs); KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); - KUNIT_EXPECT_TRUE(test, invs->has_range_inv); + KUNIT_EXPECT_EQ(test, invs->range_inv_scale_max, 31); KUNIT_EXPECT_TRUE(test, invs->has_full_cont_range_inv); while (num_invs--) { KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); @@ -709,7 +709,7 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* New array */ test_a = arm_smmu_invs_alloc(0); KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); - KUNIT_EXPECT_FALSE(test, test_a->has_range_inv); + KUNIT_EXPECT_EQ(test, test_a->range_inv_scale_max, 0); KUNIT_EXPECT_FALSE(test, test_a->has_full_cont_range_inv); /* Test1: merge invs1 (new array) */ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index de9017eb93458d..3c5d9f875bfee1 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1060,9 +1060,15 @@ static void arm_smmu_invs_update_caps(struct arm_smmu_invs *invs, invs->has_ats = true; if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - invs->has_range_inv = true; + unsigned int scale_max; + if (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RANGE_INV) invs->has_full_cont_range_inv = true; + + scale_max = (inv->smmu->features & ARM_SMMU_FEAT_DS) ? 39 : 31; + if (!invs->range_inv_scale_max || + scale_max < invs->range_inv_scale_max) + invs->range_inv_scale_max = scale_max; } } @@ -2638,7 +2644,8 @@ static unsigned int arm_smmu_compute_ttl(u8 leaf_bitmap, u8 table_bitmap, * CONT is covered by one command. */ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi, - bool single_range_inv) + bool single_range_inv, + unsigned int scale_max) { u8 tgsz_lg2 = tlbi->tgsz_lg2; unsigned int ttl = arm_smmu_compute_ttl( @@ -2660,7 +2667,7 @@ static void arm_smmu_tlbi_calc_range(struct arm_smmu_tlbi *tlbi, * address beyond alignment to tg (so long as TTL=0). */ first.scale = arm_smmu_range_inv_calc_scale(num_tg); - if (first.scale > 31) { + if (first.scale > scale_max) { /* Range too large for a single command do full invalidation */ tlbi->range.use_full_inv = true; return; @@ -2925,12 +2932,13 @@ void arm_smmu_domain_tlbi(struct arm_smmu_tlbi *tlbi, * The invs generation ensures this matches the instances being * invalidated. */ - if (invs->has_range_inv) { + if (invs->range_inv_scale_max) { if (!tlbi->range.use_full_inv) { arm_smmu_tlbi_calc_range( tlbi, smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && - invs->has_full_cont_range_inv); + invs->has_full_cont_range_inv, + invs->range_inv_scale_max); } } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 245db175386a9d..97ab8257925de7 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -758,9 +758,10 @@ static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) * Must not be greater than @num_invs * @rwlock: optional rwlock to fence ATS operations * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL - * @has_range_inv: flag if any entry's SMMU supports range invalidation * @has_full_cont_range_inv: flag if any entry's SMMU requires the CONT range * invalidation workaround + * @range_inv_scale_max: max SCALE usable by all range-capable SMMUs, or 0 if + * no SMMU supports range invalidation * @rcu: rcu head for kfree_rcu() * @inv: flexible invalidation array * @@ -790,8 +791,8 @@ struct arm_smmu_invs { size_t num_trashes; rwlock_t rwlock; bool has_ats; - bool has_range_inv; bool has_full_cont_range_inv; + u8 range_inv_scale_max; struct rcu_head rcu; struct arm_smmu_inv inv[] __counted_by(max_invs); }; -- 2.43.0