Summarize if any of the inv entries will use RIL and if any need ARM_SMMU_OPT_FULL_CONT_RIL. The next patch will use this to avoid RIL pre-calculations unless RIL is being used by the invalidation. Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Nicolin Chen Signed-off-by: Jason Gunthorpe --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 38 +++++++++++-------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 19 ++++++++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 4 ++ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index add671363c828c..6f8917ff33a02b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -24,7 +24,9 @@ struct arm_smmu_test_writer { static struct arm_smmu_ste bypass_ste; static struct arm_smmu_ste abort_ste; static struct arm_smmu_device smmu = { - .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR + .features = ARM_SMMU_FEAT_STALLS | ARM_SMMU_FEAT_ATTR_TYPES_OVR | + ARM_SMMU_FEAT_RANGE_INV, + .options = ARM_SMMU_OPT_FULL_CONT_RIL, }; static struct mm_struct sva_mm = { .pgd = (void *)0xdaedbeefdeadbeefULL, @@ -645,6 +647,8 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, { KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs); KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); + KUNIT_EXPECT_TRUE(test, invs->has_range_inv); + KUNIT_EXPECT_TRUE(test, invs->has_full_cont_ril); while (num_invs--) { KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users), @@ -655,37 +659,37 @@ static void arm_smmu_v3_invs_test_verify(struct kunit *test, static struct arm_smmu_invs invs1 = { .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, - { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, - { .type = INV_TYPE_ATS, .id = 3, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, + { .smmu = &smmu, .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 3, }, }, }; static struct arm_smmu_invs invs2 = { .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ - { .type = INV_TYPE_ATS, .id = 4, }, - { .type = INV_TYPE_ATS, .id = 5, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 4, }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, }, }; static struct arm_smmu_invs invs3 = { .num_invs = 3, - .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ - { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ - { .type = INV_TYPE_ATS, .id = 6, }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 6, }, }, }; static struct arm_smmu_invs invs4 = { .num_invs = 3, - .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, - { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, - { .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, }; static struct arm_smmu_invs invs5 = { .num_invs = 3, - .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, - { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ - { .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, + .inv = { { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ + { .smmu = &smmu, .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, }; static void arm_smmu_v3_invs_test(struct kunit *test) @@ -705,6 +709,8 @@ static void arm_smmu_v3_invs_test(struct kunit *test) /* New array */ test_a = arm_smmu_invs_alloc(0); KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + KUNIT_EXPECT_FALSE(test, test_a->has_range_inv); + KUNIT_EXPECT_FALSE(test, test_a->has_full_cont_ril); /* Test1: merge invs1 (new array) */ test_b = arm_smmu_invs_merge(test_a, &invs1); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4db8ad6f775a55..670ffa87c076ef 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1053,6 +1053,19 @@ static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l, return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]); } +static void arm_smmu_invs_update_caps(struct arm_smmu_invs *invs, + const struct arm_smmu_inv *inv) +{ + if (arm_smmu_inv_is_ats(inv)) + invs->has_ats = true; + + if (inv->smmu->features & ARM_SMMU_FEAT_RANGE_INV) { + invs->has_range_inv = true; + if (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RIL) + invs->has_full_cont_ril = true; + } +} + /** * arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for * arm_smmu_invs_merge() or arm_smmu_invs_unref() @@ -1123,8 +1136,7 @@ struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, */ if (new != new_invs->inv) WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1); - if (arm_smmu_inv_is_ats(new)) - new_invs->has_ats = true; + arm_smmu_invs_update_caps(new_invs, new); new++; } @@ -1234,8 +1246,7 @@ struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) arm_smmu_invs_for_each_entry(invs, i, inv) { new_invs->inv[num_invs] = *inv; - if (arm_smmu_inv_is_ats(inv)) - new_invs->has_ats = true; + arm_smmu_invs_update_caps(new_invs, inv); num_invs++; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 054ca19dde0502..d38cc176c2ce6d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -758,6 +758,8 @@ static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) * Must not be greater than @num_invs * @rwlock: optional rwlock to fence ATS operations * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL + * @has_range_inv: flag if any entry's SMMU supports range invalidation + * @has_full_cont_ril: flag if any entry's SMMU requires the CONT RIL workaround * @rcu: rcu head for kfree_rcu() * @inv: flexible invalidation array * @@ -787,6 +789,8 @@ struct arm_smmu_invs { size_t num_trashes; rwlock_t rwlock; bool has_ats; + bool has_range_inv; + bool has_full_cont_ril; struct rcu_head rcu; struct arm_smmu_inv inv[] __counted_by(max_invs); }; -- 2.43.0