These parameters go to a lot of different functions and the next patches will add more. Put them into a struct to keep things tidy. Reviewed-by: Nicolin Chen Reviewed-by: Mostafa Saleh Tested-by: Nicolin Chen Signed-off-by: Jason Gunthorpe --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 90 ++++++++++----------- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 9 +++ 2 files changed, 54 insertions(+), 45 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d6896e25b6632a..03723bb00de412 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -2363,8 +2363,8 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev) return IRQ_WAKE_THREAD; } -static struct arm_smmu_cmd -arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) +static struct arm_smmu_cmd arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, + struct arm_smmu_tlbi *tlbi) { size_t log2_span; size_t span_mask; @@ -2386,8 +2386,8 @@ arm_smmu_atc_inv_to_cmd(u32 sid, int ssid, unsigned long iova, size_t size) * This has the unpleasant side-effect of invalidating all PASID-tagged * ATC entries within the address range. */ - page_start = iova >> inval_grain_shift; - page_end = (iova + size - 1) >> inval_grain_shift; + page_start = tlbi->iova >> inval_grain_shift; + page_end = (tlbi->iova + tlbi->size - 1) >> inval_grain_shift; /* * In an ATS Invalidate Request, the address must be aligned on the @@ -2462,20 +2462,23 @@ static void arm_smmu_tlb_inv_context(void *cookie) static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, - struct arm_smmu_cmd *cmd, bool leaf, - unsigned long iova, size_t size, - size_t granule, size_t pgsize) + struct arm_smmu_cmd *cmd, + struct arm_smmu_tlbi *tlbi, + size_t pgsize) { - unsigned long end = iova + size, num_pages = 0, tg = pgsize; + size_t inv_range = tlbi->iopte_size; + unsigned long iova = tlbi->iova; + unsigned long end = iova + tlbi->size; + unsigned long num_pages = 0; + unsigned int tg = pgsize; u64 orig_data0 = cmd->data[0]; - size_t inv_range = granule; u8 ttl = 0, tg_enc = 0; - if (WARN_ON_ONCE(!size)) + if (WARN_ON_ONCE(!tlbi->size)) return; if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { - num_pages = size >> tg; + num_pages = tlbi->size >> tg; /* Convert page size of 12,14,16 (log2) to 1,2,3 */ tg_enc = (tg - 10) / 2; @@ -2488,8 +2491,8 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, * want to use a range command, so avoid the SVA corner case * where both scale and num could be 0 as well. */ - if (leaf) - ttl = 4 - ((ilog2(granule) - 3) / (tg - 3)); + if (tlbi->leaf_only) + ttl = 4 - ((ilog2(tlbi->iopte_size) - 3) / (tg - 3)); else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1) num_pages++; } @@ -2528,7 +2531,7 @@ static void arm_smmu_cmdq_batch_add_range(struct arm_smmu_device *smmu, * command and something would be very broken if iova had them * set. */ - cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, leaf) | + cmd->data[1] = FIELD_PREP(CMDQ_TLBI_1_LEAF, tlbi->leaf_only) | FIELD_PREP(CMDQ_TLBI_1_TTL, ttl) | FIELD_PREP(CMDQ_TLBI_1_TG, tg_enc) | (iova & ~GENMASK_U64(11, 0)); @@ -2568,13 +2571,13 @@ static bool arm_smmu_cmdq_batch_add_ril(struct arm_smmu_device *smmu, return true; } -static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, - size_t granule) +static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, + struct arm_smmu_tlbi *tlbi) { size_t max_tlbi_ops; /* 0 size means invalidate all */ - if (!size || size == SIZE_MAX) + if (!tlbi->size || tlbi->size == SIZE_MAX) return true; if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) @@ -2587,32 +2590,31 @@ static bool arm_smmu_inv_size_too_big(struct arm_smmu_device *smmu, size_t size, * invalidation feature, where there can be too many per-granule TLBIs, * resulting in a soft lockup. */ - max_tlbi_ops = 1 << (ilog2(granule) - 3); - return size >= max_tlbi_ops * granule; + max_tlbi_ops = 1 << (ilog2(tlbi->iopte_size) - 3); + return tlbi->size >= max_tlbi_ops * tlbi->iopte_size; } /* Used by non INV_TYPE_ATS* invalidations */ static void arm_smmu_inv_to_cmdq_batch(struct arm_smmu_inv *inv, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmd *cmd, - bool single_ril, bool leaf, - unsigned long iova, size_t size, - unsigned int granule) + struct arm_smmu_tlbi *tlbi) { struct arm_smmu_cmd nsize_cmd; - if (arm_smmu_inv_size_too_big(inv->smmu, size, granule)) + if (arm_smmu_inv_size_too_big(inv->smmu, tlbi)) goto full_inv; - if (single_ril && size > granule) { - if (!arm_smmu_cmdq_batch_add_ril(inv->smmu, cmds, cmd, iova, - size, inv->pgsize)) + if (tlbi->has_cont && tlbi->size > tlbi->iopte_size && + (inv->smmu->options & ARM_SMMU_OPT_FULL_CONT_RIL)) { + if (!arm_smmu_cmdq_batch_add_ril(inv->smmu, cmds, cmd, + tlbi->iova, tlbi->size, + inv->pgsize)) goto full_inv; return; } - arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, leaf, iova, size, - granule, inv->pgsize); + arm_smmu_cmdq_batch_add_range(inv->smmu, cmds, cmd, tlbi, inv->pgsize); return; full_inv: @@ -2637,10 +2639,8 @@ static inline bool arm_smmu_invs_end_batch(struct arm_smmu_inv *cur, return false; } -static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, - unsigned long iova, size_t size, - unsigned int granule, bool single_ril, - bool leaf) +static void __arm_smmu_domain_inv_range(struct arm_smmu_tlbi *tlbi, + struct arm_smmu_invs *invs) { struct arm_smmu_cmdq_batch cmds = {}; struct arm_smmu_inv *cur; @@ -2670,18 +2670,16 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, case INV_TYPE_S1_ASID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, cur->id, 0); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, single_ril, - leaf, iova, size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); break; case INV_TYPE_S2_VMID: cmd = arm_smmu_make_cmd_tlbi(cur->size_opcode, 0, cur->id); - arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, single_ril, - leaf, iova, size, granule); + arm_smmu_inv_to_cmdq_batch(cur, &cmds, &cmd, tlbi); break; case INV_TYPE_S2_VMID_S1_CLEAR: /* CMDQ_OP_TLBI_S12_VMALL already flushed S1 entries */ - if (arm_smmu_inv_size_too_big(cur->smmu, size, granule)) + if (arm_smmu_inv_size_too_big(cur->smmu, tlbi)) break; arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, @@ -2692,7 +2690,7 @@ static void __arm_smmu_domain_inv_range(struct arm_smmu_invs *invs, arm_smmu_cmdq_batch_add_cmd( smmu, &cmds, arm_smmu_atc_inv_to_cmd(cur->id, cur->ssid, - iova, size)); + tlbi)); break; case INV_TYPE_ATS_FULL: arm_smmu_cmdq_batch_add_cmd( @@ -2723,10 +2721,14 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long iova, size_t size, unsigned int granule, bool leaf) { + struct arm_smmu_tlbi tlbi = { + .iova = iova, + .size = size, + .iopte_size = granule, + .has_cont = smmu_domain->stage == ARM_SMMU_DOMAIN_SVA, + .leaf_only = leaf, + }; struct arm_smmu_invs *invs; - bool single_ril = - smmu_domain->stage == ARM_SMMU_DOMAIN_SVA && - (smmu_domain->smmu->options & ARM_SMMU_OPT_FULL_CONT_RIL); /* * An invalidation request must follow some IOPTE change and then load @@ -2766,12 +2768,10 @@ void arm_smmu_domain_inv_range(struct arm_smmu_domain *smmu_domain, unsigned long flags; read_lock_irqsave(&invs->rwlock, flags); - __arm_smmu_domain_inv_range(invs, iova, size, granule, - single_ril, leaf); + __arm_smmu_domain_inv_range(&tlbi, invs); read_unlock_irqrestore(&invs->rwlock, flags); } else { - __arm_smmu_domain_inv_range(invs, iova, size, granule, - single_ril, leaf); + __arm_smmu_domain_inv_range(&tlbi, invs); } rcu_read_unlock(); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index ab54210a8ed999..c7d64156edbba2 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -805,6 +805,15 @@ static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs) return new_invs; } +struct arm_smmu_tlbi { + unsigned long iova; + size_t size; + /* page or block size of the leaf iopte */ + unsigned int iopte_size; + bool has_cont; + bool leaf_only; +}; + struct arm_smmu_evtq { struct arm_smmu_queue q; struct iopf_queue *iopf; -- 2.43.0