DMA mappings update entries behind the page-table root installed in each attached device context. The existing bond and barrier protocol ensures that attachment either observes a completed page-table update or is included in its IOTLB invalidation. MSI forwarding also changes whether the MSI page-table configuration is installed in each device context. The first forwarded interrupt publishes the configuration to all devices bonded to the domain, while the last removes it. These domain-wide updates can run concurrently with attachment because devices in different IOMMU groups have different group mutexes. Serialize attachment to domains with MSI tables against forwarding state and device-context updates. During domain replacement, lock both old and new MSI tables because either domain may process forwarding changes for other attached devices while this device moves. This also prevents an old-domain RCU walk from overwriting the newly installed device context. Order the locks by address to prevent opposite-direction replacements from deadlocking. Domains without MSI tables continue to use only the bond lock for list updates. Signed-off-by: Andrew Jones --- drivers/iommu/riscv/iommu.c | 64 +++++++++++++++++++++++++++++++++++++ drivers/iommu/riscv/iommu.h | 2 ++ 2 files changed, 66 insertions(+) diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index 09ec8c3e4a72..57f2884dec42 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -874,6 +874,60 @@ struct riscv_iommu_info { struct riscv_iommu_domain *domain; }; +static struct riscv_iommu_msi_table *riscv_iommu_domain_msi_table(struct iommu_domain *iommu_domain) +{ + struct riscv_iommu_domain *domain; + + if (!iommu_domain || !(iommu_domain->type & __IOMMU_DOMAIN_PAGING)) + return NULL; + + domain = iommu_domain_to_riscv(iommu_domain); + if (!domain->msi_table.nr_ptes) + return NULL; + + return &domain->msi_table; +} + +static unsigned long riscv_iommu_msi_tables_lock(struct iommu_domain *domain1, + struct iommu_domain *domain2) +{ + struct riscv_iommu_msi_table *first = riscv_iommu_domain_msi_table(domain1); + struct riscv_iommu_msi_table *second = riscv_iommu_domain_msi_table(domain2); + unsigned long flags = 0; + + /* Address order is stable when the domains reverse roles. */ + if (!first || (second && first > second)) + swap(first, second); + + if (!first) + return flags; + + raw_spin_lock_irqsave(&first->lock, flags); + if (second && second != first) + raw_spin_lock_nested(&second->lock, SINGLE_DEPTH_NESTING); + + return flags; +} + +static void riscv_iommu_msi_tables_unlock(struct iommu_domain *domain1, + struct iommu_domain *domain2, + unsigned long flags) +{ + struct riscv_iommu_msi_table *first = riscv_iommu_domain_msi_table(domain1); + struct riscv_iommu_msi_table *second = riscv_iommu_domain_msi_table(domain2); + + /* Recreate the lock order and release the pair in reverse. */ + if (!first || (second && first > second)) + swap(first, second); + + if (!first) + return; + + if (second && second != first) + raw_spin_unlock(&second->lock); + raw_spin_unlock_irqrestore(&first->lock, flags); +} + /* * Linkage between an iommu_domain and attached devices. * @@ -1388,6 +1442,7 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain, struct riscv_iommu_bond *bond; struct pt_iommu_riscv_64_hw_info pt_info; struct riscv_iommu_dc dc = {0}; + unsigned long flags; int ret; pt_iommu_riscv_64_hw_info(&domain->riscvpt, &pt_info); @@ -1421,10 +1476,12 @@ static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain, return -ENOMEM; bond->dev = dev; + flags = riscv_iommu_msi_tables_lock(old, iommu_domain); riscv_iommu_bond_link(domain, bond); riscv_iommu_iodir_update(iommu, dev, &dc); riscv_iommu_bond_unlink(info->domain, dev); info->domain = domain; + riscv_iommu_msi_tables_unlock(old, iommu_domain, flags); return 0; } @@ -1474,6 +1531,7 @@ riscv_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags, INIT_LIST_HEAD_RCU(&domain->bonds); raw_spin_lock_init(&domain->lock); + raw_spin_lock_init(&domain->msi_table.lock); mutex_init(&domain->mutex); iommu = dev_to_iommu(dev); cfg.common.hw_max_oasz_lg2 = 56; @@ -1569,13 +1627,16 @@ static int riscv_iommu_attach_blocking_domain(struct iommu_domain *iommu_domain, struct riscv_iommu_device *iommu = dev_to_iommu(dev); struct riscv_iommu_info *info = dev_iommu_priv_get(dev); struct riscv_iommu_dc dc = {0}; + unsigned long flags; dc.fsc = RISCV_IOMMU_FSC_BARE; /* Make device context invalid, translation requests will fault w/ #258 */ + flags = riscv_iommu_msi_tables_lock(old, NULL); riscv_iommu_iodir_update(iommu, dev, &dc); riscv_iommu_bond_unlink(info->domain, dev); info->domain = NULL; + riscv_iommu_msi_tables_unlock(old, NULL, flags); return 0; } @@ -1594,13 +1655,16 @@ static int riscv_iommu_attach_identity_domain(struct iommu_domain *iommu_domain, struct riscv_iommu_device *iommu = dev_to_iommu(dev); struct riscv_iommu_info *info = dev_iommu_priv_get(dev); struct riscv_iommu_dc dc = {0}; + unsigned long flags; dc.fsc = RISCV_IOMMU_FSC_BARE; dc.ta = RISCV_IOMMU_PC_TA_V; + flags = riscv_iommu_msi_tables_lock(old, NULL); riscv_iommu_iodir_update(iommu, dev, &dc); riscv_iommu_bond_unlink(info->domain, dev); info->domain = NULL; + riscv_iommu_msi_tables_unlock(old, NULL, flags); return 0; } diff --git a/drivers/iommu/riscv/iommu.h b/drivers/iommu/riscv/iommu.h index 6bea9da71ff3..2876703a6698 100644 --- a/drivers/iommu/riscv/iommu.h +++ b/drivers/iommu/riscv/iommu.h @@ -69,6 +69,8 @@ struct riscv_iommu_device { }; struct riscv_iommu_msi_table { + /* Protects attachment, interrupt forwarding state, and MSI PTE updates. */ + raw_spinlock_t lock; unsigned int nr_ptes; struct riscv_iommu_msipte *root; u64 msi_addr_mask; -- 2.43.0