summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd/ppr.c3
-rw-r--r--drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c56
-rw-r--r--drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c8
-rw-r--r--drivers/iommu/intel/dmar.c15
-rw-r--r--drivers/iommu/intel/iommu.c8
-rw-r--r--drivers/iommu/intel/iommu.h2
-rw-r--r--drivers/iommu/iommu-sva.c2
-rw-r--r--drivers/iommu/iommu.c2
-rw-r--r--drivers/iommu/iommufd/device.c4
-rw-r--r--drivers/iommu/iommufd/ioas.c2
-rw-r--r--drivers/iommu/iommufd/selftest.c31
-rw-r--r--drivers/iommu/msm_iommu.c10
12 files changed, 111 insertions, 32 deletions
diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c
index 1f8d2823bea4..80369ca1e316 100644
--- a/drivers/iommu/amd/ppr.c
+++ b/drivers/iommu/amd/ppr.c
@@ -151,7 +151,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw)
/* Submit event */
iommu_report_device_fault(&pdev->dev, &event);
-
+ pci_dev_put(pdev);
return;
out:
@@ -159,6 +159,7 @@ out:
amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]),
IOMMU_PAGE_RESP_FAILURE,
PPR_TAG(raw[0]) & 0x1FF);
+ pci_dev_put(pdev);
}
void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 7d18c6f9328a..daf8b87fdf08 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3837,6 +3837,20 @@ int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
return 0;
}
+static void arm_smmu_free_iopf_action(void *data)
+{
+ struct iopf_queue *queue = data;
+
+ iopf_queue_free(queue);
+}
+
+static void arm_smmu_destroy_vmid_map(void *data)
+{
+ struct ida *ida = data;
+
+ ida_destroy(ida);
+}
+
static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
{
int ret;
@@ -3864,6 +3878,11 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
if (!smmu->evtq.iopf)
return -ENOMEM;
+ ret = devm_add_action_or_reset(smmu->dev,
+ arm_smmu_free_iopf_action,
+ smmu->evtq.iopf);
+ if (ret)
+ return ret;
}
/* priq */
@@ -3942,7 +3961,8 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
ida_init(&smmu->vmid_map);
- return 0;
+ return devm_add_action_or_reset(smmu->dev, arm_smmu_destroy_vmid_map,
+ &smmu->vmid_map);
}
static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
@@ -4155,6 +4175,13 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
return ret;
}
+static void arm_smmu_disable_action(void *data)
+{
+ struct arm_smmu_device *smmu = data;
+
+ arm_smmu_device_disable(smmu);
+}
+
static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
{
struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
@@ -4878,7 +4905,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
/* Initialise in-memory data structures */
ret = arm_smmu_init_structures(smmu);
if (ret)
- goto err_free_iopf;
+ return ret;
/* Record our private device structure */
platform_set_drvdata(pdev, smmu);
@@ -4888,30 +4915,30 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
/* Reset the device */
ret = arm_smmu_device_reset(smmu);
+ if (ret) {
+ arm_smmu_device_disable(smmu);
+ return ret;
+ }
+
+ /* Register last so it unwinds first, while the CMDQ is still up. */
+ ret = devm_add_action_or_reset(smmu->dev, arm_smmu_disable_action, smmu);
if (ret)
- goto err_disable;
+ return ret;
/* And we're up. Go go go! */
ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
"smmu3.%pa", &ioaddr);
if (ret)
- goto err_disable;
+ return ret;
ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
if (ret) {
dev_err(dev, "Failed to register iommu\n");
- goto err_free_sysfs;
+ iommu_device_sysfs_remove(&smmu->iommu);
+ return ret;
}
return 0;
-
-err_free_sysfs:
- iommu_device_sysfs_remove(&smmu->iommu);
-err_disable:
- arm_smmu_device_disable(smmu);
-err_free_iopf:
- iopf_queue_free(smmu->evtq.iopf);
- return ret;
}
static void arm_smmu_device_remove(struct platform_device *pdev)
@@ -4920,9 +4947,6 @@ static void arm_smmu_device_remove(struct platform_device *pdev)
iommu_device_unregister(&smmu->iommu);
iommu_device_sysfs_remove(&smmu->iommu);
- arm_smmu_device_disable(smmu);
- iopf_queue_free(smmu->evtq.iopf);
- ida_destroy(&smmu->vmid_map);
}
static void arm_smmu_device_shutdown(struct platform_device *pdev)
diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
index 2f0f9a2f74d9..cee489855311 100644
--- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
+++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c
@@ -58,6 +58,8 @@
#define VINTF_ENABLED BIT(0)
#define TEGRA241_VINTF_SID_MATCH(s) (0x0040 + 0x4*(s))
+#define VINTF_SID_MATCH_VIRT_SID GENMASK(20, 1)
+#define VINTF_SID_MATCH_ENABLE BIT(0)
#define TEGRA241_VINTF_SID_REPLACE(s) (0x0080 + 0x4*(s))
#define TEGRA241_VINTF_LVCMDQ_ERR_MAP_64(m) \
@@ -1249,7 +1251,7 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
u64 virt_sid = vdev->virt_id;
int sidx;
- if (virt_sid > UINT_MAX)
+ if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID))
return -EINVAL;
WARN_ON_ONCE(master->num_streams != 1);
@@ -1261,7 +1263,9 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
return sidx;
writel(stream->id, REG_VINTF(vintf, SID_REPLACE(sidx)));
- writel(virt_sid << 1 | 0x1, REG_VINTF(vintf, SID_MATCH(sidx)));
+ writel(FIELD_PREP(VINTF_SID_MATCH_VIRT_SID, virt_sid) |
+ VINTF_SID_MATCH_ENABLE,
+ REG_VINTF(vintf, SID_MATCH(sidx)));
dev_dbg(vintf->cmdqv->dev,
"VINTF%u: allocated SID_REPLACE%d for pSID=%x, vSID=%x\n",
vintf->idx, sidx, stream->id, (u32)virt_sid);
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index 6938800e9884..f4bdc5ffa9d9 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -915,6 +915,18 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
return 0;
}
+static bool dmar_required(void)
+{
+ /* tboot supersedes any user/platform opt */
+ if (!intel_iommu_tboot_noforce && tboot_enabled())
+ return true;
+
+ if (!no_iommu && (!dmar_disabled || dmar_platform_optin()))
+ return true;
+
+ return false;
+}
+
void __init detect_intel_iommu(void)
{
int ret;
@@ -928,8 +940,7 @@ void __init detect_intel_iommu(void)
if (!ret)
ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
&validate_drhd_cb);
- if (!ret && !no_iommu && !iommu_detected &&
- (!dmar_disabled || dmar_platform_optin())) {
+ if (!ret && !iommu_detected && dmar_required()) {
iommu_detected = 1;
/* Make sure ACS will be enabled */
pci_request_acs();
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 8b38c65f403b..aec1a05d48d0 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -64,7 +64,7 @@ static int rwbf_quirk;
* (used when kernel is launched w/ TXT)
*/
static int force_on = 0;
-static int intel_iommu_tboot_noforce;
+int intel_iommu_tboot_noforce;
static int no_platform_optin;
#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
@@ -2956,10 +2956,11 @@ static bool has_external_pci(void)
static int __init platform_optin_force_iommu(void)
{
- if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
+ if (no_iommu || !dmar_platform_optin() || no_platform_optin ||
+ !has_external_pci())
return 0;
- if (no_iommu || dmar_disabled)
+ if (dmar_disabled)
pr_info("Intel-IOMMU force enabled due to platform opt in\n");
/*
@@ -2970,7 +2971,6 @@ static int __init platform_optin_force_iommu(void)
iommu_set_default_passthrough(false);
dmar_disabled = 0;
- no_iommu = 0;
return 1;
}
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 6b11081e690f..8e2df00e881c 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1434,6 +1434,7 @@ static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu)
extern int dmar_disabled;
extern int intel_iommu_enabled;
+extern int intel_iommu_tboot_noforce;
#else
static inline int iommu_calculate_agaw(struct intel_iommu *iommu)
{
@@ -1446,6 +1447,7 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
#define dmar_disabled (1)
#define intel_iommu_enabled (0)
#define intel_iommu_sm (0)
+#define intel_iommu_tboot_noforce (0)
#endif
static inline const char *decode_prq_descriptor(char *str, size_t size,
diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c
index fd735aaae9e3..838563fc4431 100644
--- a/drivers/iommu/iommu-sva.c
+++ b/drivers/iommu/iommu-sva.c
@@ -145,8 +145,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
list_add(&domain->next, &iommu_mm->sva_domains);
out:
refcount_set(&handle->users, 1);
- mutex_unlock(&iommu_sva_lock);
handle->dev = dev;
+ mutex_unlock(&iommu_sva_lock);
return handle;
out_free_domain:
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 59244c744eab..1547f3587a4e 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -360,7 +360,7 @@ int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu)
rc = device_add(dev);
if (rc)
- iommu_fwspec_free(dev);
+ dev_iommu_free(dev);
return rc;
}
EXPORT_SYMBOL_GPL(iommu_mock_device_add);
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index 5c7ead633edd..c40515bf5017 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -1307,8 +1307,8 @@ void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
xa_lock(&ioas->iopt.access_list);
xa_for_each(&ioas->iopt.access_list, index, access) {
- if (!iommufd_lock_obj(&access->obj) ||
- iommufd_access_is_internal(access))
+ if (iommufd_access_is_internal(access) ||
+ !iommufd_lock_obj(&access->obj))
continue;
xa_unlock(&ioas->iopt.access_list);
diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c
index cb1d2248393a..164612195905 100644
--- a/drivers/iommu/iommufd/ioas.c
+++ b/drivers/iommu/iommufd/ioas.c
@@ -433,6 +433,8 @@ static int iommufd_take_all_iova_rwsem(struct iommufd_ctx *ictx,
rc = xa_err(xa_store(ioas_list, index, ioas, GFP_KERNEL));
if (rc) {
+ up_write(&ioas->iopt.iova_rwsem);
+ refcount_dec(&ioas->obj.users);
iommufd_release_all_iova_rwsem(ictx, ioas_list);
return rc;
}
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index 8fd27f65d623..35c42ff4355a 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -183,6 +183,7 @@ struct mock_dev {
struct device dev;
struct mock_viommu *viommu;
struct rw_semaphore viommu_rwsem;
+ struct rw_semaphore iopf_rwsem;
unsigned long flags;
unsigned long vdev_id;
int id;
@@ -1101,6 +1102,7 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags)
return ERR_PTR(-ENOMEM);
init_rwsem(&mdev->viommu_rwsem);
+ init_rwsem(&mdev->iopf_rwsem);
device_initialize(&mdev->dev);
mdev->flags = dev_flags;
mdev->dev.release = mock_dev_release;
@@ -1126,7 +1128,9 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags)
goto err_put;
}
+ down_write(&mdev->iopf_rwsem);
rc = iommu_mock_device_add(&mdev->dev, &mock_iommu.iommu_dev);
+ up_write(&mdev->iopf_rwsem);
if (rc)
goto err_put;
return mdev;
@@ -1181,7 +1185,9 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
}
sobj->idev.idev = idev;
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
rc = iommufd_device_attach(idev, IOMMU_NO_PASID, &pt_id);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
if (rc)
goto out_unbind;
@@ -1196,7 +1202,9 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd,
return 0;
out_detach:
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
iommufd_device_detach(idev, IOMMU_NO_PASID);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
out_unbind:
iommufd_device_unbind(idev);
out_mdev:
@@ -1240,7 +1248,9 @@ static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd,
if (IS_ERR(sobj))
return PTR_ERR(sobj);
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
rc = iommufd_device_replace(sobj->idev.idev, IOMMU_NO_PASID, &pt_id);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
if (rc)
goto out_sobj;
@@ -1852,10 +1862,16 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd,
{
struct iopf_fault event = {};
struct iommufd_device *idev;
+ struct mock_dev *mdev;
idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id);
if (IS_ERR(idev))
return PTR_ERR(idev);
+ if (!iommufd_selftest_is_mock_dev(idev->dev)) {
+ iommufd_put_object(ucmd->ictx, &idev->obj);
+ return -EINVAL;
+ }
+ mdev = to_mock_dev(idev->dev);
event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID)
@@ -1866,7 +1882,9 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd,
event.fault.prm.grpid = cmd->trigger_iopf.grpid;
event.fault.prm.perm = cmd->trigger_iopf.perm;
+ down_read(&mdev->iopf_rwsem);
iommu_report_device_fault(idev->dev, &event);
+ up_read(&mdev->iopf_rwsem);
iommufd_put_object(ucmd->ictx, &idev->obj);
return 0;
@@ -1974,14 +1992,19 @@ static int iommufd_test_pasid_attach(struct iommufd_ucmd *ucmd,
if (IS_ERR(sobj))
return PTR_ERR(sobj);
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
rc = iommufd_device_attach(sobj->idev.idev, cmd->pasid_attach.pasid,
&cmd->pasid_attach.pt_id);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
if (rc)
goto out_sobj;
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
- if (rc)
+ if (rc) {
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
iommufd_device_detach(sobj->idev.idev, cmd->pasid_attach.pasid);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
+ }
out_sobj:
iommufd_put_object(ucmd->ictx, &sobj->obj);
@@ -1998,8 +2021,10 @@ static int iommufd_test_pasid_replace(struct iommufd_ucmd *ucmd,
if (IS_ERR(sobj))
return PTR_ERR(sobj);
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
rc = iommufd_device_replace(sobj->idev.idev, cmd->pasid_attach.pasid,
&cmd->pasid_attach.pt_id);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
if (rc)
goto out_sobj;
@@ -2019,7 +2044,9 @@ static int iommufd_test_pasid_detach(struct iommufd_ucmd *ucmd,
if (IS_ERR(sobj))
return PTR_ERR(sobj);
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
iommufd_device_detach(sobj->idev.idev, cmd->pasid_detach.pasid);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
iommufd_put_object(ucmd->ictx, &sobj->obj);
return 0;
}
@@ -2030,7 +2057,9 @@ void iommufd_selftest_destroy(struct iommufd_object *obj)
switch (sobj->type) {
case TYPE_IDEV:
+ down_write(&sobj->idev.mock_dev->iopf_rwsem);
iommufd_device_detach(sobj->idev.idev, IOMMU_NO_PASID);
+ up_write(&sobj->idev.mock_dev->iopf_rwsem);
iommufd_device_unbind(sobj->idev.idev);
mock_dev_destroy(sobj->idev.mock_dev);
break;
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 43a61ba021a5..d5e2fc9b0111 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -783,19 +783,25 @@ static int msm_iommu_probe(struct platform_device *pdev)
"msm-smmu.%pa", &ioaddr);
if (ret) {
pr_err("Could not add msm-smmu at %pa to sysfs\n", &ioaddr);
- return ret;
+ goto err_remove_list;
}
ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev);
if (ret) {
pr_err("Could not register msm-smmu at %pa\n", &ioaddr);
- return ret;
+ goto err_remove_sysfs;
}
pr_info("device mapped at %p, irq %d with %d ctx banks\n",
iommu->base, iommu->irq, iommu->ncb);
return ret;
+
+err_remove_sysfs:
+ iommu_device_sysfs_remove(&iommu->iommu);
+err_remove_list:
+ list_del(&iommu->dev_node);
+ return ret;
}
static const struct of_device_id msm_iommu_dt_match[] = {
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-09 06:47:24 +0000