summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/amd/iommu.c3
-rw-r--r--drivers/iommu/intel/iommu.c29
-rw-r--r--drivers/iommu/intel/iommu.h11
-rw-r--r--drivers/iommu/intel/nested.c9
-rw-r--r--drivers/iommu/intel/pasid.c190
-rw-r--r--drivers/iommu/intel/pasid.h14
-rw-r--r--drivers/iommu/intel/svm.c12
-rw-r--r--drivers/iommu/iommufd/eventq.c26
-rw-r--r--drivers/iommu/iommufd/hw_pagetable.c11
-rw-r--r--drivers/iommu/iommufd/iommufd_private.h2
10 files changed, 66 insertions, 241 deletions
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index dfbd09e2e0ca..5fef68797fda 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1689,7 +1689,8 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain,
{
lockdep_assert_held(&domain->lock);
- if (likely(!amd_iommu_np_cache)) {
+ if (likely(!amd_iommu_np_cache) ||
+ size >= (1ULL<<52)) {
__domain_flush_pages(domain, address, size);
/* Wait until IOMMU TLB and all device IOTLB flushes are complete */
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 79676188f60f..db0cf436c839 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1734,12 +1734,10 @@ int __domain_setup_first_level(struct intel_iommu *iommu, struct device *dev,
ioasid_t pasid, u16 did, phys_addr_t fsptptr,
int flags, struct iommu_domain *old)
{
- if (!old)
- return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid,
- did, flags);
- return intel_pasid_replace_first_level(iommu, dev, fsptptr, pasid, did,
- iommu_domain_did(old, iommu),
- flags);
+ if (old)
+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
+
+ return intel_pasid_setup_first_level(iommu, dev, fsptptr, pasid, did, flags);
}
static int domain_setup_second_level(struct intel_iommu *iommu,
@@ -1747,23 +1745,20 @@ static int domain_setup_second_level(struct intel_iommu *iommu,
struct device *dev, ioasid_t pasid,
struct iommu_domain *old)
{
- if (!old)
- return intel_pasid_setup_second_level(iommu, domain,
- dev, pasid);
- return intel_pasid_replace_second_level(iommu, domain, dev,
- iommu_domain_did(old, iommu),
- pasid);
+ if (old)
+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
+
+ return intel_pasid_setup_second_level(iommu, domain, dev, pasid);
}
static int domain_setup_passthrough(struct intel_iommu *iommu,
struct device *dev, ioasid_t pasid,
struct iommu_domain *old)
{
- if (!old)
- return intel_pasid_setup_pass_through(iommu, dev, pasid);
- return intel_pasid_replace_pass_through(iommu, dev,
- iommu_domain_did(old, iommu),
- pasid);
+ if (old)
+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
+
+ return intel_pasid_setup_pass_through(iommu, dev, pasid);
}
static int domain_setup_first_level(struct intel_iommu *iommu,
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 9198ac7f6bba..6b11081e690f 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1334,18 +1334,29 @@ void intel_iommu_disable_iopf(struct device *dev);
static inline int iopf_for_domain_set(struct iommu_domain *domain,
struct device *dev)
{
+ struct device_domain_info *info = dev_iommu_priv_get(dev);
+
if (!domain || !domain->iopf_handler)
return 0;
+ /* SVA with non-IOMMU/PRI IOPF handling is allowed. */
+ if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported)
+ return 0;
+
return intel_iommu_enable_iopf(dev);
}
static inline void iopf_for_domain_remove(struct iommu_domain *domain,
struct device *dev)
{
+ struct device_domain_info *info = dev_iommu_priv_get(dev);
+
if (!domain || !domain->iopf_handler)
return;
+ if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported)
+ return;
+
intel_iommu_disable_iopf(dev);
}
diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c
index e91b07475fb8..a909ab1a665a 100644
--- a/drivers/iommu/intel/nested.c
+++ b/drivers/iommu/intel/nested.c
@@ -141,11 +141,10 @@ static int domain_setup_nested(struct intel_iommu *iommu,
struct device *dev, ioasid_t pasid,
struct iommu_domain *old)
{
- if (!old)
- return intel_pasid_setup_nested(iommu, dev, pasid, domain);
- return intel_pasid_replace_nested(iommu, dev, pasid,
- iommu_domain_did(old, iommu),
- domain);
+ if (old)
+ intel_pasid_tear_down_entry(iommu, dev, pasid, false);
+
+ return intel_pasid_setup_nested(iommu, dev, pasid, domain);
}
static int intel_nested_set_dev_pasid(struct iommu_domain *domain,
diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 7a64a55fb588..787897e61efa 100644
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -417,50 +417,6 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev,
return 0;
}
-int intel_pasid_replace_first_level(struct intel_iommu *iommu,
- struct device *dev, phys_addr_t fsptptr,
- u32 pasid, u16 did, u16 old_did,
- int flags)
-{
- struct pasid_entry *pte, new_pte;
-
- if (!ecap_flts(iommu->ecap)) {
- pr_err("No first level translation support on %s\n",
- iommu->name);
- return -EINVAL;
- }
-
- if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {
- pr_err("No 5-level paging support for first-level on %s\n",
- iommu->name);
- return -EINVAL;
- }
-
- pasid_pte_config_first_level(iommu, &new_pte, fsptptr, did, flags);
-
- spin_lock(&iommu->lock);
- pte = intel_pasid_get_entry(dev, pasid);
- if (!pte) {
- spin_unlock(&iommu->lock);
- return -ENODEV;
- }
-
- if (!pasid_pte_is_present(pte)) {
- spin_unlock(&iommu->lock);
- return -EINVAL;
- }
-
- WARN_ON(old_did != pasid_get_domain_id(pte));
-
- *pte = new_pte;
- spin_unlock(&iommu->lock);
-
- intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
- intel_iommu_drain_pasid_prq(dev, pasid);
-
- return 0;
-}
-
/*
* Set up the scalable mode pasid entry for second only translation type.
*/
@@ -528,57 +484,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
return 0;
}
-int intel_pasid_replace_second_level(struct intel_iommu *iommu,
- struct dmar_domain *domain,
- struct device *dev, u16 old_did,
- u32 pasid)
-{
- struct pasid_entry *pte, new_pte;
- struct dma_pte *pgd;
- u64 pgd_val;
- u16 did;
-
- /*
- * If hardware advertises no support for second level
- * translation, return directly.
- */
- if (!ecap_slts(iommu->ecap)) {
- pr_err("No second level translation support on %s\n",
- iommu->name);
- return -EINVAL;
- }
-
- pgd = domain->pgd;
- pgd_val = virt_to_phys(pgd);
- did = domain_id_iommu(domain, iommu);
-
- pasid_pte_config_second_level(iommu, &new_pte, pgd_val,
- domain->agaw, did,
- domain->dirty_tracking);
-
- spin_lock(&iommu->lock);
- pte = intel_pasid_get_entry(dev, pasid);
- if (!pte) {
- spin_unlock(&iommu->lock);
- return -ENODEV;
- }
-
- if (!pasid_pte_is_present(pte)) {
- spin_unlock(&iommu->lock);
- return -EINVAL;
- }
-
- WARN_ON(old_did != pasid_get_domain_id(pte));
-
- *pte = new_pte;
- spin_unlock(&iommu->lock);
-
- intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
- intel_iommu_drain_pasid_prq(dev, pasid);
-
- return 0;
-}
-
/*
* Set up dirty tracking on a second only or nested translation type.
*/
@@ -691,38 +596,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
return 0;
}
-int intel_pasid_replace_pass_through(struct intel_iommu *iommu,
- struct device *dev, u16 old_did,
- u32 pasid)
-{
- struct pasid_entry *pte, new_pte;
- u16 did = FLPT_DEFAULT_DID;
-
- pasid_pte_config_pass_through(iommu, &new_pte, did);
-
- spin_lock(&iommu->lock);
- pte = intel_pasid_get_entry(dev, pasid);
- if (!pte) {
- spin_unlock(&iommu->lock);
- return -ENODEV;
- }
-
- if (!pasid_pte_is_present(pte)) {
- spin_unlock(&iommu->lock);
- return -EINVAL;
- }
-
- WARN_ON(old_did != pasid_get_domain_id(pte));
-
- *pte = new_pte;
- spin_unlock(&iommu->lock);
-
- intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
- intel_iommu_drain_pasid_prq(dev, pasid);
-
- return 0;
-}
-
/*
* Set the page snoop control for a pasid entry which has been set up.
*/
@@ -853,69 +726,6 @@ int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
return 0;
}
-int intel_pasid_replace_nested(struct intel_iommu *iommu,
- struct device *dev, u32 pasid,
- u16 old_did, struct dmar_domain *domain)
-{
- struct iommu_hwpt_vtd_s1 *s1_cfg = &domain->s1_cfg;
- struct dmar_domain *s2_domain = domain->s2_domain;
- u16 did = domain_id_iommu(domain, iommu);
- struct pasid_entry *pte, new_pte;
-
- /* Address width should match the address width supported by hardware */
- switch (s1_cfg->addr_width) {
- case ADDR_WIDTH_4LEVEL:
- break;
- case ADDR_WIDTH_5LEVEL:
- if (!cap_fl5lp_support(iommu->cap)) {
- dev_err_ratelimited(dev,
- "5-level paging not supported\n");
- return -EINVAL;
- }
- break;
- default:
- dev_err_ratelimited(dev, "Invalid stage-1 address width %d\n",
- s1_cfg->addr_width);
- return -EINVAL;
- }
-
- if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) {
- pr_err_ratelimited("No supervisor request support on %s\n",
- iommu->name);
- return -EINVAL;
- }
-
- if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) {
- pr_err_ratelimited("No extended access flag support on %s\n",
- iommu->name);
- return -EINVAL;
- }
-
- pasid_pte_config_nestd(iommu, &new_pte, s1_cfg, s2_domain, did);
-
- spin_lock(&iommu->lock);
- pte = intel_pasid_get_entry(dev, pasid);
- if (!pte) {
- spin_unlock(&iommu->lock);
- return -ENODEV;
- }
-
- if (!pasid_pte_is_present(pte)) {
- spin_unlock(&iommu->lock);
- return -EINVAL;
- }
-
- WARN_ON(old_did != pasid_get_domain_id(pte));
-
- *pte = new_pte;
- spin_unlock(&iommu->lock);
-
- intel_pasid_flush_present(iommu, dev, pasid, old_did, pte);
- intel_iommu_drain_pasid_prq(dev, pasid);
-
- return 0;
-}
-
/*
* Interfaces to setup or teardown a pasid table to the scalable-mode
* context table entry:
diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h
index 637373995be8..0296ca55ea5e 100644
--- a/drivers/iommu/intel/pasid.h
+++ b/drivers/iommu/intel/pasid.h
@@ -315,20 +315,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
struct device *dev, u32 pasid);
int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
u32 pasid, struct dmar_domain *domain);
-int intel_pasid_replace_first_level(struct intel_iommu *iommu,
- struct device *dev, phys_addr_t fsptptr,
- u32 pasid, u16 did, u16 old_did, int flags);
-int intel_pasid_replace_second_level(struct intel_iommu *iommu,
- struct dmar_domain *domain,
- struct device *dev, u16 old_did,
- u32 pasid);
-int intel_pasid_replace_pass_through(struct intel_iommu *iommu,
- struct device *dev, u16 old_did,
- u32 pasid);
-int intel_pasid_replace_nested(struct intel_iommu *iommu,
- struct device *dev, u32 pasid,
- u16 old_did, struct dmar_domain *domain);
-
void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
struct device *dev, u32 pasid,
bool fault_ignore);
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index 4cbbebc02749..e147f71f91b7 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -164,12 +164,9 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
if (IS_ERR(dev_pasid))
return PTR_ERR(dev_pasid);
- /* SVA with non-IOMMU/PRI IOPF handling is allowed. */
- if (info->pri_supported) {
- ret = iopf_for_domain_replace(domain, old, dev);
- if (ret)
- goto out_remove_dev_pasid;
- }
+ ret = iopf_for_domain_replace(domain, old, dev);
+ if (ret)
+ goto out_remove_dev_pasid;
/* Setup the pasid table: */
sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
@@ -183,8 +180,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
return 0;
out_unwind_iopf:
- if (info->pri_supported)
- iopf_for_domain_replace(old, domain, dev);
+ iopf_for_domain_replace(old, domain, dev);
out_remove_dev_pasid:
domain_remove_dev_pasid(domain, dev, pasid);
return ret;
diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c
index 8e45752f500e..617e20b67bf2 100644
--- a/drivers/iommu/iommufd/eventq.c
+++ b/drivers/iommu/iommufd/eventq.c
@@ -139,9 +139,14 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
mutex_lock(&fault->mutex);
while ((group = iommufd_fault_deliver_fetch(fault))) {
+ size_t group_done = done;
+
if (done >= count ||
group->fault_count * fault_size > count - done) {
iommufd_fault_deliver_restore(fault, group);
+ /* Read count doesn't fit the first fault group */
+ if (done == 0)
+ rc = -EINVAL;
break;
}
@@ -157,14 +162,17 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
iommufd_compose_fault_message(&iopf->fault,
&data, idev,
group->cookie);
- if (copy_to_user(buf + done, &data, fault_size)) {
+ if (copy_to_user(buf + group_done, &data, fault_size)) {
xa_erase(&fault->response, group->cookie);
iommufd_fault_deliver_restore(fault, group);
rc = -EFAULT;
break;
}
- done += fault_size;
+ group_done += fault_size;
}
+ if (rc)
+ break;
+ done = group_done;
}
mutex_unlock(&fault->mutex);
@@ -310,6 +318,9 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
if (*ppos)
return -ESPIPE;
+ /* Minimum read count is a vEVENT header */
+ if (count < sizeof(*hdr))
+ return -EINVAL;
while ((cur = iommufd_veventq_deliver_fetch(veventq))) {
/* Validate the remaining bytes against the header size */
@@ -321,8 +332,11 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
/* If being a normal vEVENT, validate against the full size */
if (!vevent_for_lost_events_header(cur) &&
- sizeof(hdr) + cur->data_len > count - done) {
+ sizeof(*hdr) + cur->data_len > count - done) {
iommufd_veventq_deliver_restore(veventq, cur);
+ /* Read count doesn't fit a single normal vEVENT */
+ if (done == 0)
+ rc = -EINVAL;
break;
}
@@ -336,6 +350,7 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
if (cur->data_len &&
copy_to_user(buf + done, cur->event_data, cur->data_len)) {
iommufd_veventq_deliver_restore(veventq, cur);
+ done -= sizeof(*hdr);
rc = -EFAULT;
break;
}
@@ -473,6 +488,9 @@ int iommufd_fault_iopf_handler(struct iopf_group *group)
static const struct file_operations iommufd_veventq_fops =
INIT_EVENTQ_FOPS(iommufd_veventq_fops_read, NULL);
+/* An arbitrary upper bound for veventq_depth that fits all existing HWs */
+#define VEVENTQ_MAX_DEPTH (1U << 19)
+
int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd)
{
struct iommu_veventq_alloc *cmd = ucmd->cmd;
@@ -484,7 +502,7 @@ int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd)
if (cmd->flags || cmd->__reserved ||
cmd->type == IOMMU_VEVENTQ_TYPE_DEFAULT)
return -EOPNOTSUPP;
- if (!cmd->veventq_depth)
+ if (!cmd->veventq_depth || cmd->veventq_depth > VEVENTQ_MAX_DEPTH)
return -EINVAL;
viommu = iommufd_get_viommu(ucmd, cmd->viommu_id);
diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c
index fe789c2dc0c9..623cc608ca0c 100644
--- a/drivers/iommu/iommufd/hw_pagetable.c
+++ b/drivers/iommu/iommufd/hw_pagetable.c
@@ -489,6 +489,9 @@ int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd)
return rc;
}
+/* An arbitrary entry_num cap, far above any realistic invalidation batch */
+#define IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX (1U << 19)
+
int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
{
struct iommu_hwpt_invalidate *cmd = ucmd->cmd;
@@ -507,7 +510,13 @@ int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
goto out;
}
- if (cmd->entry_num && (!cmd->data_uptr || !cmd->entry_len)) {
+ /*
+ * Bound entry_num and entry_len so a single call cannot pin the CPU;
+ * entry_len also caps the copy_struct_from_user() trailing-zero scan.
+ */
+ if (cmd->entry_num &&
+ (!cmd->data_uptr || !cmd->entry_len || cmd->entry_len > PAGE_SIZE ||
+ cmd->entry_num > IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX)) {
rc = -EINVAL;
goto out;
}
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 85d0843ed07b..53d37e77c4b9 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -599,7 +599,7 @@ struct iommufd_vevent {
struct iommufd_vevent_header header;
struct list_head node; /* for iommufd_eventq::deliver */
ssize_t data_len;
- u64 event_data[] __counted_by(data_len);
+ u8 event_data[] __counted_by(data_len);
};
#define vevent_for_lost_events_header(vevent) \
generated by cgit v1.2.3 (git 2.25.1) at 2026-07-21 20:33:59 +0000