summaryrefslogtreecommitdiff
path: root/arch/s390
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390')
-rw-r--r--arch/s390/include/asm/pgtable.h4
-rw-r--r--arch/s390/include/asm/uaccess.h3
-rw-r--r--arch/s390/kernel/diag/diag310.c7
-rw-r--r--arch/s390/kernel/perf_cpum_cf.c3
-rw-r--r--arch/s390/kernel/process.c2
-rw-r--r--arch/s390/kvm/dat.c2
-rw-r--r--arch/s390/kvm/gaccess.c73
-rw-r--r--arch/s390/kvm/gmap.c7
-rw-r--r--arch/s390/kvm/gmap.h14
-rw-r--r--arch/s390/kvm/kvm-s390.c2
-rw-r--r--arch/s390/kvm/pci.c12
-rw-r--r--arch/s390/mm/gmap_helpers.c9
-rw-r--r--arch/s390/mm/mmap.c2
13 files changed, 87 insertions, 53 deletions
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 2c6cee8241e0..4740c75649eb 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -980,6 +980,8 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
static inline void set_pte(pte_t *ptep, pte_t pte)
{
+ if (pte_present(pte))
+ pte = clear_pte_bit(pte, __pgprot(_PAGE_UNUSED));
WRITE_ONCE(*ptep, pte);
}
@@ -1332,8 +1334,6 @@ pgprot_t pgprot_writecombine(pgprot_t prot);
static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry, unsigned int nr)
{
- if (pte_present(entry))
- entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
page_table_check_ptes_set(mm, addr, ptep, entry, nr);
for (;;) {
set_pte(ptep, entry);
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index dff035372601..a9f32c53f699 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -30,8 +30,7 @@ void debug_user_asce(int exit);
#define uaccess_kmsan_or_inline __always_inline
#endif
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
static uaccess_kmsan_or_inline __must_check unsigned long
raw_copy_from_user(void *to, const void __user *from, unsigned long size)
diff --git a/arch/s390/kernel/diag/diag310.c b/arch/s390/kernel/diag/diag310.c
index f411562aa7f6..c68ec9d28513 100644
--- a/arch/s390/kernel/diag/diag310.c
+++ b/arch/s390/kernel/diag/diag310.c
@@ -190,17 +190,18 @@ static int memtop_get_stride_len(unsigned long *res)
static int memtop_get_page_count(unsigned long *res, unsigned long level)
{
static unsigned long memtop_pages[DIAG310_LEVELMAX];
- unsigned long pages;
+ unsigned long pages, idx;
int rc;
if (level > DIAG310_LEVELMAX || level < DIAG310_LEVELMIN)
return -EINVAL;
- pages = READ_ONCE(memtop_pages[level - 1]);
+ idx = array_index_nospec(level - 1, ARRAY_SIZE(memtop_pages));
+ pages = READ_ONCE(memtop_pages[idx]);
if (!pages) {
rc = diag310_get_memtop_size(&pages, level);
if (rc)
return rc;
- WRITE_ONCE(memtop_pages[level - 1], pages);
+ WRITE_ONCE(memtop_pages[idx], pages);
}
*res = pages;
return 0;
diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c
index 7aa655664ecc..2076ac22e2c4 100644
--- a/arch/s390/kernel/perf_cpum_cf.c
+++ b/arch/s390/kernel/perf_cpum_cf.c
@@ -15,6 +15,7 @@
#include <linux/init.h>
#include <linux/miscdevice.h>
#include <linux/perf_event.h>
+#include <linux/nospec.h>
#include <asm/cpu_mf.h>
#include <asm/hwctrset.h>
@@ -768,6 +769,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
if (!is_userspace_event(ev)) {
if (ev >= ARRAY_SIZE(cpumf_generic_events_user))
return -EOPNOTSUPP;
+ ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_user));
ev = cpumf_generic_events_user[ev];
}
} else if (!attr->exclude_kernel && attr->exclude_user) {
@@ -778,6 +780,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
if (!is_userspace_event(ev)) {
if (ev >= ARRAY_SIZE(cpumf_generic_events_basic))
return -EOPNOTSUPP;
+ ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_basic));
ev = cpumf_generic_events_basic[ev];
}
}
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 0df95dcb2101..e4bd273e31f4 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -50,7 +50,7 @@ void ret_from_fork(void) asm("ret_from_fork");
void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs)
{
- void (*func)(void *arg);
+ int (*func)(void *arg);
schedule_tail(prev);
diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c
index 4a41c0247ffa..a4fe664f65ee 100644
--- a/arch/s390/kvm/dat.c
+++ b/arch/s390/kvm/dat.c
@@ -45,7 +45,7 @@ int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc)
mc->pts[mc->n_pts] = o;
}
for ( ; mc->n_rmaps < KVM_S390_MMU_CACHE_N_RMAPS; mc->n_rmaps++) {
- o = kzalloc_obj(*mc->rmaps[0], GFP_KERNEL_ACCOUNT);
+ o = kzalloc_obj(struct vsie_rmap, GFP_KERNEL_ACCOUNT);
if (!o)
return -ENOMEM;
mc->rmaps[mc->n_rmaps] = o;
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index 20e28b183c1a..36102b2727fb 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -1419,8 +1419,8 @@ edat_applies:
return kvm_s390_get_guest_page(kvm, entries + LEVEL_MEM, table.pte.pfra, wr);
}
-static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union pte *ptep,
- struct guest_fault *f, bool p)
+static int _do_shadow_pte(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gpa_t raddr,
+ union pte *ptep_h, union pte *ptep, struct guest_fault *f, bool p)
{
union pgste pgste;
union pte newpte;
@@ -1430,7 +1430,7 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union
lockdep_assert_held(&sg->parent->children_lock);
scoped_guard(spinlock, &sg->host_to_rmap_lock)
- rc = gmap_insert_rmap(sg, f->gfn, gpa_to_gfn(raddr), TABLE_TYPE_PAGE_TABLE);
+ rc = gmap_insert_rmap(mc, sg, f->gfn, gpa_to_gfn(raddr), TABLE_TYPE_PAGE_TABLE);
if (rc)
return rc;
@@ -1462,8 +1462,8 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union
return 0;
}
-static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, union crste *table,
- struct guest_fault *f, bool p)
+static int _do_shadow_crste(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gpa_t raddr,
+ union crste *host, union crste *table, struct guest_fault *f, bool p)
{
union crste newcrste, oldcrste;
unsigned long mask;
@@ -1476,7 +1476,7 @@ static int _do_shadow_crste(struct gmap *sg, gpa_t raddr, union crste *host, uni
mask = is_pmd(*table) ? _SEGMENT_FR_MASK : _REGION3_FR_MASK;
r_gfn = gpa_to_gfn(raddr) & mask;
scoped_guard(spinlock, &sg->host_to_rmap_lock)
- rc = gmap_insert_rmap(sg, f->gfn & mask, r_gfn, host->h.tt);
+ rc = gmap_insert_rmap(mc, sg, f->gfn & mask, r_gfn, host->h.tt);
if (rc)
return rc;
@@ -1578,39 +1578,52 @@ real_address_space:
if (KVM_BUG_ON(l > TABLE_TYPE_REGION3, sg->kvm))
return -EFAULT;
if (l == TABLE_TYPE_PAGE_TABLE)
- return _do_shadow_pte(sg, saddr, ptep_h, ptep, entries + LEVEL_MEM, w->p);
- return _do_shadow_crste(sg, saddr, host, table, entries + LEVEL_MEM, w->p);
+ return _do_shadow_pte(mc, sg, saddr, ptep_h, ptep, entries + LEVEL_MEM, w->p);
+ return _do_shadow_crste(mc, sg, saddr, host, table, entries + LEVEL_MEM, w->p);
}
-static inline int _gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
- unsigned long seq, struct pgtwalk *walk)
+static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
+ unsigned long seq, struct pgtwalk *walk)
{
struct gmap *parent;
int rc;
- if (kvm_s390_array_needs_retry_unsafe(vcpu->kvm, seq, walk->raw_entries))
+ if (kvm_s390_array_needs_retry_safe(vcpu->kvm, seq, walk->raw_entries))
return -EAGAIN;
-again:
- rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc);
- if (rc)
- return rc;
- scoped_guard(read_lock, &vcpu->kvm->mmu_lock) {
- if (kvm_s390_array_needs_retry_safe(vcpu->kvm, seq, walk->raw_entries))
- return -EAGAIN;
- parent = READ_ONCE(sg->parent);
- if (!parent)
+ parent = READ_ONCE(sg->parent);
+ if (!parent)
+ return -EAGAIN;
+ scoped_guard(spinlock, &parent->children_lock) {
+ if (READ_ONCE(sg->parent) != parent)
return -EAGAIN;
- scoped_guard(spinlock, &parent->children_lock) {
- if (READ_ONCE(sg->parent) != parent)
- return -EAGAIN;
- sg->invalidated = false;
- rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk);
- }
- if (rc == -ENOMEM)
- goto again;
- if (!rc)
- kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false);
+ sg->invalidated = false;
+ rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk);
}
+ if (!rc)
+ kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false);
+ return rc;
+}
+
+static inline int _gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg, gpa_t saddr,
+ unsigned long seq, struct pgtwalk *walk)
+{
+ int rc;
+
+ if (kvm_s390_array_needs_retry_unsafe(vcpu->kvm, seq, walk->raw_entries))
+ return -EAGAIN;
+
+ do {
+ rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc);
+ if (rc)
+ return rc;
+ rc = radix_tree_preload(GFP_KERNEL);
+ if (rc)
+ return rc;
+ scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
+ rc = ___gaccess_shadow_fault(vcpu, sg, saddr, seq, walk);
+ radix_tree_preload_end();
+ } while (rc == -ENOMEM);
+
return rc;
}
diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c
index 52d55ddea8d4..1d289f8fa3b2 100644
--- a/arch/s390/kvm/gmap.c
+++ b/arch/s390/kvm/gmap.c
@@ -1000,7 +1000,8 @@ int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interr
return 0;
}
-int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level)
+int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn,
+ gfn_t r_gfn, int level)
{
struct vsie_rmap *rmap __free(kvfree) = NULL;
struct vsie_rmap *temp;
@@ -1010,7 +1011,7 @@ int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level)
KVM_BUG_ON(!is_shadow(sg), sg->kvm);
lockdep_assert_held(&sg->host_to_rmap_lock);
- rmap = kzalloc_obj(*rmap, GFP_ATOMIC);
+ rmap = kvm_s390_mmu_cache_alloc_rmap(mc);
if (!rmap)
return -ENOMEM;
@@ -1057,7 +1058,7 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf
if (level <= TABLE_TYPE_REGION1) {
bitmask = -1UL << (8 + 11 * level);
scoped_guard(spinlock, &sg->host_to_rmap_lock)
- rc = gmap_insert_rmap(sg, p_gfn, r_gfn & bitmask, level);
+ rc = gmap_insert_rmap(mc, sg, p_gfn, r_gfn & bitmask, level);
}
if (rc)
return rc;
diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h
index 5374f21aaf8d..1c040472f56d 100644
--- a/arch/s390/kvm/gmap.h
+++ b/arch/s390/kvm/gmap.h
@@ -100,7 +100,8 @@ int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long cou
void gmap_ucas_unmap(struct gmap *gmap, gfn_t c_gfn, unsigned long count);
int gmap_enable_skeys(struct gmap *gmap);
int gmap_pv_destroy_range(struct gmap *gmap, gfn_t start, gfn_t end, bool interruptible);
-int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level);
+int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn,
+ gfn_t r_gfn, int level);
int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn,
kvm_pfn_t pfn, int level, bool wr);
void gmap_set_cmma_all_dirty(struct gmap *gmap);
@@ -279,7 +280,16 @@ static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, unio
gmap_handle_vsie_unshadow_event(gmap, gfn);
else
_gmap_handle_vsie_unshadow_event(gmap, gfn);
- dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce);
+ if (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, gmap->asce))
+ return false;
+ /*
+ * Return false even if the swap was successful, as it only
+ * indicates that the best effort clearing of the vsie_notif
+ * bit was successful. The caller will have to try again
+ * regardless, since the desired value has not been set.
+ * This pointless check is needed to silence a potential
+ * __must_check warning.
+ */
return false;
}
if (!oldcrste.s.fc1.d && newcrste.s.fc1.d && !newcrste.s.fc1.s)
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index ffb20a64d328..6afd45b5f7dc 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -2252,7 +2252,7 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm,
return 0;
}
- values = vmalloc(args->count);
+ values = vzalloc(args->count);
if (!values)
return -ENOMEM;
diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c
index 5b075c38998e..720bb58cabe2 100644
--- a/arch/s390/kvm/pci.c
+++ b/arch/s390/kvm/pci.c
@@ -300,9 +300,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib,
gaite->gisc = fib->fmt0.isc;
gaite->count++;
- gaite->aisbo = fib->fmt0.aisbo;
- gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb &
- ~PAGE_MASK));
+ if (fib->fmt0.sum == 1) {
+ gaite->aisbo = fib->fmt0.aisbo;
+ gaite->aisb = virt_to_phys(page_address(aisb_page) +
+ (fib->fmt0.aisb & ~PAGE_MASK));
+ } else {
+ gaite->aisbo = 0;
+ gaite->aisb = 0;
+ }
aift->kzdev[zdev->aisb] = zdev->kzdev;
spin_unlock_irq(&aift->gait_lock);
@@ -328,6 +333,7 @@ unpin2:
unpin1:
unpin_user_page(aibv_page);
out:
+ kvm_s390_gisc_unregister(kvm, fib->fmt0.isc);
return rc;
}
diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c
index 1cfe4724fbe2..4bf7c9012feb 100644
--- a/arch/s390/mm/gmap_helpers.c
+++ b/arch/s390/mm/gmap_helpers.c
@@ -51,15 +51,15 @@ pte_t *try_get_locked_pte(struct mm_struct *mm, unsigned long vmaddr, spinlock_t
pgd = pgdp_get(pgdp);
if (pgd_none(pgd) || !pgd_present(pgd))
return NULL;
- p4dp = p4d_offset(pgdp, vmaddr);
+ p4dp = p4d_offset_lockless(pgdp, pgd, vmaddr);
p4d = p4dp_get(p4dp);
if (p4d_none(p4d) || !p4d_present(p4d))
return NULL;
- pudp = pud_offset(p4dp, vmaddr);
+ pudp = pud_offset_lockless(p4dp, p4d, vmaddr);
pud = pudp_get(pudp);
if (pud_none(pud) || pud_leaf(pud) || !pud_present(pud))
return NULL;
- pmdp = pmd_offset(pudp, vmaddr);
+ pmdp = pmd_offset_lockless(pudp, pud, vmaddr);
pmd = pmdp_get_lockless(pmdp);
if (pmd_none(pmd) || pmd_leaf(pmd) || !pmd_present(pmd))
return NULL;
@@ -181,7 +181,8 @@ void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr)
if (IS_ERR_OR_NULL(ptep))
return;
- __atomic64_or(_PAGE_UNUSED, (long *)ptep);
+ if (pte_present(*ptep))
+ __atomic64_or(_PAGE_UNUSED, (long *)ptep);
pte_unmap_unlock(ptep, ptl);
}
EXPORT_SYMBOL_GPL(gmap_helper_try_set_pte_unused);
diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c
index 2a222a7e14f4..ef7bfc87758c 100644
--- a/arch/s390/mm/mmap.c
+++ b/arch/s390/mm/mmap.c
@@ -64,7 +64,7 @@ static inline unsigned long mmap_base(unsigned long rnd,
return PAGE_ALIGN(STACK_TOP - gap - rnd);
}
-static int get_align_mask(struct file *filp, unsigned long flags)
+static unsigned long get_align_mask(struct file *filp, unsigned long flags)
{
if (filp && is_file_hugepages(filp))
return huge_page_mask_align(filp);
generated by cgit v1.2.3 (git 2.25.1) at 2026-07-25 17:25:53 +0000