From 258df8fce42fecc23cd04242de3d39f1fe836433 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 22 May 2026 07:22:12 -1000 Subject: mm: Add ptep_try_set() for lockless empty-slot installs Add ptep_try_set(ptep, new_pte): atomically set *ptep to new_pte iff it is currently pte_none(). Returns true on success, false if the slot was already populated or the arch has no implementation. The intended caller is the upcoming bpf_arena kernel-side fault recovery path. The install runs from a page fault that can be nested under locks held by the faulting kernel caller (e.g. a BPF program holding raw_res_spin_lock_irqsave on its arena's spinlock), so trylock-and-retry would A-A deadlock. Lock-free cmpxchg is the only viable option, which constrains this helper to special kernel page tables where concurrent writers cooperate via atomic accessors. The generic version in returns false. x86 and arm64 override with try_cmpxchg-based implementations on the underlying pteval. Other architectures get the false stub - the callers there already fall through to oops. v2: Rename to ptep_try_set(). Tighten kerneldoc. (David, Alexei) v3: Note that strict-zero cmpxchg is narrower than pte_none(). (Andrea) Suggested-by: Kumar Kartikeya Dwivedi Suggested-by: Alexei Starovoitov Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi Cc: David Hildenbrand Acked-by: David Hildenbrand (arm) Link: https://lore.kernel.org/r/20260522172219.1423324-2-tj@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/pgtable.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'include/linux/pgtable.h') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a..b5739bb99fc1 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1036,6 +1036,31 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres } #endif +#ifndef ptep_try_set +/** + * ptep_try_set - atomically set an empty kernel PTE + * @ptep: page table entry + * @new_pte: value to install + * + * Atomically set *@ptep to @new_pte iff *@ptep is pte_none(). Return true on + * success, false if the slot was already populated or the arch has no + * implementation. + * + * For special kernel page tables only - never user page tables. The caller must + * prevent concurrent teardown of @ptep and must accept that other writers may + * race. Concurrent clearers must use ptep_get_and_clear() so racing accesses + * agree on the outcome. + * + * Architectures opt in by providing a cmpxchg-based override and defining + * ptep_try_set as an identity macro. The generic stub returns false, which is + * correct for callers that fall through to oops on failure. + */ +static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte) +{ + return false; +} +#endif + #ifndef wrprotect_ptes /** * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same -- cgit v1.2.3 From 0aae825f1ed7ce3eedfadc54684aa86bbbe188b0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 29 May 2026 17:02:04 +0200 Subject: mm: Make empty_zero_page[] const The empty zero page is used to back any kernel or user space mapping that is supposed to remain cleared, and so the page itself is never supposed to be modified. So mark it as const, which moves it into .rodata rather than .bss: on most architectures, this ensures that both the kernel's mapping of it and any aliases that are accessible via the kernel direct (linear) map are mapped read-only, and cannot be used (inadvertently or maliciously) to corrupt the contents of the zero page. Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Kevin Brodsky Acked-by: David Hildenbrand (Arm) Reviewed-by: Jann Horn Reviewed-by: Feng Tang Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- include/linux/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/pgtable.h') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a..67aa23814010 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1993,7 +1993,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern const uint8_t empty_zero_page[PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) -- cgit v1.2.3 From f64c723741c911544cca4c838d7a291b06b3ad1d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 1 Jun 2026 08:37:28 -1000 Subject: bpf: Replace scratch PTE atomically when allocating arena pages apply_range_set_cb() maps the pages for a new arena allocation and returned -EBUSY when the target PTE was already populated. Kernel-fault recovery leaves the per-arena scratch page in unallocated arena PTEs, so a later bpf_arena_alloc_pages() over such a page hits that -EBUSY, and every subsequent allocation of it fails the same way. Allocation must install the real page over scratch instead. Overwriting the scratch PTE in place is a valid->valid change, which arm64 forbids without break-before-make. Route through an invalid entry instead: ptep_try_set() fills only a none slot, so the PTE goes scratch->none->page. On finding scratch, clear it and flush_tlb_before_set() before retrying. The new flush_tlb_before_set() is a no-op except on arches like arm64 that need the break-before-make TLB invalidate. The loop also copes with a concurrent fault re-scratching the slot. Arches without ptep_try_set() never install the scratch page, so keep the must-be-empty check and set_pte_at() for them. Fixes: dc11a4dba246 ("bpf: Recover arena kernel faults with scratch page") Signed-off-by: Tejun Heo Cc: Alexei Starovoitov Cc: David Hildenbrand Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/r/20260601183728.1800490-1-tj@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/pgtable.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux/pgtable.h') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index b5739bb99fc1..4c6c4081ef71 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1061,6 +1061,24 @@ static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte) } #endif +#ifndef flush_tlb_before_set +/** + * flush_tlb_before_set - invalidate a kernel PTE's TLB before re-setting it + * @addr: kernel virtual address whose PTE was just cleared + * + * Some architectures (e.g. arm64) do not allow a live page-table entry to be + * repointed at a different page in one step. The old entry must first be made + * invalid and its translation flushed from every TLB, and only then may the new + * entry be written. + * + * This is only for the lockless atomic kernel-PTE installers (ptep_try_set()). + * It must be callable with interrupts disabled. + */ +static inline void flush_tlb_before_set(unsigned long addr) +{ +} +#endif + #ifndef wrprotect_ptes /** * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same -- cgit v1.2.3 From 811f0d8caa407b7acd8d1ff9e2dd69835a7b3d9a Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Tue, 26 May 2026 17:05:12 +0200 Subject: mm: Refactor lazy_mmu_mode_pause() and lazy_mmu_mode_resume() In order to allow pausing and resuming MMU lazy mode for other tasks than current, refactor lazy_mmu_mode_pause() and lazy_mmu_mode_resume(). This will be needed when dropping the Xen PV private lazy MMU bookkeeping. Acked-by: "David Hildenbrand (Arm)" Signed-off-by: Juergen Gross Message-ID: <20260526150514.129330-4-jgross@suse.com> --- include/linux/pgtable.h | 56 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 11 deletions(-) (limited to 'include/linux/pgtable.h') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a..07483ed9b3ce 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -300,6 +300,28 @@ static inline void lazy_mmu_mode_disable(void) } +/** + * __task_lazy_mmu_mode_pause() - Pause the lazy MMU mode for a task. + * @tsk: The task to check. + * + * Pauses the lazy MMU mode of @tsk. + * + * This function only operates on the state saved in task_struct; to pause + * current lazy_mmu_mode_pause() should be used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline void __task_lazy_mmu_mode_pause(struct task_struct *tsk) +{ + struct lazy_mmu_state *state = &tsk->lazy_mmu_state; + + VM_WARN_ON_ONCE(state->pause_count == U8_MAX); + + if (state->pause_count++ == 0 && state->enable_count > 0) + arch_leave_lazy_mmu_mode(); +} + /** * lazy_mmu_mode_pause() - Pause the lazy MMU mode. * @@ -315,15 +337,32 @@ static inline void lazy_mmu_mode_disable(void) */ static inline void lazy_mmu_mode_pause(void) { - struct lazy_mmu_state *state = ¤t->lazy_mmu_state; - if (in_interrupt()) return; - VM_WARN_ON_ONCE(state->pause_count == U8_MAX); + __task_lazy_mmu_mode_pause(current); +} - if (state->pause_count++ == 0 && state->enable_count > 0) - arch_leave_lazy_mmu_mode(); +/** + * __task_lazy_mmu_mode_resume() - Resume the lazy MMU mode for a task. + * @tsk: The task to check. + * + * Resumes the lazy MMU mode of @tsk. + * + * This function only operates on the state saved in task_struct; to resume + * current lazy_mmu_mode_resume() should be used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline void __task_lazy_mmu_mode_resume(struct task_struct *tsk) +{ + struct lazy_mmu_state *state = &tsk->lazy_mmu_state; + + VM_WARN_ON_ONCE(state->pause_count == 0); + + if (--state->pause_count == 0 && state->enable_count > 0) + arch_enter_lazy_mmu_mode(); } /** @@ -341,15 +380,10 @@ static inline void lazy_mmu_mode_pause(void) */ static inline void lazy_mmu_mode_resume(void) { - struct lazy_mmu_state *state = ¤t->lazy_mmu_state; - if (in_interrupt()) return; - VM_WARN_ON_ONCE(state->pause_count == 0); - - if (--state->pause_count == 0 && state->enable_count > 0) - arch_enter_lazy_mmu_mode(); + __task_lazy_mmu_mode_resume(current); } #else static inline void lazy_mmu_mode_enable(void) {} -- cgit v1.2.3