From 215bb51fff467cdaa40d2df098a62ed2ae3e212d Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:53:57 +0200 Subject: mm: move struct slabobj_ext to mm/slab.h Users of include/linux/memcontrol.h don't need to see this internal structure. Further changes to the struct will reduce recompiling. Reviewed-by: Suren Baghdasaryan Reviewed-by: Harry Yoo (Oracle) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-3-c29ef0f1f257@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/memcontrol.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index e1f46a0016fc..93869cc35c25 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1440,19 +1440,6 @@ static inline void mem_cgroup_flush_workqueue(void) { } static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ -/* - * Extended information for slab objects stored as an array in page->memcg_data - * if MEMCG_DATA_OBJEXTS is set. - */ -struct slabobj_ext { -#ifdef CONFIG_MEMCG - struct obj_cgroup *objcg; -#endif -#ifdef CONFIG_MEM_ALLOC_PROFILING - union codetag_ref ref; -#endif -} __aligned(8); - static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) { struct mem_cgroup *memcg; -- cgit v1.2.3 From acc6fdade62c11d822f7b73f16092ebd365fc1c2 Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:14 +0900 Subject: mm/slab: introduce struct kvfree_rcu_head for kvfree_rcu batching rcu_head is overkill for kvfree_rcu() because the callback function is always either kfree(), vfree(), or free_large_kmalloc(), and thus there is no need for a function pointer. kvfree_rcu batching reuses the field to store the start address of an object, however, this is not strictly needed because we can calculate the start address in the slowpath. For the purpose of kvfree_rcu batching, it is sufficient to implement a linked list using a single pointer. Introduce a new struct called kvfree_rcu_head (the name was suggested by Vlastimil Babka), which is similar to rcu_head but is only a single pointer to build a linked list, without a function pointer, when CONFIG_KVFREE_RCU_BATCHED=y. When kvfree_rcu is not batched, kvfree_rcu_head is the same size as rcu_head. Note that shrinking struct kvfree_rcu_head on CONFIG_KVFREE_RCU_BATCHED=n kernels would inevitably require additional complexity and also some sort of batching (which defeats the purpose of the config option) because it cannot fall back to call_rcu(). For now there are no user-visible changes to the API. k[v]free_rcu() simply casts rcu_head to kvfree_rcu_head. While this does not affect the API, it allows kfree_rcu_nolock() to reuse kvfree_rcu batching as a fallback when trylock or sheaf allocation fails. Stop storing the object pointer in rcu_head.func and instead calculate the object's start address in kvfree_rcu_list(). Factor out the existing logic to calculate the start address from kvfree_rcu_cb() to kvmalloc_obj_start_addr(). To avoid losing the KASAN tag, calculate the offset and subtract it from the address of the kvfree_rcu_head. Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-6-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/rcupdate.h | 10 ++++++---- include/linux/types.h | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..ef5bb6981133 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1098,19 +1098,21 @@ static inline void rcu_read_unlock_migrate(void) /* * In mm/slab_common.c, no suitable header to include here. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr); +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the * comment of kfree_rcu() for details. */ -#define kvfree_rcu_arg_2(ptr, rhf) \ +#define kvfree_rcu_arg_2(ptr, kvrhf) \ do { \ typeof (ptr) ___p = (ptr); \ + struct kvfree_rcu_head *___head; \ \ if (___p) { \ - BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \ - kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + ___head = (struct kvfree_rcu_head *) &(___p)->kvrhf; \ + kvfree_call_rcu(___head, (void *) (___p)); \ } \ } while (0) diff --git a/include/linux/types.h b/include/linux/types.h index 93166b0b0617..7d1d305a763e 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -255,6 +255,16 @@ struct callback_head { } __attribute__((aligned(sizeof(void *)))); #define rcu_head callback_head +#ifdef CONFIG_KVFREE_RCU_BATCHED +struct kvfree_rcu_head { + struct kvfree_rcu_head *next; +}; +#else +struct kvfree_rcu_head { + struct rcu_head head; +}; +#endif + typedef void (*rcu_callback_t)(struct rcu_head *head); typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); -- cgit v1.2.3 From 3bc999d944b35dead1755b2bde1911cd5892225e Mon Sep 17 00:00:00 2001 From: "Harry Yoo (Oracle)" Date: Wed, 29 Jul 2026 17:20:15 +0900 Subject: mm/slab: introduce kfree_rcu_nolock() Currently, k[v]free_rcu() cannot be called in unknown context since it could lead to a deadlock when called in the middle of k[v]free_rcu(). Make users' lives easier by introducing kfree_rcu_nolock() variant, now that kfree_rcu_sheaf() is available on PREEMPT_RT and __kfree_rcu_sheaf() handles unknown context. When sheaves path fails, kfree_rcu_nolock() falls back to defer_kfree_rcu() that uses an irq work to free the object via kvfree_call_rcu(). In most cases, the sheaves path is expected to succeed and therefore it's unnecessary to introduce additional complexity to the existing kvfree_rcu batching by teaching it how to handle unknown context. Since defer_kfree_rcu() can be called on caches without sheaves, move deferred_work_barrier() and rcu_barrier() outside the branch in kvfree_rcu_barrier_on_cache(). Now that deferred kvfree_rcu objects are submitted to kvfree_call_rcu() after deferred_work_barrier() and may end up in RCU sheaves, deferred_work_barrier() must be invoked before flush_rcu_sheaves_on_cache(). Since the RCU sheaf path has not been used on !KVFREE_RCU_BATCHED kernels, always fall back when kvfree_rcu() is not batched, for consistency. kvfree_rcu_barrier{,_on_cache()}() on !KVFREE_RCU_BATCHED are moved to mm/slab_common.c to invoke deferred_work_barrier() before rcu_barrier(). Signed-off-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-7-a28cdcda9673@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/rcupdate.h | 22 ++++++++++++++++++++++ include/linux/slab.h | 16 +++------------- 2 files changed, 25 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index ef5bb6981133..aede77bd0387 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1099,6 +1099,7 @@ static inline void rcu_read_unlock_migrate(void) * In mm/slab_common.c, no suitable header to include here. */ void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the @@ -1124,6 +1125,27 @@ do { \ kvfree_call_rcu(NULL, (void *) (___p)); \ } while (0) +/** + * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context. + * @ptr: pointer to kfree for double-argument invocations. + * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr. + * + * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects + * without any deferred processing, but may still defer freeing. + * Large kmalloc and vmalloc objects are always deferred. + * + * kfree_rcu_nolock() supports 2-arg variant only. + */ +#define kfree_rcu_nolock(ptr, kvrhf) \ +do { \ + typeof (ptr) ___p = (ptr); \ + \ + if (___p) { \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p)); \ + } \ +} while (0) + /* * Place this after a lock-acquisition primitive to guarantee that * an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies diff --git a/include/linux/slab.h b/include/linux/slab.h index 32c9f8ed7ae2..066a25cc6966 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -1427,25 +1427,15 @@ extern void kvfree_sensitive(const void *addr, size_t len); unsigned int kmem_cache_size(struct kmem_cache *s); #ifndef CONFIG_KVFREE_RCU_BATCHED -static inline void kvfree_rcu_barrier(void) -{ - rcu_barrier(); -} - -static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) -{ - rcu_barrier(); -} - static inline void kfree_rcu_scheduler_running(void) { } #else +void kfree_rcu_scheduler_running(void); +#endif + void kvfree_rcu_barrier(void); void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); -void kfree_rcu_scheduler_running(void); -#endif - /** * kmalloc_size_roundup - Report allocation bucket size for the given size * -- cgit v1.2.3 From 7def2e8549e5186cd4de97ab5ce56f7944d3da59 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Mon, 27 Jul 2026 14:54:05 +0200 Subject: mm/slab: add cache_ and slab_needs_objcg() helpers Slabs of some caches never need the objcg part of struct slabobj_ext. Introduce helpers to query this for a cache or a slab. Introduce SLAB_MAY_ACCOUNT flag that is currently only internal and all caches have it set except: - KMALLOC_NORMAL caches, as long as KMALLOC_RECLAIM caches are separate - KMALLOC_NO_OBJ_EXT caches, if they exist For named caches we currently can't derive SLAB_MAY_ACCOUNT from SLAB_ACCOUNT because some caches might be created without SLAB_ACCOUNT and then used both with and without __GFP_ACCOUNT concurrently, allocating obj_ext arrays on demand. So just add the SLAB_MAY_ACCOUNT to all kmem caches, unless kmem accounting is disabled. This can be improved later by finding out all caches used with __GFP_ACCOUNT, creating them with the SLAB_MAY_ACCOUNT flag explicitly, and then ignoring __GFP_ACCOUNT for all other caches (possibly with a warning). To make the evaluation of slab_needs_objcg() faster in the allocation and free fast paths, add a obj_exts_needs_objcg flag into slab itself. This optimization is only available on 64bit architectures where free bits are available for the flag. Reviewed-by: Hao Li Link: https://patch.msgid.link/20260727-b4-objext_split-v3-11-c29ef0f1f257@kernel.org Reviewed-by: Harry Yoo Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/slab.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index 32c9f8ed7ae2..e1915026e030 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -45,6 +45,7 @@ enum _slab_flag_bits { #endif #ifdef CONFIG_MEMCG _SLAB_ACCOUNT, + _SLAB_MAY_ACCOUNT, #endif #ifdef CONFIG_KASAN_GENERIC _SLAB_KASAN, @@ -204,8 +205,10 @@ enum _slab_flag_bits { */ #ifdef CONFIG_MEMCG # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) +# define SLAB_MAY_ACCOUNT __SLAB_FLAG_BIT(_SLAB_MAY_ACCOUNT) #else # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED +# define SLAB_MAY_ACCOUNT __SLAB_FLAG_UNUSED #endif #ifdef CONFIG_KASAN_GENERIC -- cgit v1.2.3