mm/slab: introduce struct kvfree_rcu_head for kvfree_rcu batching

rcu_head is overkill for kvfree_rcu() because the callback
function is always either kfree(), vfree(), or free_large_kmalloc(),
and thus there is no need for a function pointer.

kvfree_rcu batching reuses the field to store the start address
of an object, however, this is not strictly needed because we can
calculate the start address in the slowpath. For the purpose of
kvfree_rcu batching, it is sufficient to implement a linked list using
a single pointer.

Introduce a new struct called kvfree_rcu_head (the name was suggested
by Vlastimil Babka), which is similar to rcu_head but is only a single
pointer to build a linked list, without a function pointer, when
CONFIG_KVFREE_RCU_BATCHED=y.

When kvfree_rcu is not batched, kvfree_rcu_head is the same size
as rcu_head. Note that shrinking struct kvfree_rcu_head on
CONFIG_KVFREE_RCU_BATCHED=n kernels would inevitably require additional
complexity and also some sort of batching (which defeats the purpose of
the config option) because it cannot fall back to call_rcu().

For now there are no user-visible changes to the API. k[v]free_rcu()
simply casts rcu_head to kvfree_rcu_head. While this does not affect
the API, it allows kfree_rcu_nolock() to reuse kvfree_rcu batching
as a fallback when trylock or sheaf allocation fails.

Stop storing the object pointer in rcu_head.func and instead calculate
the object's start address in kvfree_rcu_list(). Factor out the existing
logic to calculate the start address from kvfree_rcu_cb() to
kvmalloc_obj_start_addr(). To avoid losing the KASAN tag, calculate
the offset and subtract it from the address of the kvfree_rcu_head.

Signed-off-by: Harry Yoo (Oracle) <harry@kernel.org>
Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-6-a28cdcda9673@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

3 files changed, 17 insertions, 5 deletions

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 5e95acc33989..ef5bb6981133 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1098,19 +1098,21 @@ static inline void rcu_read_unlock_migrate(void)
 /*
  * In mm/slab_common.c, no suitable header to include here.
  */
-void kvfree_call_rcu(struct rcu_head *head, void *ptr);
+void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr);
 
 /*
  * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the
  * comment of kfree_rcu() for details.
  */
-#define kvfree_rcu_arg_2(ptr, rhf)					\
+#define kvfree_rcu_arg_2(ptr, kvrhf)					\
 do {									\
 	typeof (ptr) ___p = (ptr);					\
+	struct kvfree_rcu_head *___head;				\
 									\
 	if (___p) {							\
-		BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096);	\
-		kvfree_call_rcu(&((___p)->rhf), (void *) (___p));	\
+		BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096);	\
+		___head = (struct kvfree_rcu_head *) &(___p)->kvrhf;	\
+		kvfree_call_rcu(___head, (void *) (___p));		\
 	}								\
 } while (0)
 
diff --git a/include/linux/types.h b/include/linux/types.h
index 93166b0b0617..7d1d305a763e 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -255,6 +255,16 @@ struct callback_head {
 } __attribute__((aligned(sizeof(void *))));
 #define rcu_head callback_head
 
+#ifdef CONFIG_KVFREE_RCU_BATCHED
+struct kvfree_rcu_head {
+	struct kvfree_rcu_head *next;
+};
+#else
+struct kvfree_rcu_head {
+	struct rcu_head head;
+};
+#endif
+
 typedef void (*rcu_callback_t)(struct rcu_head *head);
 typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func);
 
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 5fbdabe3faea..a74027126a2f 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -625,7 +625,7 @@ TRACE_EVENT_RCU(rcu_invoke_callback,
  */
 TRACE_EVENT_RCU(rcu_invoke_kvfree_callback,
 
-	TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset),
+	TP_PROTO(const char *rcuname, struct kvfree_rcu_head *rhp, unsigned long offset),
 
 	TP_ARGS(rcuname, rhp, offset),