Merge branch 'slab/for-7.3/kfree_rcu_nolock' into slab/for-next

Merge series "mm/slab: introduce kfree_rcu_nolock() and improve
slub_kunit coverage" from Harry Yoo. From the cover letter [1]:

This series improves kmalloc_nolock() and kfree_nolock() coverage in
slub_kunit and introduces kfree_rcu_nolock() for unknown context as
suggested by Alexei Starovoitov.

Unknown context means the caller does not know whether spinning on a
lock is safe (e.g., a BPF program attached to an arbitrary kernel
function or in NMI context).

The slab allocator already supports unknown context via kmalloc_nolock()
and kfree_nolock(), but te slab allocator does not support freeing
objects by RCU in unknown context.

It is not ideal to have completely separate batching for unknown context
because the worst scenario where spinning on a lock would lead to
deadlock is very rare, and in most cases, it is safe to use the existing
mechanism (kfree_rcu_sheaf()).

Since most part of the slab allocator already supports unknown context
and sheaves support batching kvfree_rcu() calls for slab objects,
implement kfree_rcu_nolock() with minimal changes by teaching
kfree_rcu_sheaf() how to support unknown context and making it a little
bit harder to allocate an empty sheaf, instead of making intrusive
changes to the existing kvfree_rcu batching logic.

kfree_rcu_nolock() tries to free the object to the rcu sheaf if trylock
succeeds. Once the rcu sheaf becomes full, it is submitted to RCU via
call_rcu() if spinning is allowed or IRQs are enabled (to avoid calling
call_rcu() in the middle of call_rcu()). Otherwise, call_rcu() is
deferred via irq work.

When there is no sheaf available, kfree_rcu_sheaf() falls back to
defer_kfree_rcu(). It submits the object to kvfree_rcu batching via irq
work. To do this, patch 6 converts kvfree_rcu to use kvfree_rcu_head
without visible changes to the API for now.

Unlike kfree_rcu(), only the 2-argument variant is supported.  This is
because the last resort of the 1-arg variant is synchronize_rcu(), which
cannot be used in an unknown context.

As suggested by Alexei Starovoitov, kfree_rcu_nolock() can be used with
struct kvfree_rcu_head (8 bytes), which is smaller than struct rcu_head
(16 bytes).

Link: https://lore.kernel.org/all/20260729-kfree_rcu_nolock-v5-0-a28cdcda9673@kernel.org/ [1]

3 files changed, 41 insertions, 17 deletions

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 5e95acc33989..aede77bd0387 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1098,19 +1098,22 @@ static inline void rcu_read_unlock_migrate(void)
 /*
  * In mm/slab_common.c, no suitable header to include here.
  */
-void kvfree_call_rcu(struct rcu_head *head, void *ptr);
+void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr);
+void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr);
 
 /*
  * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the
  * comment of kfree_rcu() for details.
  */
-#define kvfree_rcu_arg_2(ptr, rhf)					\
+#define kvfree_rcu_arg_2(ptr, kvrhf)					\
 do {									\
 	typeof (ptr) ___p = (ptr);					\
+	struct kvfree_rcu_head *___head;				\
 									\
 	if (___p) {							\
-		BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096);	\
-		kvfree_call_rcu(&((___p)->rhf), (void *) (___p));	\
+		BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096);	\
+		___head = (struct kvfree_rcu_head *) &(___p)->kvrhf;	\
+		kvfree_call_rcu(___head, (void *) (___p));		\
 	}								\
 } while (0)
 
@@ -1122,6 +1125,27 @@ do {								\
 		kvfree_call_rcu(NULL, (void *) (___p));		\
 } while (0)
 
+/**
+ * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context.
+ * @ptr: pointer to kfree for double-argument invocations.
+ * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr.
+ *
+ * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects
+ * without any deferred processing, but may still defer freeing.
+ * Large kmalloc and vmalloc objects are always deferred.
+ *
+ * kfree_rcu_nolock() supports 2-arg variant only.
+ */
+#define kfree_rcu_nolock(ptr, kvrhf)						\
+do {										\
+	typeof (ptr) ___p = (ptr);						\
+										\
+	if (___p) {								\
+		BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096);		\
+		kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p));	\
+	}									\
+} while (0)
+
 /*
  * Place this after a lock-acquisition primitive to guarantee that
  * an UNLOCK+LOCK pair acts as a full barrier.  This guarantee applies
diff --git a/include/linux/slab.h b/include/linux/slab.h
index e1915026e030..cda126def67a 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -1430,25 +1430,15 @@ extern void kvfree_sensitive(const void *addr, size_t len);
 unsigned int kmem_cache_size(struct kmem_cache *s);
 
 #ifndef CONFIG_KVFREE_RCU_BATCHED
-static inline void kvfree_rcu_barrier(void)
-{
-	rcu_barrier();
-}
-
-static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
-{
-	rcu_barrier();
-}
-
 static inline void kfree_rcu_scheduler_running(void) { }
 #else
+void kfree_rcu_scheduler_running(void);
+#endif
+
 void kvfree_rcu_barrier(void);
 
 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);
 
-void kfree_rcu_scheduler_running(void);
-#endif
-
 /**
  * kmalloc_size_roundup - Report allocation bucket size for the given size
  *
diff --git a/include/linux/types.h b/include/linux/types.h
index 93166b0b0617..7d1d305a763e 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -255,6 +255,16 @@ struct callback_head {
 } __attribute__((aligned(sizeof(void *))));
 #define rcu_head callback_head
 
+#ifdef CONFIG_KVFREE_RCU_BATCHED
+struct kvfree_rcu_head {
+	struct kvfree_rcu_head *next;
+};
+#else
+struct kvfree_rcu_head {
+	struct rcu_head head;
+};
+#endif
+
 typedef void (*rcu_callback_t)(struct rcu_head *head);
 typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func);