Merge tag 'slab-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab

Pull slab updates from Vlastimil Babka:

 - Add kfree_rcu_nolock() that can be used from contexts where spinning
   on a lock might be unsafe, such as a BPF program attached to an
   arbitrary function, or in NMI context. This complements the existing
   kfree_nolock() support (Harry Yoo)

 - Runtime instead of compile-time slabobj_ext sizing.

   Avoid wasting memory when memory allocation profiling is compiled but
   not enabled, with initial partial support to also avoid wasting
   memory for objcg pointers when those are not needed, while profiling
   is enabled (Vlastimil Babka)

 - Various non-urgent fixes, cleanups and optimizations (Hao Li,
   Hongling Zeng, Li RongQing, Li Xiasong, Seongjun Hong, Shengming Hu)

* tag 'slab-for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab: (31 commits)
  mm/slab, kfence, memcg: completely remove obj_ext for kfence objects
  mm/slab: stop allocating objcg pointers when unnecessary
  mm/slab: add cache_ and slab_needs_objcg() helpers
  mm/slab: stop exporting kvfree_rcu_barrier[_on_cache]()
  slub_kunit: extend the test for kfree_rcu_nolock()
  mm/slab: introduce kfree_rcu_nolock()
  mm/slab: introduce struct kvfree_rcu_head for kvfree_rcu batching
  mm/slab: reduce slabobj_ext memory with allocation profiling disabled
  mm/slab: introduce slab_obj_ext_has_codetag()
  mm/slab: allow kfree_rcu_sheaf() on PREEMPT_RT
  mm/slab: extend deferred free mechanism to handle rcu sheaves
  mm/slab: use call_rcu() in unknown context if irqs are enabled
  mm/slab: handle the !allow_spin case in kfree_rcu_sheaf()
  mm/slab: change struct slabobj_ext to a union
  mm/slab: replace slab.stride with obj_exts_in_object
  mm/slab: abstract slabobj_ext.ref access
  mm/slab: abstract slabobj_ext.objcg access
  mm/slab: make slab_obj_ext() determine object index
  mm: move struct slabobj_ext to mm/slab.h
  mm/slab: remove objs_per_slab()
  ...

4 files changed, 44 insertions, 30 deletions

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 8170bb8066a2..71d045fe127f 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -1461,19 +1461,6 @@ static inline void mem_cgroup_flush_workqueue(void) { }
 static inline int mem_cgroup_init(void) { return 0; }
 #endif /* CONFIG_MEMCG */
 
-/*
- * Extended information for slab objects stored as an array in page->memcg_data
- * if MEMCG_DATA_OBJEXTS is set.
- */
-struct slabobj_ext {
-#ifdef CONFIG_MEMCG
-	struct obj_cgroup *objcg;
-#endif
-#ifdef CONFIG_MEM_ALLOC_PROFILING
-	union codetag_ref ref;
-#endif
-} __aligned(8);
-
 static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
 {
 	struct mem_cgroup *memcg;
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 9a741ce05885..44c07a66edff 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1107,19 +1107,22 @@ static inline void rcu_read_unlock_migrate(void)
 /*
  * In mm/slab_common.c, no suitable header to include here.
  */
-void kvfree_call_rcu(struct rcu_head *head, void *ptr);
+void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr);
+void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr);
 
 /*
  * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the
  * comment of kfree_rcu() for details.
  */
-#define kvfree_rcu_arg_2(ptr, rhf)					\
+#define kvfree_rcu_arg_2(ptr, kvrhf)					\
 do {									\
 	typeof (ptr) ___p = (ptr);					\
+	struct kvfree_rcu_head *___head;				\
 									\
 	if (___p) {							\
-		BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096);	\
-		kvfree_call_rcu(&((___p)->rhf), (void *) (___p));	\
+		BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096);	\
+		___head = (struct kvfree_rcu_head *) &(___p)->kvrhf;	\
+		kvfree_call_rcu(___head, (void *) (___p));		\
 	}								\
 } while (0)
 
@@ -1131,6 +1134,27 @@ do {								\
 		kvfree_call_rcu(NULL, (void *) (___p));		\
 } while (0)
 
+/**
+ * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context.
+ * @ptr: pointer to kfree for double-argument invocations.
+ * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr.
+ *
+ * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects
+ * without any deferred processing, but may still defer freeing.
+ * Large kmalloc and vmalloc objects are always deferred.
+ *
+ * kfree_rcu_nolock() supports 2-arg variant only.
+ */
+#define kfree_rcu_nolock(ptr, kvrhf)						\
+do {										\
+	typeof (ptr) ___p = (ptr);						\
+										\
+	if (___p) {								\
+		BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096);		\
+		kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p));	\
+	}									\
+} while (0)
+
 /*
  * Place this after a lock-acquisition primitive to guarantee that
  * an UNLOCK+LOCK pair acts as a full barrier.  This guarantee applies
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 32c9f8ed7ae2..cda126def67a 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -45,6 +45,7 @@ enum _slab_flag_bits {
 #endif
 #ifdef CONFIG_MEMCG
 	_SLAB_ACCOUNT,
+	_SLAB_MAY_ACCOUNT,
 #endif
 #ifdef CONFIG_KASAN_GENERIC
 	_SLAB_KASAN,
@@ -204,8 +205,10 @@ enum _slab_flag_bits {
  */
 #ifdef CONFIG_MEMCG
 # define SLAB_ACCOUNT		__SLAB_FLAG_BIT(_SLAB_ACCOUNT)
+# define SLAB_MAY_ACCOUNT	__SLAB_FLAG_BIT(_SLAB_MAY_ACCOUNT)
 #else
 # define SLAB_ACCOUNT		__SLAB_FLAG_UNUSED
+# define SLAB_MAY_ACCOUNT	__SLAB_FLAG_UNUSED
 #endif
 
 #ifdef CONFIG_KASAN_GENERIC
@@ -1427,25 +1430,15 @@ extern void kvfree_sensitive(const void *addr, size_t len);
 unsigned int kmem_cache_size(struct kmem_cache *s);
 
 #ifndef CONFIG_KVFREE_RCU_BATCHED
-static inline void kvfree_rcu_barrier(void)
-{
-	rcu_barrier();
-}
-
-static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
-{
-	rcu_barrier();
-}
-
 static inline void kfree_rcu_scheduler_running(void) { }
 #else
+void kfree_rcu_scheduler_running(void);
+#endif
+
 void kvfree_rcu_barrier(void);
 
 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s);
 
-void kfree_rcu_scheduler_running(void);
-#endif
-
 /**
  * kmalloc_size_roundup - Report allocation bucket size for the given size
  *
diff --git a/include/linux/types.h b/include/linux/types.h
index bc5dda2a3d86..53e0adca4b9f 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -257,6 +257,16 @@ struct callback_head {
 } __attribute__((aligned(sizeof(void *))));
 #define rcu_head callback_head
 
+#ifdef CONFIG_KVFREE_RCU_BATCHED
+struct kvfree_rcu_head {
+	struct kvfree_rcu_head *next;
+};
+#else
+struct kvfree_rcu_head {
+	struct rcu_head head;
+};
+#endif
+
 typedef void (*rcu_callback_t)(struct rcu_head *head);
 typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func);