summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-08-27 09:17:06 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-08-27 09:17:06 -0700
commit18fbf5151d2c0bfe433c7428eef03cabf5fdb2fa (patch)
tree6e151850bf46f900bbf9dcc682e0c75d91924865 /include
parent5e6ff28676dd92a608eb00eeb8d1319ad34024dc (diff)
parent0685630fdccb62dcb0e3f44525a40578da5f6dc8 (diff)
Merge tag 'mm-stable-2026-08-26-15-22' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull more MM updates from Andrew Morton: - "mm/rmap: index MAP_PRIVATE file-backed folios by anonymous pgoff" (Lorenzo Stoakes) Index MAP_PRIVATE file-backed folios by their anonymous page offset to resolve confusion around reverse mapping for zeroed and CoW'd file-backed memory. Use this new VMA anonymous page offset tracking to eliminate index conflicts and lay the foundation for scalable CoW performance improvements. - "promote mapped executable folios after first usage for MGLRU" (Baolin Wang) Make MGLRU's protection of mapped executable file folios more reliable. Follow the classical LRU's logic, promoting mapped executable file folios after their first usage to give executable code a better chance to stay in memory and improve workload performance. - "mm: vmscan: fix node reclaim ignoring swappiness parameter" (Ridong Chen) Fix per-node proactive reclaim interface's ignoring the swappiness parameter when CONFIG_MEMCG is disabled by consolidating sc_swappiness() into a single function that checks proactive_swappiness regardless of kernel configuration. - "mm/vmscan: reduce lru_lock contention via vmstat-derived scan-balance cost" (Usama Arif) Reduce lru_lock contention in the reclaim path by deriving scan-balance costs from vmstat counters rather than lock-acquired producer updates. Read and decay these cost signals on the reclaim side under a dedicated per-lruvec lock, reducing total LRU lock wait time by over 60% without impacting scan throughput. - "zram: fix zram issues reported by sashiko" (Sergey Senozhatsky) Fix two low-risk zram bugs which Sashiko spotted in drive-by review. - "Honor XA_FLAGS_ACCOUNT in xas_split_alloc() and charge to folio's memcg" (Zi Yan) Fix xas_split_alloc() by enabling target folio memcg charging during splits and adding the missing __GFP_ACCOUNT flag for proper XArray node memory accounting. - "selftests/mm: use pattern matching in .gitignore" (Pratyush Mallick) Replace hardcoded binary names in selftests/mm/.gitignore with a generic pattern-matching rule to automatically ignore generated test files and avoid manual updates when adding new tests. - "mm/page_ext: remove pgdat_page_ext_init()" (Sang-Heon Jeon) Make the incompatibility between FLATMEM and NUMA explicit in mm/Kconfig and remove the unused pgdat_page_ext_init() function. - "zram: fix zstd error paths and add parameter validation" (Haoqin Huang) Clean up zram compression backends by removing redundant error cleanup, adding parameter and dictionary validation, auto-prefixing algorithm error logs, and resetting parameters prior to reinitialization. - "zram: fix stale scan bounds after reinitialization" (Longlong Xia) Prevent out-of-bounds slot accesses during concurrent zram resets by moving table scan bound calculations under dev_lock in writeback_store() and read_block_state(). - "add anon mTHP collapse test cases" (Baolin Wang) Extend selftests helper functions to support arbitrary page orders and add new test cases and options for mTHP collapse in khugepaged. - "selftests/mm: Handle unsupported and transient test conditions" (Muhammad Usama Anjum) Update MM selftests to report a SKIP status instead of a failure when required kernel or filesystem features are unsupported, while adding retry logic for transient page migration errors. - "mm/zswap: Fixes and improves the zswap shrink" (Hao Jia) Fix the missing zswap global shrinker when CONFIG_MEMCG is disabled and extend shrink_memcg() to support batch writeback for improved writeback efficiency. - "alloc_tag: introduce IOCTL-based filtering for MAP" (Suren Baghdasaryan) Introduce an IOCTL-based binary interface for memory allocation profiling that enables kernel-side filtering before per-CPU counter aggregation. This eliminates the text-parsing overhead of /proc/allocinfo and provides up to a 20x speedup by transferring only filtered allocation data to userspace. - "better block swap batching and a different take on swap_ops v5" (Christoph Hellwig) Refactor block swap I/O to use swap_iocb for batching instead of single-bio requests and rebase the swap_ops interface, achieving faster swap throughput during kernel builds. - "mm: kmemleak: reduce transient false positives by confirming leaks" (Catalin Marinas) Reduce false-positive kmemleak reports by combining two kmemleak enhancements that add a second confirmation scan and a configurable minimum unreferenced scan count module parameter. - "mm: kmemleak: default min_unref_scans to 2 for verbose kernels" (Breno Leitao) Auto-scanning kernels can generate false-positive memory leak reports on single scans, so this patch defaults min_unref_scans to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled to require a second confirming scan. - "swap_ops updates" (Christoph Hellwig) Batching I/O for synchronous swap devices causes performance regressions and filesystem-based swap suffers from double-indirection overhead. This series resolves both issues by reintroducing per-folio writes for synchronous swap and allowing filesystems to directly export their own swap_ops. - "mm/khugepaged: several cleanups" (Nico Pache) khugepaged accumulated redundant state-checking patterns and outdated comments following mTHP integration. Introduce dedicated helpers for PTE validation and event counting while refreshing the internal documentation. - "maple_tree: lock checking and clean ups" (Liam Howlett) Syzbot reports incorrectly blame memory management exit paths for locking bugs, maple tree erase operations risk allocation failures without gfp flags and internal documentation lacks clarity. Improve lock error detection, update docs, fix race and allocation edge cases and optimize erase allocations using a fallback to GFP_KERNEL | GFP_NOFAIL. * tag 'mm-stable-2026-08-26-15-22' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (172 commits) selftests/proc: make proc-maps-race work with READ_IMPLIES_EXEC memcg: move LRU size accounting on reparenting instead of copying it mm/vmscan: fix comment logic in balance_pgdat maple_tree: add helper mas_make_walkable() maple_tree: avoid extra gap calculation maple_tree: fix argument name in header maple_tree: change two GFP flags in tests maple_tree: document erase and allocations better maple_tree: avoid mas_erase() and mtree_erase() failures maple_tree: document that erase may use GFP_KERNEL for allocations maple_tree: catch race in mas_alloc_cyclic() maple_tree: add bulk parent set helper maple_tree: micro optimisation of mas_wr_store_type() maple_tree: optimise mas_wr_node_store() when not in rcu mode maple_tree: use prefetched value in mas_wr_store_type() maple_tree: clarify comments on mas_nomem() maple_tree: drop MAPLE_ALLOC_SLOTS maple_tree: drop dead code from mas_extend_spanning_null() maple_tree: documentation fix maple_tree: add write lock checking with lockdep sequence numbers ...
Diffstat (limited to 'include')
-rw-r--r--include/linux/codetag.h2
-rw-r--r--include/linux/execmem.h2
-rw-r--r--include/linux/fs.h1
-rw-r--r--include/linux/lockdep.h3
-rw-r--r--include/linux/lockdep_types.h3
-rw-r--r--include/linux/maple_tree.h16
-rw-r--r--include/linux/memcontrol.h21
-rw-r--r--include/linux/mm.h132
-rw-r--r--include/linux/mm_types.h40
-rw-r--r--include/linux/mmzone.h16
-rw-r--r--include/linux/nfs_fs.h1
-rw-r--r--include/linux/page_ext.h5
-rw-r--r--include/linux/pagemap.h40
-rw-r--r--include/linux/percpu-defs.h7
-rw-r--r--include/linux/rmap.h11
-rw-r--r--include/linux/sched.h1
-rw-r--r--include/linux/shmem_fs.h5
-rw-r--r--include/linux/swap.h30
-rw-r--r--include/linux/swap_ops.h44
-rw-r--r--include/linux/swapops.h7
-rw-r--r--include/linux/vm_event_item.h4
-rw-r--r--include/linux/vmalloc.h2
-rw-r--r--include/linux/vmstat.h17
-rw-r--r--include/rv/da_monitor.h2
-rw-r--r--include/uapi/linux/alloc_tag.h99
25 files changed, 422 insertions, 89 deletions
diff --git a/include/linux/codetag.h b/include/linux/codetag.h
index ddae7484ca45..a25a085c2df1 100644
--- a/include/linux/codetag.h
+++ b/include/linux/codetag.h
@@ -77,6 +77,8 @@ struct codetag_iterator {
void codetag_lock_module_list(struct codetag_type *cttype);
bool codetag_trylock_module_list(struct codetag_type *cttype);
void codetag_unlock_module_list(struct codetag_type *cttype);
+unsigned long codetag_get_content_id(struct codetag_type *cttype);
+unsigned int codetag_get_count(struct codetag_type *cttype);
struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);
struct codetag *codetag_next_ct(struct codetag_iterator *iter);
diff --git a/include/linux/execmem.h b/include/linux/execmem.h
index 7de229134e30..1bd34925d1aa 100644
--- a/include/linux/execmem.h
+++ b/include/linux/execmem.h
@@ -89,7 +89,7 @@ static inline int execmem_restore_rox(void *ptr, size_t size) { return 0; }
* @end: address space end (inclusive)
* @fallback_start: start of the secondary address space range for fallback
* allocations on architectures that require it
- * @fallback_end: start of the secondary address space (inclusive)
+ * @fallback_end: end of the secondary address space (inclusive)
* @pgprot: permissions for memory in this address space
* @alignment: alignment required for text allocations
* @flags: options for memory allocations for this range
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 072d8cd09a0b..f9d1e05e8ae6 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -438,7 +438,6 @@ struct address_space_operations {
int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
sector_t *span);
void (*swap_deactivate)(struct file *file);
- int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
};
extern const struct address_space_operations empty_aops;
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index 621566345406..a6451ecbbe9a 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock);
extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);
extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);
+extern u32 lock_sequence(struct lockdep_map *lock);
+#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map)
+
#define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0)
#define lockdep_assert(cond) \
diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h
index eae115a26488..55c4b152fedf 100644
--- a/include/linux/lockdep_types.h
+++ b/include/linux/lockdep_types.h
@@ -253,7 +253,8 @@ struct held_lock {
unsigned int hardirqs_off:1;
unsigned int sync:1;
unsigned int references:11; /* 32 bits */
- unsigned int pin_count;
+ unsigned int pin_count:24;
+ unsigned int seq_count:8;
};
#else /* !CONFIG_LOCKDEP */
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 4a5631906aff..e595ae5cd0ee 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -11,7 +11,6 @@
#include <linux/kernel.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
-/* #define CONFIG_MAPLE_RCU_DISABLED */
/*
* Allocated nodes are mutable until they have been inserted into the tree,
@@ -29,13 +28,11 @@
#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */
#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */
#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */
-#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1)
#else
/* 32bit sizes */
#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */
#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */
#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */
-#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2)
#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */
#define MAPLE_NODE_MASK 255UL
@@ -485,6 +482,12 @@ struct ma_state {
unsigned char mas_flags;
unsigned char end; /* The end of the node */
enum store_type store_type; /* The type of store needed for this operation */
+#ifdef CONFIG_LOCKDEP
+ u32 ld_seq;
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+ unsigned long rcu_gp;
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+#endif /* CONFIG_LOCKDEP */
};
struct ma_wr_state {
@@ -567,12 +570,14 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp,
unsigned long *next, gfp_t gfp);
bool mas_nomem(struct ma_state *mas, gfp_t gfp);
+bool mas_nomem_nofail(struct ma_state *mas, unsigned long index,
+ unsigned long last);
void mas_pause(struct ma_state *mas);
void maple_tree_init(void);
void mas_destroy(struct ma_state *mas);
void *mas_prev(struct ma_state *mas, unsigned long min);
-void *mas_prev_range(struct ma_state *mas, unsigned long max);
+void *mas_prev_range(struct ma_state *mas, unsigned long min);
void *mas_next(struct ma_state *mas, unsigned long max);
void *mas_next_range(struct ma_state *mas, unsigned long max);
@@ -864,9 +869,6 @@ static inline void mt_init(struct maple_tree *mt)
static inline bool mt_in_rcu(struct maple_tree *mt)
{
-#ifdef CONFIG_MAPLE_RCU_DISABLED
- return false;
-#endif
return mt->ma_flags & MT_FLAGS_USE_RCU;
}
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 71d045fe127f..7d1c0ce189a8 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -239,8 +239,6 @@ struct mem_cgroup {
*/
bool oom_group;
- int swappiness;
-
/* memory.events and memory.events.local */
struct cgroup_file events_file;
struct cgroup_file events_local_file;
@@ -270,10 +268,15 @@ struct mem_cgroup {
#endif
int kmemcg_id;
- struct memcg_vmstats_percpu __percpu *vmstats_percpu;
-
#ifdef CONFIG_CGROUP_WRITEBACK
struct list_head cgwb_list;
+#endif
+
+ /* Keep the hot per-CPU stats pointer away from memory event counters. */
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu
+ ____cacheline_aligned_in_smp;
+
+#ifdef CONFIG_CGROUP_WRITEBACK
struct wb_domain cgwb_domain;
struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
#endif
@@ -318,6 +321,8 @@ struct mem_cgroup {
/* List of events which userspace want to receive */
struct list_head event_list;
spinlock_t event_list_lock;
+
+ int swappiness;
#endif /* CONFIG_MEMCG_V1 */
struct mem_cgroup_per_node *nodeinfo[];
@@ -947,6 +952,8 @@ unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item);
bool memcg_stat_item_valid(int idx);
bool memcg_vm_event_item_valid(enum vm_event_item idx);
unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
+unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+ enum node_stat_item idx);
unsigned long lruvec_page_state_local(struct lruvec *lruvec,
enum node_stat_item idx);
@@ -1399,6 +1406,12 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
return node_page_state(lruvec_pgdat(lruvec), idx);
}
+static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+ enum node_stat_item idx)
+{
+ return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
+}
+
static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
enum node_stat_item idx)
{
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 87feaa5a2b78..dd09c438fa23 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1556,7 +1556,7 @@ static inline void vma_desc_set_anonymous(struct vm_area_desc *desc)
desc->vm_ops = NULL;
}
-static inline bool vma_is_anonymous(struct vm_area_struct *vma)
+static inline bool vma_is_anonymous(const struct vm_area_struct *vma)
{
return !vma->vm_ops;
}
@@ -2271,17 +2271,76 @@ void unpin_user_pages(struct page **pages, unsigned long npages);
void unpin_user_folio(struct folio *folio, unsigned long npages);
void unpin_folios(struct folio **folios, unsigned long nfolios);
-static inline bool is_cow_mapping(vm_flags_t flags)
+/**
+ * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping?
+ * @flags: The VMA flags to check.
+ *
+ * Mappings which could be CoW'd (subject to Copy-On-Write faults) are
+ * described as CoW mappings.
+ *
+ * All mappings backed by anonymous folios (all anonymous mappings and most
+ * MAP_PRIVATE-file backed ranges) are CoW mappings.
+ *
+ * All other mappings (including all MAP_SHARED mappings) are non-CoW.
+ *
+ * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT.
+ *
+ * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both
+ * future mprotect() calls which can render a read-only mapping writable, and
+ * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping.
+ *
+ * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT.
+ *
+ * - Writes to anonymous mappings do not immediately result in CoW faults but
+ * may do so after the process is forked or if a read is followed by a
+ * write.
+ *
+ * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may
+ * do so again after fork.
+ *
+ * - MAP_SHARED mappings of a file opened read-only are transformed into
+ * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain
+ * non-CoW.
+ *
+ * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot
+ * mark themselves anonymous. Having cleared this flag it is not valid for
+ * them to leave the VMA_WRITE_BIT flag set.
+ *
+ * As a consequence, the anonymous reverse mapping only tracks CoW mappings.
+ *
+ * Returns: true if the flags indicate a CoW mapping, otherwise false.
+ */
+static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags)
{
- return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
+ return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
+ !vma_flags_test(flags, VMA_SHARED_BIT);
}
-static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)
+/**
+ * vma_is_cow_mapping() - Is this VMA a CoW mapping?
+ * @vma: The VMA to check.
+ *
+ * See vma_flags_is_cow_mapping() for details.
+ *
+ * Returns: true if the VMA is a CoW mapping, otherwise false.
+ */
+static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma)
{
- const vma_flags_t *flags = &desc->vma_flags;
+ return vma_flags_is_cow_mapping(&vma->flags);
+}
- return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
- !vma_flags_test(flags, VMA_SHARED_BIT);
+/**
+ * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping?
+ * @desc: The VMA descriptor to check.
+ *
+ * See vma_flags_is_cow_mapping() for details.
+ *
+ * Returns: true if the VMA descriptor describes a CoW mapping, otherwise
+ * false.
+ */
+static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)
+{
+ return vma_flags_is_cow_mapping(&desc->vma_flags);
}
#ifndef CONFIG_MMU
@@ -4393,6 +4452,65 @@ static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma)
return vma_end_pgoff(vma) - 1;
}
+/**
+ * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma
+ * @vma: The VMA whose anonymous page offset is required.
+ *
+ * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the
+ * anonymous page offset at the time of first fault.
+ *
+ * If the VMA is anonymous, this returns the same value as vma_start_pgoff().
+ *
+ * This value is used for tracking MAP_PRIVATE file-backed mappings by their
+ * anonymous page offset.
+ *
+ * Returns: The anonymous page offset of the start of @vma.
+ */
+static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma)
+{
+ pgoff_t pgoff = 0;
+
+#ifdef CONFIG_64BIT
+ pgoff += vma->__vm_anon_pgoff_hi;
+ pgoff <<= 32;
+#endif
+ pgoff += vma->__vm_anon_pgoff_lo;
+ return pgoff;
+}
+
+/**
+ * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of
+ * @vma.
+ * @vma: The VMA whose end anonymous page offset is required.
+ *
+ * This returns the anonymous exclusive end page offset of @vma, which is useful
+ * for expressing page offset ranges.
+ *
+ * See the description of vma_start_anon_pgoff() for a description of VMA
+ * anonymous page offsets.
+ *
+ * Returns: The exclusive end anonymous page offset of @vma.
+ */
+static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma)
+{
+ return vma_start_anon_pgoff(vma) + vma_pages(vma);
+}
+
+/**
+ * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in
+ * @vma.
+ * @vma: The VMA whose last anonymous page offset is required.
+ *
+ * See the description of vma_start_anon_pgoff() for a description of VMA
+ * anonymous page offsets.
+ *
+ * Returns: The last anonymous page offset of @vma.
+ */
+static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma)
+{
+ return vma_end_anon_pgoff(vma) - 1;
+}
+
static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)
{
return desc->end - desc->start;
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 939b5ea8c9e0..6d815f6440c9 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -968,6 +968,11 @@ struct vm_area_struct {
unsigned int vm_lock_seq;
#endif
/*
+ * Low 32-bits of anonymous page offset.
+ * See vma_start_anon_pgoff() comment for details.
+ */
+ unsigned int __vm_anon_pgoff_lo;
+ /*
* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
* list, after a COW of one of the file pages. A MAP_SHARED vma
* can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack
@@ -1042,6 +1047,13 @@ struct vm_area_struct {
struct lockdep_map vmlock_dep_map;
#endif
#endif
+#ifdef CONFIG_64BIT
+ /*
+ * High 32-bits of anonymous page offset.
+ * See vma_start_anon_pgoff() comment for details.
+ */
+ unsigned int __vm_anon_pgoff_hi;
+#endif
/*
* For areas with an address space and backing store,
* linkage into the address_space->i_mmap interval tree.
@@ -1706,20 +1718,20 @@ enum vm_fault_reason {
VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON | \
VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)
-#define VM_FAULT_RESULT_TRACE \
- { VM_FAULT_OOM, "OOM" }, \
- { VM_FAULT_SIGBUS, "SIGBUS" }, \
- { VM_FAULT_MAJOR, "MAJOR" }, \
- { VM_FAULT_HWPOISON, "HWPOISON" }, \
- { VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \
- { VM_FAULT_SIGSEGV, "SIGSEGV" }, \
- { VM_FAULT_NOPAGE, "NOPAGE" }, \
- { VM_FAULT_LOCKED, "LOCKED" }, \
- { VM_FAULT_RETRY, "RETRY" }, \
- { VM_FAULT_FALLBACK, "FALLBACK" }, \
- { VM_FAULT_DONE_COW, "DONE_COW" }, \
- { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \
- { VM_FAULT_COMPLETED, "COMPLETED" }
+#define VM_FAULT_RESULT_TRACE \
+ { (__force u32)VM_FAULT_OOM, "OOM" }, \
+ { (__force u32)VM_FAULT_SIGBUS, "SIGBUS" }, \
+ { (__force u32)VM_FAULT_MAJOR, "MAJOR" }, \
+ { (__force u32)VM_FAULT_HWPOISON, "HWPOISON" }, \
+ { (__force u32)VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \
+ { (__force u32)VM_FAULT_SIGSEGV, "SIGSEGV" }, \
+ { (__force u32)VM_FAULT_NOPAGE, "NOPAGE" }, \
+ { (__force u32)VM_FAULT_LOCKED, "LOCKED" }, \
+ { (__force u32)VM_FAULT_RETRY, "RETRY" }, \
+ { (__force u32)VM_FAULT_FALLBACK, "FALLBACK" }, \
+ { (__force u32)VM_FAULT_DONE_COW, "DONE_COW" }, \
+ { (__force u32)VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \
+ { (__force u32)VM_FAULT_COMPLETED, "COMPLETED" }
struct vm_special_mapping {
const char *name; /* The name, e.g. "[vdso]". */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 0507193b3ae3..94f9c3ff5416 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -323,6 +323,8 @@ enum node_stat_item {
PGSCAN_PROACTIVE,
PGSCAN_ANON,
PGSCAN_FILE,
+ PGROTATE_ANON,
+ PGROTATE_FILE,
PGREFILL,
#ifdef CONFIG_HUGETLB_PAGE
NR_HUGETLB,
@@ -755,6 +757,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
#endif /* CONFIG_LRU_GEN */
+struct lru_cost {
+ unsigned long count;
+ unsigned long last_rotated;
+ unsigned long last_io;
+};
+
struct lruvec {
struct list_head lists[NR_LRU_LISTS];
/* per lruvec lru_lock for memcg */
@@ -763,9 +771,12 @@ struct lruvec {
* These track the cost of reclaiming one LRU - file or anon -
* over the other. As the observed cost of reclaiming one LRU
* increases, the reclaim scan balance tips toward the other.
+ * Updated and decayed at prepare_scan_control() time; cost_lock
+ * serialises that update.
*/
- unsigned long anon_cost;
- unsigned long file_cost;
+ struct lru_cost cost[ANON_AND_FILE];
+ /* Protects cost[]. */
+ spinlock_t cost_lock;
/* Non-resident age, driven by LRU movement */
atomic_long_t nonresident_age;
/* Refaults at the time of last reclaim cycle */
@@ -2058,7 +2069,6 @@ static inline struct mem_section *__nr_to_section(unsigned long nr)
#endif
return &mem_section[root][nr & SECTION_ROOT_MASK];
}
-extern size_t mem_section_usage_size(void);
/*
* We use the lower bits of the mem_map pointer to store a little bit of
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 48b806aa3a2f..b85a73ae7919 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -552,7 +552,6 @@ static inline const struct cred *nfs_file_cred(struct file *file)
/*
* linux/fs/nfs/direct.c
*/
-int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter);
ssize_t nfs_file_direct_read(struct kiocb *iocb,
struct iov_iter *iter, bool swap);
ssize_t nfs_file_direct_write(struct kiocb *iocb,
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index f23d4b218da0..79c53ec45dfa 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -55,7 +55,6 @@ struct page_ext {
extern bool early_page_ext;
extern unsigned long page_ext_size;
-extern void pgdat_page_ext_init(struct pglist_data *pgdat);
static inline bool early_page_ext_enabled(void)
{
@@ -202,10 +201,6 @@ static inline bool early_page_ext_enabled(void)
return false;
}
-static inline void pgdat_page_ext_init(struct pglist_data *pgdat)
-{
-}
-
static inline void page_ext_init(void)
{
}
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index c6fc783aaee5..0adfa6605653 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -1094,10 +1094,44 @@ static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma,
static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,
const unsigned long address)
{
- pgoff_t pgoff;
+ return linear_page_delta(vma, address) + vma_start_pgoff(vma);
+}
+
+static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma,
+ const unsigned long address)
+{
+ return linear_page_delta(vma, address) + vma_start_anon_pgoff(vma);
+}
+
+/**
+ * linear_anon_page_index() - Determine the absolute anonymous page offset of
+ * @address within @vma.
+ * @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides.
+ * @address: The address whose absolute page offset is required.
+ *
+ * This returns the anonymous page offset of @address, which is the page offset
+ * the address possessed at the time the VMA was first faulted.
+ *
+ * For anonymous mappings, this returns the same value as linear_page_index().
+ *
+ * For MAP_PRIVATE file-backed mappings, this returns the anonymous page offset
+ * of @address, which is the page offset the address possessed at the time the
+ * VMA was first faulted.
+ *
+ * It is not valid to call this function for shared file-backed mappings.
+ *
+ * Returns: The absolute anonymous page offset of @address within @vma.
+ */
+static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma,
+ const unsigned long address)
+{
+ const pgoff_t pgoff = __linear_anon_page_index(vma, address);
+
+ VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));
+ /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */
+ if (vma_is_anonymous(vma) && !vma->vm_file)
+ VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address));
- pgoff = linear_page_delta(vma, address);
- pgoff += vma_start_pgoff(vma);
return pgoff;
}
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 2cba7cc2b01f..dbe3267a0a13 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -65,13 +65,8 @@
*
* Archs which need weak percpu definitions should set
* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU when necessary.
- *
- * To ensure that the generic code observes the above two
- * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
- * definition is used for all cases.
*/
-#if (defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)) || \
- defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
+#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)
/*
* __pcpu_scope_* dummy variable is used to enforce scope. It
* receives the static modifier when it's used in front of
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 8dc0871e5f00..0b332770abee 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -843,7 +843,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio,
* Called from mm/vmscan.c to handle paging out
*/
int folio_referenced(struct folio *, int is_locked,
- struct mem_cgroup *memcg, vm_flags_t *vm_flags);
+ struct mem_cgroup *memcg, vma_flags_t *vma_flags);
void try_to_migrate(struct folio *folio, enum ttu_flags flags);
void try_to_unmap(struct folio *, enum ttu_flags flags);
@@ -864,13 +864,14 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr,
struct page_vma_mapped_walk {
unsigned long pfn;
unsigned long nr_pages;
- pgoff_t pgoff;
+ pgoff_t pgoff; /* Only meaningful if nr_pages > 1 and not a KSM walk */
struct vm_area_struct *vma;
unsigned long address;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
unsigned int flags;
+ bool pgoff_is_anon : 1;
};
#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
@@ -881,6 +882,7 @@ struct page_vma_mapped_walk {
.vma = _vma, \
.address = _address, \
.flags = _flags, \
+ .pgoff_is_anon = folio_test_anon(_folio), \
}
static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
@@ -975,10 +977,9 @@ struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
#define anon_vma_prepare(vma) (0)
static inline int folio_referenced(struct folio *folio, int is_locked,
- struct mem_cgroup *memcg,
- vm_flags_t *vm_flags)
+ struct mem_cgroup *memcg, vma_flags_t *vma_flags)
{
- *vm_flags = 0;
+ vma_flags_clear_all(vma_flags);
return 0;
}
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c7cdbb3e4aa6..8b3d47a325cc 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1298,6 +1298,7 @@ struct task_struct {
u64 curr_chain_key;
int lockdep_depth;
unsigned int lockdep_recursion;
+ unsigned int lockdep_seq;
struct held_lock held_locks[MAX_LOCK_DEPTH];
#endif
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index e729b9b0e38d..5663dff53186 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -12,8 +12,6 @@
#include <linux/userfaultfd_k.h>
#include <linux/bits.h>
-struct swap_iocb;
-
/* inode in-kernel data */
#ifdef CONFIG_TMPFS_QUOTA
@@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)
void shmem_unlock_mapping(struct address_space *mapping);
struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask);
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
- struct list_head *folio_list);
+int shmem_write_folio(struct folio *folio);
void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
int shmem_unuse(unsigned int type);
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 696ed01709c2..5658a1634b85 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -202,7 +202,6 @@ enum {
SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */
SWP_BLKDEV = (1 << 6), /* its a block device */
SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */
- SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */
SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */
SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
@@ -276,6 +275,7 @@ struct swap_info_struct {
struct work_struct reclaim_work; /* reclaim worker */
struct list_head discard_clusters; /* discard clusters list */
struct plist_node avail_list; /* entry in swap_avail_head */
+ const struct swap_ops *ops;
};
static inline swp_entry_t page_swap_entry(struct page *page)
@@ -298,6 +298,14 @@ void folio_add_lru(struct folio *folio);
void folio_mark_accessed(struct folio *folio);
void lru_add_drain_all(void);
+enum lru_cache_drained {
+ LRU_CACHE_NOT_DRAINED,
+ LRU_CACHE_DRAINED,
+ LRU_CACHE_DRAINED_ALL,
+};
+void lru_cache_drain_for_folio(const struct folio *folio,
+ unsigned int extra_refs, enum lru_cache_drained *drained);
+
/* linux/mm/folio-compat.c */
void mark_page_accessed(struct page *page);
@@ -309,7 +317,6 @@ static inline bool lru_cache_disabled(void)
}
extern unsigned long shrink_all_memory(unsigned long nr_pages);
-extern int vm_swappiness;
long remove_mapping(struct address_space *mapping, struct folio *folio);
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
@@ -334,7 +341,6 @@ extern void __meminit kswapd_run(int nid);
extern void __meminit kswapd_stop(int nid);
#ifdef CONFIG_SWAP
-
int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
unsigned long nr_pages, sector_t start_block);
int generic_swapfile_activate(struct swap_info_struct *, struct file *,
@@ -468,25 +474,7 @@ static inline int add_swap_extent(struct swap_info_struct *sis,
}
#endif /* CONFIG_SWAP */
#ifdef CONFIG_MEMCG
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
-{
- /* Cgroup2 doesn't have per-cgroup swappiness */
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
- return READ_ONCE(vm_swappiness);
-
- /* root ? */
- if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
- return READ_ONCE(vm_swappiness);
-
- return READ_ONCE(memcg->swappiness);
-}
-
void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
-#else
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
-{
- return READ_ONCE(vm_swappiness);
-}
#endif
#if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
diff --git a/include/linux/swap_ops.h b/include/linux/swap_ops.h
new file mode 100644
index 000000000000..57ac6c703f68
--- /dev/null
+++ b/include/linux/swap_ops.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MM_SWAP_OPS_H
+#define _MM_SWAP_OPS_H
+
+#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */
+
+struct swap_iocb {
+ union {
+ struct kiocb iocb;
+ struct bio bio;
+ };
+ struct bio_vec bvecs[SWAP_CLUSTER_MAX];
+ int nr_bvecs;
+ int len;
+};
+
+struct swap_io_ctx {
+ struct swap_iocb *sio;
+ struct swap_info_struct *sis;
+};
+
+/*
+ * SWAP_OPS_F_REQUIRE_NOFS:
+ * When set, all reclaim operations must operated as GFS_NOFS and not
+ * just GFP_NOIO, as GFP_NOIO allocations could recourse into the
+ * file system backing this swap file.
+ */
+#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0)
+
+struct swap_ops {
+ unsigned int flags;
+
+ bool (*can_merge)(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size, int rw);
+ void (*submit_write)(struct swap_io_ctx *ctx);
+ void (*submit_read)(struct swap_io_ctx *ctx);
+};
+
+void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter);
+bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size, int rw);
+int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops);
+
+#endif /* _MM_SWAP_OPS_H */
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index c956bc445ee0..e7d0d529f3e0 100644
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -5,6 +5,7 @@
#include <linux/radix-tree.h>
#include <linux/bug.h>
#include <linux/mm_types.h>
+#include <linux/swap.h>
#ifdef CONFIG_MMU
@@ -325,8 +326,8 @@ struct page_vma_mapped_walk;
extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
struct page *page);
-extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
- struct page *new);
+void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
+ struct folio *folio);
extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);
@@ -346,7 +347,7 @@ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
}
static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
- struct page *new)
+ struct folio *folio)
{
BUILD_BUG();
}
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 03fe95f5a020..2628ccda076a 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
KSTACK_REST,
#endif
#endif /* CONFIG_DEBUG_STACK_USAGE */
+#ifdef CONFIG_SWAP
+ NRSWPIN,
+ NRSWPOUT,
+#endif /* CONFIG_SWAP */
NR_VM_EVENT_ITEMS
};
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index e4d8d0a9f30f..aed121d729b0 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -62,7 +62,7 @@ struct vm_struct {
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
unsigned int page_order;
#endif
- unsigned int nr_pages;
+ unsigned long nr_pages;
phys_addr_t phys_addr;
const void *caller;
unsigned long requested_size;
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 3c9c266cf782..5b31d8e7ae40 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -20,7 +20,6 @@ struct reclaim_stat {
unsigned nr_congested;
unsigned nr_writeback;
unsigned nr_immediate;
- unsigned nr_pageout;
unsigned nr_activate[ANON_AND_FILE];
unsigned nr_ref_keep;
unsigned nr_unmap_fail;
@@ -194,6 +193,19 @@ unsigned long global_node_page_state_pages(enum node_stat_item item)
return x;
}
+/*
+ * Non-clamping variant of global_node_page_state() intended for callers that
+ * snapshot a monotonically-incremented counter and subtract two samples.
+ * Returns the raw wrapping value so that unsigned modular subtraction stays
+ * correct across a signed-long overflow (a real hazard on 32-bit) that the
+ * clamp in global_node_page_state() would otherwise turn into a huge spurious
+ * delta. Do NOT use for non-monotonic page-count reads.
+ */
+static inline unsigned long global_node_page_state_monotonic(enum node_stat_item item)
+{
+ return (unsigned long)atomic_long_read(&vm_node_stat[item]);
+}
+
static inline unsigned long global_node_page_state(enum node_stat_item item)
{
VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
@@ -259,11 +271,14 @@ extern unsigned long node_page_state(struct pglist_data *pgdat,
enum node_stat_item item);
extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
enum node_stat_item item);
+extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
+ enum node_stat_item item);
extern void fold_vm_numa_events(void);
#else
#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
#define node_page_state(node, item) global_node_page_state(item)
#define node_page_state_pages(node, item) global_node_page_state_pages(item)
+#define node_page_state_monotonic(node, item) global_node_page_state_monotonic(item)
static inline void fold_vm_numa_events(void)
{
}
diff --git a/include/rv/da_monitor.h b/include/rv/da_monitor.h
index e3cf85c9ce55..6fc5ef8142ac 100644
--- a/include/rv/da_monitor.h
+++ b/include/rv/da_monitor.h
@@ -25,7 +25,7 @@
/*
* Per-cpu variables require a unique name although static in some
- * configurations (e.g. CONFIG_DEBUG_FORCE_WEAK_PER_CPU or alpha modules).
+ * configurations (e.g. alpha modules).
*/
#define DA_MON_NAME CONCATENATE(da_mon_, MONITOR_NAME)
diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h
new file mode 100644
index 000000000000..e3ad94444864
--- /dev/null
+++ b/include/uapi/linux/alloc_tag.h
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * alloc_tag IOCTL API definition
+ *
+ * Copyright (C) 2026 Google, LLC. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _UAPI_ALLOC_TAG_H
+#define _UAPI_ALLOC_TAG_H
+
+#include <linux/types.h>
+
+/*
+ * Function, file and module names often have the same prefixes, therefore
+ * when filtering by these criteria, we compare the last 64 characters to
+ * minimize the chances of name collisions
+ */
+#define ALLOCINFO_STR_SIZE 64
+
+struct allocinfo_content_id {
+ __u64 id;
+};
+
+struct allocinfo_tag {
+ /* Longer names are trimmed */
+ char modname[ALLOCINFO_STR_SIZE];
+ char function[ALLOCINFO_STR_SIZE];
+ char filename[ALLOCINFO_STR_SIZE];
+ __u64 lineno;
+};
+
+/* The alignment ensures 32-bit compatible interfaces are not broken */
+struct allocinfo_counter {
+ __u64 bytes;
+ __u64 calls;
+ __u8 accurate;
+} __attribute__((aligned(8)));
+
+struct allocinfo_tag_data {
+ struct allocinfo_tag tag;
+ struct allocinfo_counter counter;
+};
+
+enum {
+ ALLOCINFO_FILTER_MODNAME,
+ ALLOCINFO_FILTER_FUNCTION,
+ ALLOCINFO_FILTER_FILENAME,
+ ALLOCINFO_FILTER_LINENO,
+ ALLOCINFO_FILTER_INACCURATE,
+ ALLOCINFO_FILTER_MIN_SIZE,
+ ALLOCINFO_FILTER_MAX_SIZE,
+ __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE
+};
+
+#define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME)
+#define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION)
+#define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME)
+#define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO)
+#define ALLOCINFO_FILTER_MASK_INACCURATE (1 << ALLOCINFO_FILTER_INACCURATE)
+#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE)
+#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE)
+
+#define ALLOCINFO_FILTER_MASKS \
+ ((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1)
+
+struct allocinfo_filter {
+ __u64 mask; /* bitmask of the filter fields used */
+ struct allocinfo_tag fields;
+ __u64 min_size;
+ __u64 max_size;
+ /* filter criteria only; see allocinfo_counter.accurate for actual accuracy */
+ __u64 inaccurate;
+};
+
+struct allocinfo_get_at {
+ /* inputs */
+ __u64 pos;
+ struct allocinfo_filter filter;
+ /* output */
+ struct allocinfo_tag_data data;
+};
+
+#define _ALLOCINFO_IOC_CONTENT_ID 0
+#define _ALLOCINFO_IOC_GET_AT 1
+#define _ALLOCINFO_IOC_GET_NEXT 2
+
+#define ALLOCINFO_IOC_BASE 0xA6
+#define ALLOCINFO_IOC_CONTENT_ID _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID, \
+ struct allocinfo_content_id)
+#define ALLOCINFO_IOC_GET_AT _IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT, \
+ struct allocinfo_get_at)
+#define ALLOCINFO_IOC_GET_NEXT _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT, \
+ struct allocinfo_tag_data)
+
+#endif /* _UAPI_ALLOC_TAG_H */