summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
Diffstat (limited to 'include')
-rw-r--r--include/linux/codetag.h2
-rw-r--r--include/linux/execmem.h2
-rw-r--r--include/linux/fs.h1
-rw-r--r--include/linux/lockdep.h3
-rw-r--r--include/linux/lockdep_types.h3
-rw-r--r--include/linux/maple_tree.h16
-rw-r--r--include/linux/memcontrol.h21
-rw-r--r--include/linux/mm.h132
-rw-r--r--include/linux/mm_types.h40
-rw-r--r--include/linux/mmzone.h16
-rw-r--r--include/linux/nfs_fs.h1
-rw-r--r--include/linux/page_ext.h5
-rw-r--r--include/linux/pagemap.h40
-rw-r--r--include/linux/percpu-defs.h7
-rw-r--r--include/linux/rmap.h11
-rw-r--r--include/linux/sched.h1
-rw-r--r--include/linux/shmem_fs.h5
-rw-r--r--include/linux/swap.h30
-rw-r--r--include/linux/swap_ops.h44
-rw-r--r--include/linux/swapops.h7
-rw-r--r--include/linux/vm_event_item.h4
-rw-r--r--include/linux/vmalloc.h2
-rw-r--r--include/linux/vmstat.h17
-rw-r--r--include/rv/da_monitor.h2
-rw-r--r--include/uapi/linux/alloc_tag.h99
25 files changed, 422 insertions, 89 deletions
diff --git a/include/linux/codetag.h b/include/linux/codetag.h
index ddae7484ca45..a25a085c2df1 100644
--- a/include/linux/codetag.h
+++ b/include/linux/codetag.h
@@ -77,6 +77,8 @@ struct codetag_iterator {
void codetag_lock_module_list(struct codetag_type *cttype);
bool codetag_trylock_module_list(struct codetag_type *cttype);
void codetag_unlock_module_list(struct codetag_type *cttype);
+unsigned long codetag_get_content_id(struct codetag_type *cttype);
+unsigned int codetag_get_count(struct codetag_type *cttype);
struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);
struct codetag *codetag_next_ct(struct codetag_iterator *iter);
diff --git a/include/linux/execmem.h b/include/linux/execmem.h
index 7de229134e30..1bd34925d1aa 100644
--- a/include/linux/execmem.h
+++ b/include/linux/execmem.h
@@ -89,7 +89,7 @@ static inline int execmem_restore_rox(void *ptr, size_t size) { return 0; }
* @end: address space end (inclusive)
* @fallback_start: start of the secondary address space range for fallback
* allocations on architectures that require it
- * @fallback_end: start of the secondary address space (inclusive)
+ * @fallback_end: end of the secondary address space (inclusive)
* @pgprot: permissions for memory in this address space
* @alignment: alignment required for text allocations
* @flags: options for memory allocations for this range
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 072d8cd09a0b..f9d1e05e8ae6 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -438,7 +438,6 @@ struct address_space_operations {
int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
sector_t *span);
void (*swap_deactivate)(struct file *file);
- int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
};
extern const struct address_space_operations empty_aops;
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index 621566345406..a6451ecbbe9a 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock);
extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);
extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);
+extern u32 lock_sequence(struct lockdep_map *lock);
+#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map)
+
#define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0)
#define lockdep_assert(cond) \
diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h
index eae115a26488..55c4b152fedf 100644
--- a/include/linux/lockdep_types.h
+++ b/include/linux/lockdep_types.h
@@ -253,7 +253,8 @@ struct held_lock {
unsigned int hardirqs_off:1;
unsigned int sync:1;
unsigned int references:11; /* 32 bits */
- unsigned int pin_count;
+ unsigned int pin_count:24;
+ unsigned int seq_count:8;
};
#else /* !CONFIG_LOCKDEP */
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 4a5631906aff..e595ae5cd0ee 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -11,7 +11,6 @@
#include <linux/kernel.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
-/* #define CONFIG_MAPLE_RCU_DISABLED */
/*
* Allocated nodes are mutable until they have been inserted into the tree,
@@ -29,13 +28,11 @@
#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */
#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */
#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */
-#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1)
#else
/* 32bit sizes */
#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */
#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */
#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */
-#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2)
#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */
#define MAPLE_NODE_MASK 255UL
@@ -485,6 +482,12 @@ struct ma_state {
unsigned char mas_flags;
unsigned char end; /* The end of the node */
enum store_type store_type; /* The type of store needed for this operation */
+#ifdef CONFIG_LOCKDEP
+ u32 ld_seq;
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+ unsigned long rcu_gp;
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+#endif /* CONFIG_LOCKDEP */
};
struct ma_wr_state {
@@ -567,12 +570,14 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp,
unsigned long *next, gfp_t gfp);
bool mas_nomem(struct ma_state *mas, gfp_t gfp);
+bool mas_nomem_nofail(struct ma_state *mas, unsigned long index,
+ unsigned long last);
void mas_pause(struct ma_state *mas);
void maple_tree_init(void);
void mas_destroy(struct ma_state *mas);
void *mas_prev(struct ma_state *mas, unsigned long min);
-void *mas_prev_range(struct ma_state *mas, unsigned long max);
+void *mas_prev_range(struct ma_state *mas, unsigned long min);
void *mas_next(struct ma_state *mas, unsigned long max);
void *mas_next_range(struct ma_state *mas, unsigned long max);
@@ -864,9 +869,6 @@ static inline void mt_init(struct maple_tree *mt)
static inline bool mt_in_rcu(struct maple_tree *mt)
{
-#ifdef CONFIG_MAPLE_RCU_DISABLED
- return false;
-#endif
return mt->ma_flags & MT_FLAGS_USE_RCU;
}
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 71d045fe127f..7d1c0ce189a8 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -239,8 +239,6 @@ struct mem_cgroup {
*/
bool oom_group;
- int swappiness;
-
/* memory.events and memory.events.local */
struct cgroup_file events_file;
struct cgroup_file events_local_file;
@@ -270,10 +268,15 @@ struct mem_cgroup {
#endif
int kmemcg_id;
- struct memcg_vmstats_percpu __percpu *vmstats_percpu;
-
#ifdef CONFIG_CGROUP_WRITEBACK
struct list_head cgwb_list;
+#endif
+
+ /* Keep the hot per-CPU stats pointer away from memory event counters. */
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu
+ ____cacheline_aligned_in_smp;
+
+#ifdef CONFIG_CGROUP_WRITEBACK
struct wb_domain cgwb_domain;
struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
#endif
@@ -318,6 +321,8 @@ struct mem_cgroup {
/* List of events which userspace want to receive */
struct list_head event_list;
spinlock_t event_list_lock;
+
+ int swappiness;
#endif /* CONFIG_MEMCG_V1 */
struct mem_cgroup_per_node *nodeinfo[];
@@ -947,6 +952,8 @@ unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item);
bool memcg_stat_item_valid(int idx);
bool memcg_vm_event_item_valid(enum vm_event_item idx);
unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
+unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+ enum node_stat_item idx);
unsigned long lruvec_page_state_local(struct lruvec *lruvec,
enum node_stat_item idx);
@@ -1399,6 +1406,12 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
return node_page_state(lruvec_pgdat(lruvec), idx);
}
+static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+ enum node_stat_item idx)
+{
+ return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
+}
+
static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
enum node_stat_item idx)
{
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 87feaa5a2b78..dd09c438fa23 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1556,7 +1556,7 @@ static inline void vma_desc_set_anonymous(struct vm_area_desc *desc)
desc->vm_ops = NULL;
}
-static inline bool vma_is_anonymous(struct vm_area_struct *vma)
+static inline bool vma_is_anonymous(const struct vm_area_struct *vma)
{
return !vma->vm_ops;
}
@@ -2271,17 +2271,76 @@ void unpin_user_pages(struct page **pages, unsigned long npages);
void unpin_user_folio(struct folio *folio, unsigned long npages);
void unpin_folios(struct folio **folios, unsigned long nfolios);
-static inline bool is_cow_mapping(vm_flags_t flags)
+/**
+ * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping?
+ * @flags: The VMA flags to check.
+ *
+ * Mappings which could be CoW'd (subject to Copy-On-Write faults) are
+ * described as CoW mappings.
+ *
+ * All mappings backed by anonymous folios (all anonymous mappings and most
+ * MAP_PRIVATE-file backed ranges) are CoW mappings.
+ *
+ * All other mappings (including all MAP_SHARED mappings) are non-CoW.
+ *
+ * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT.
+ *
+ * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both
+ * future mprotect() calls which can render a read-only mapping writable, and
+ * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping.
+ *
+ * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT.
+ *
+ * - Writes to anonymous mappings do not immediately result in CoW faults but
+ * may do so after the process is forked or if a read is followed by a
+ * write.
+ *
+ * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may
+ * do so again after fork.
+ *
+ * - MAP_SHARED mappings of a file opened read-only are transformed into
+ * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain
+ * non-CoW.
+ *
+ * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot
+ * mark themselves anonymous. Having cleared this flag it is not valid for
+ * them to leave the VMA_WRITE_BIT flag set.
+ *
+ * As a consequence, the anonymous reverse mapping only tracks CoW mappings.
+ *
+ * Returns: true if the flags indicate a CoW mapping, otherwise false.
+ */
+static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags)
{
- return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
+ return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
+ !vma_flags_test(flags, VMA_SHARED_BIT);
}
-static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)
+/**
+ * vma_is_cow_mapping() - Is this VMA a CoW mapping?
+ * @vma: The VMA to check.
+ *
+ * See vma_flags_is_cow_mapping() for details.
+ *
+ * Returns: true if the VMA is a CoW mapping, otherwise false.
+ */
+static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma)
{
- const vma_flags_t *flags = &desc->vma_flags;
+ return vma_flags_is_cow_mapping(&vma->flags);
+}
- return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
- !vma_flags_test(flags, VMA_SHARED_BIT);
+/**
+ * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping?
+ * @desc: The VMA descriptor to check.
+ *
+ * See vma_flags_is_cow_mapping() for details.
+ *
+ * Returns: true if the VMA descriptor describes a CoW mapping, otherwise
+ * false.
+ */
+static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)
+{
+ return vma_flags_is_cow_mapping(&desc->vma_flags);
}
#ifndef CONFIG_MMU
@@ -4393,6 +4452,65 @@ static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma)
return vma_end_pgoff(vma) - 1;
}
+/**
+ * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma
+ * @vma: The VMA whose anonymous page offset is required.
+ *
+ * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the
+ * anonymous page offset at the time of first fault.
+ *
+ * If the VMA is anonymous, this returns the same value as vma_start_pgoff().
+ *
+ * This value is used for tracking MAP_PRIVATE file-backed mappings by their
+ * anonymous page offset.
+ *
+ * Returns: The anonymous page offset of the start of @vma.
+ */
+static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma)
+{
+ pgoff_t pgoff = 0;
+
+#ifdef CONFIG_64BIT
+ pgoff += vma->__vm_anon_pgoff_hi;
+ pgoff <<= 32;
+#endif
+ pgoff += vma->__vm_anon_pgoff_lo;
+ return pgoff;
+}
+
+/**
+ * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of
+ * @vma.
+ * @vma: The VMA whose end anonymous page offset is required.
+ *
+ * This returns the anonymous exclusive end page offset of @vma, which is useful
+ * for expressing page offset ranges.
+ *
+ * See the description of vma_start_anon_pgoff() for a description of VMA
+ * anonymous page offsets.
+ *
+ * Returns: The exclusive end anonymous page offset of @vma.
+ */
+static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma)
+{
+ return vma_start_anon_pgoff(vma) + vma_pages(vma);
+}
+
+/**
+ * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in
+ * @vma.
+ * @vma: The VMA whose last anonymous page offset is required.
+ *
+ * See the description of vma_start_anon_pgoff() for a description of VMA
+ * anonymous page offsets.
+ *
+ * Returns: The last anonymous page offset of @vma.
+ */
+static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma)
+{
+ return vma_end_anon_pgoff(vma) - 1;
+}
+
static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)
{
return desc->end - desc->start;
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 939b5ea8c9e0..6d815f6440c9 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -968,6 +968,11 @@ struct vm_area_struct {
unsigned int vm_lock_seq;
#endif
/*
+ * Low 32-bits of anonymous page offset.
+ * See vma_start_anon_pgoff() comment for details.
+ */
+ unsigned int __vm_anon_pgoff_lo;
+ /*
* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
* list, after a COW of one of the file pages. A MAP_SHARED vma
* can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack
@@ -1042,6 +1047,13 @@ struct vm_area_struct {
struct lockdep_map vmlock_dep_map;
#endif
#endif
+#ifdef CONFIG_64BIT
+ /*
+ * High 32-bits of anonymous page offset.
+ * See vma_start_anon_pgoff() comment for details.
+ */
+ unsigned int __vm_anon_pgoff_hi;
+#endif
/*
* For areas with an address space and backing store,
* linkage into the address_space->i_mmap interval tree.
@@ -1706,20 +1718,20 @@ enum vm_fault_reason {
VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON | \
VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)
-#define VM_FAULT_RESULT_TRACE \
- { VM_FAULT_OOM, "OOM" }, \
- { VM_FAULT_SIGBUS, "SIGBUS" }, \
- { VM_FAULT_MAJOR, "MAJOR" }, \
- { VM_FAULT_HWPOISON, "HWPOISON" }, \
- { VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \
- { VM_FAULT_SIGSEGV, "SIGSEGV" }, \
- { VM_FAULT_NOPAGE, "NOPAGE" }, \
- { VM_FAULT_LOCKED, "LOCKED" }, \
- { VM_FAULT_RETRY, "RETRY" }, \
- { VM_FAULT_FALLBACK, "FALLBACK" }, \
- { VM_FAULT_DONE_COW, "DONE_COW" }, \
- { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \
- { VM_FAULT_COMPLETED, "COMPLETED" }
+#define VM_FAULT_RESULT_TRACE \
+ { (__force u32)VM_FAULT_OOM, "OOM" }, \
+ { (__force u32)VM_FAULT_SIGBUS, "SIGBUS" }, \
+ { (__force u32)VM_FAULT_MAJOR, "MAJOR" }, \
+ { (__force u32)VM_FAULT_HWPOISON, "HWPOISON" }, \
+ { (__force u32)VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \
+ { (__force u32)VM_FAULT_SIGSEGV, "SIGSEGV" }, \
+ { (__force u32)VM_FAULT_NOPAGE, "NOPAGE" }, \
+ { (__force u32)VM_FAULT_LOCKED, "LOCKED" }, \
+ { (__force u32)VM_FAULT_RETRY, "RETRY" }, \
+ { (__force u32)VM_FAULT_FALLBACK, "FALLBACK" }, \
+ { (__force u32)VM_FAULT_DONE_COW, "DONE_COW" }, \
+ { (__force u32)VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \
+ { (__force u32)VM_FAULT_COMPLETED, "COMPLETED" }
struct vm_special_mapping {
const char *name; /* The name, e.g. "[vdso]". */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 0507193b3ae3..94f9c3ff5416 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -323,6 +323,8 @@ enum node_stat_item {
PGSCAN_PROACTIVE,
PGSCAN_ANON,
PGSCAN_FILE,
+ PGROTATE_ANON,
+ PGROTATE_FILE,
PGREFILL,
#ifdef CONFIG_HUGETLB_PAGE
NR_HUGETLB,
@@ -755,6 +757,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent,
#endif /* CONFIG_LRU_GEN */
+struct lru_cost {
+ unsigned long count;
+ unsigned long last_rotated;
+ unsigned long last_io;
+};
+
struct lruvec {
struct list_head lists[NR_LRU_LISTS];
/* per lruvec lru_lock for memcg */
@@ -763,9 +771,12 @@ struct lruvec {
* These track the cost of reclaiming one LRU - file or anon -
* over the other. As the observed cost of reclaiming one LRU
* increases, the reclaim scan balance tips toward the other.
+ * Updated and decayed at prepare_scan_control() time; cost_lock
+ * serialises that update.
*/
- unsigned long anon_cost;
- unsigned long file_cost;
+ struct lru_cost cost[ANON_AND_FILE];
+ /* Protects cost[]. */
+ spinlock_t cost_lock;
/* Non-resident age, driven by LRU movement */
atomic_long_t nonresident_age;
/* Refaults at the time of last reclaim cycle */
@@ -2058,7 +2069,6 @@ static inline struct mem_section *__nr_to_section(unsigned long nr)
#endif
return &mem_section[root][nr & SECTION_ROOT_MASK];
}
-extern size_t mem_section_usage_size(void);
/*
* We use the lower bits of the mem_map pointer to store a little bit of
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 48b806aa3a2f..b85a73ae7919 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -552,7 +552,6 @@ static inline const struct cred *nfs_file_cred(struct file *file)
/*
* linux/fs/nfs/direct.c
*/
-int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter);
ssize_t nfs_file_direct_read(struct kiocb *iocb,
struct iov_iter *iter, bool swap);
ssize_t nfs_file_direct_write(struct kiocb *iocb,
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index f23d4b218da0..79c53ec45dfa 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -55,7 +55,6 @@ struct page_ext {
extern bool early_page_ext;
extern unsigned long page_ext_size;
-extern void pgdat_page_ext_init(struct pglist_data *pgdat);
static inline bool early_page_ext_enabled(void)
{
@@ -202,10 +201,6 @@ static inline bool early_page_ext_enabled(void)
return false;
}
-static inline void pgdat_page_ext_init(struct pglist_data *pgdat)
-{
-}
-
static inline void page_ext_init(void)
{
}
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index c6fc783aaee5..0adfa6605653 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -1094,10 +1094,44 @@ static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma,
static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,
const unsigned long address)
{
- pgoff_t pgoff;
+ return linear_page_delta(vma, address) + vma_start_pgoff(vma);
+}
+
+static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma,
+ const unsigned long address)
+{
+ return linear_page_delta(vma, address) + vma_start_anon_pgoff(vma);
+}
+
+/**
+ * linear_anon_page_index() - Determine the absolute anonymous page offset of
+ * @address within @vma.
+ * @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides.
+ * @address: The address whose absolute page offset is required.
+ *
+ * This returns the anonymous page offset of @address, which is the page offset
+ * the address possessed at the time the VMA was first faulted.
+ *
+ * For anonymous mappings, this returns the same value as linear_page_index().
+ *
+ * For MAP_PRIVATE file-backed mappings, this returns the anonymous page offset
+ * of @address, which is the page offset the address possessed at the time the
+ * VMA was first faulted.
+ *
+ * It is not valid to call this function for shared file-backed mappings.
+ *
+ * Returns: The absolute anonymous page offset of @address within @vma.
+ */
+static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma,
+ const unsigned long address)
+{
+ const pgoff_t pgoff = __linear_anon_page_index(vma, address);
+
+ VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));
+ /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */
+ if (vma_is_anonymous(vma) && !vma->vm_file)
+ VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address));
- pgoff = linear_page_delta(vma, address);
- pgoff += vma_start_pgoff(vma);
return pgoff;
}
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 2cba7cc2b01f..dbe3267a0a13 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -65,13 +65,8 @@
*
* Archs which need weak percpu definitions should set
* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU when necessary.
- *
- * To ensure that the generic code observes the above two
- * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
- * definition is used for all cases.
*/
-#if (defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)) || \
- defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
+#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)
/*
* __pcpu_scope_* dummy variable is used to enforce scope. It
* receives the static modifier when it's used in front of
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 8dc0871e5f00..0b332770abee 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -843,7 +843,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio,
* Called from mm/vmscan.c to handle paging out
*/
int folio_referenced(struct folio *, int is_locked,
- struct mem_cgroup *memcg, vm_flags_t *vm_flags);
+ struct mem_cgroup *memcg, vma_flags_t *vma_flags);
void try_to_migrate(struct folio *folio, enum ttu_flags flags);
void try_to_unmap(struct folio *, enum ttu_flags flags);
@@ -864,13 +864,14 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr,
struct page_vma_mapped_walk {
unsigned long pfn;
unsigned long nr_pages;
- pgoff_t pgoff;
+ pgoff_t pgoff; /* Only meaningful if nr_pages > 1 and not a KSM walk */
struct vm_area_struct *vma;
unsigned long address;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
unsigned int flags;
+ bool pgoff_is_anon : 1;
};
#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \
@@ -881,6 +882,7 @@ struct page_vma_mapped_walk {
.vma = _vma, \
.address = _address, \
.flags = _flags, \
+ .pgoff_is_anon = folio_test_anon(_folio), \
}
static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
@@ -975,10 +977,9 @@ struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
#define anon_vma_prepare(vma) (0)
static inline int folio_referenced(struct folio *folio, int is_locked,
- struct mem_cgroup *memcg,
- vm_flags_t *vm_flags)
+ struct mem_cgroup *memcg, vma_flags_t *vma_flags)
{
- *vm_flags = 0;
+ vma_flags_clear_all(vma_flags);
return 0;
}
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c7cdbb3e4aa6..8b3d47a325cc 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1298,6 +1298,7 @@ struct task_struct {
u64 curr_chain_key;
int lockdep_depth;
unsigned int lockdep_recursion;
+ unsigned int lockdep_seq;
struct held_lock held_locks[MAX_LOCK_DEPTH];
#endif
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index e729b9b0e38d..5663dff53186 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -12,8 +12,6 @@
#include <linux/userfaultfd_k.h>
#include <linux/bits.h>
-struct swap_iocb;
-
/* inode in-kernel data */
#ifdef CONFIG_TMPFS_QUOTA
@@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)
void shmem_unlock_mapping(struct address_space *mapping);
struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask);
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
- struct list_head *folio_list);
+int shmem_write_folio(struct folio *folio);
void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
int shmem_unuse(unsigned int type);
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 696ed01709c2..5658a1634b85 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -202,7 +202,6 @@ enum {
SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */
SWP_BLKDEV = (1 << 6), /* its a block device */
SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */
- SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */
SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */
SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */
SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */
@@ -276,6 +275,7 @@ struct swap_info_struct {
struct work_struct reclaim_work; /* reclaim worker */
struct list_head discard_clusters; /* discard clusters list */
struct plist_node avail_list; /* entry in swap_avail_head */
+ const struct swap_ops *ops;
};
static inline swp_entry_t page_swap_entry(struct page *page)
@@ -298,6 +298,14 @@ void folio_add_lru(struct folio *folio);
void folio_mark_accessed(struct folio *folio);
void lru_add_drain_all(void);
+enum lru_cache_drained {
+ LRU_CACHE_NOT_DRAINED,
+ LRU_CACHE_DRAINED,
+ LRU_CACHE_DRAINED_ALL,
+};
+void lru_cache_drain_for_folio(const struct folio *folio,
+ unsigned int extra_refs, enum lru_cache_drained *drained);
+
/* linux/mm/folio-compat.c */
void mark_page_accessed(struct page *page);
@@ -309,7 +317,6 @@ static inline bool lru_cache_disabled(void)
}
extern unsigned long shrink_all_memory(unsigned long nr_pages);
-extern int vm_swappiness;
long remove_mapping(struct address_space *mapping, struct folio *folio);
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
@@ -334,7 +341,6 @@ extern void __meminit kswapd_run(int nid);
extern void __meminit kswapd_stop(int nid);
#ifdef CONFIG_SWAP
-
int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
unsigned long nr_pages, sector_t start_block);
int generic_swapfile_activate(struct swap_info_struct *, struct file *,
@@ -468,25 +474,7 @@ static inline int add_swap_extent(struct swap_info_struct *sis,
}
#endif /* CONFIG_SWAP */
#ifdef CONFIG_MEMCG
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
-{
- /* Cgroup2 doesn't have per-cgroup swappiness */
- if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
- return READ_ONCE(vm_swappiness);
-
- /* root ? */
- if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
- return READ_ONCE(vm_swappiness);
-
- return READ_ONCE(memcg->swappiness);
-}
-
void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
-#else
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
-{
- return READ_ONCE(vm_swappiness);
-}
#endif
#if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
diff --git a/include/linux/swap_ops.h b/include/linux/swap_ops.h
new file mode 100644
index 000000000000..57ac6c703f68
--- /dev/null
+++ b/include/linux/swap_ops.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _MM_SWAP_OPS_H
+#define _MM_SWAP_OPS_H
+
+#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */
+
+struct swap_iocb {
+ union {
+ struct kiocb iocb;
+ struct bio bio;
+ };
+ struct bio_vec bvecs[SWAP_CLUSTER_MAX];
+ int nr_bvecs;
+ int len;
+};
+
+struct swap_io_ctx {
+ struct swap_iocb *sio;
+ struct swap_info_struct *sis;
+};
+
+/*
+ * SWAP_OPS_F_REQUIRE_NOFS:
+ * When set, all reclaim operations must operated as GFS_NOFS and not
+ * just GFP_NOIO, as GFP_NOIO allocations could recourse into the
+ * file system backing this swap file.
+ */
+#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0)
+
+struct swap_ops {
+ unsigned int flags;
+
+ bool (*can_merge)(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size, int rw);
+ void (*submit_write)(struct swap_io_ctx *ctx);
+ void (*submit_read)(struct swap_io_ctx *ctx);
+};
+
+void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter);
+bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
+ size_t prev_folio_size, int rw);
+int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops);
+
+#endif /* _MM_SWAP_OPS_H */
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index c956bc445ee0..e7d0d529f3e0 100644
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -5,6 +5,7 @@
#include <linux/radix-tree.h>
#include <linux/bug.h>
#include <linux/mm_types.h>
+#include <linux/swap.h>
#ifdef CONFIG_MMU
@@ -325,8 +326,8 @@ struct page_vma_mapped_walk;
extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
struct page *page);
-extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
- struct page *new);
+void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
+ struct folio *folio);
extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);
@@ -346,7 +347,7 @@ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
}
static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
- struct page *new)
+ struct folio *folio)
{
BUILD_BUG();
}
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 03fe95f5a020..2628ccda076a 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
KSTACK_REST,
#endif
#endif /* CONFIG_DEBUG_STACK_USAGE */
+#ifdef CONFIG_SWAP
+ NRSWPIN,
+ NRSWPOUT,
+#endif /* CONFIG_SWAP */
NR_VM_EVENT_ITEMS
};
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index e4d8d0a9f30f..aed121d729b0 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -62,7 +62,7 @@ struct vm_struct {
#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
unsigned int page_order;
#endif
- unsigned int nr_pages;
+ unsigned long nr_pages;
phys_addr_t phys_addr;
const void *caller;
unsigned long requested_size;
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 3c9c266cf782..5b31d8e7ae40 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -20,7 +20,6 @@ struct reclaim_stat {
unsigned nr_congested;
unsigned nr_writeback;
unsigned nr_immediate;
- unsigned nr_pageout;
unsigned nr_activate[ANON_AND_FILE];
unsigned nr_ref_keep;
unsigned nr_unmap_fail;
@@ -194,6 +193,19 @@ unsigned long global_node_page_state_pages(enum node_stat_item item)
return x;
}
+/*
+ * Non-clamping variant of global_node_page_state() intended for callers that
+ * snapshot a monotonically-incremented counter and subtract two samples.
+ * Returns the raw wrapping value so that unsigned modular subtraction stays
+ * correct across a signed-long overflow (a real hazard on 32-bit) that the
+ * clamp in global_node_page_state() would otherwise turn into a huge spurious
+ * delta. Do NOT use for non-monotonic page-count reads.
+ */
+static inline unsigned long global_node_page_state_monotonic(enum node_stat_item item)
+{
+ return (unsigned long)atomic_long_read(&vm_node_stat[item]);
+}
+
static inline unsigned long global_node_page_state(enum node_stat_item item)
{
VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
@@ -259,11 +271,14 @@ extern unsigned long node_page_state(struct pglist_data *pgdat,
enum node_stat_item item);
extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
enum node_stat_item item);
+extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
+ enum node_stat_item item);
extern void fold_vm_numa_events(void);
#else
#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
#define node_page_state(node, item) global_node_page_state(item)
#define node_page_state_pages(node, item) global_node_page_state_pages(item)
+#define node_page_state_monotonic(node, item) global_node_page_state_monotonic(item)
static inline void fold_vm_numa_events(void)
{
}
diff --git a/include/rv/da_monitor.h b/include/rv/da_monitor.h
index e3cf85c9ce55..6fc5ef8142ac 100644
--- a/include/rv/da_monitor.h
+++ b/include/rv/da_monitor.h
@@ -25,7 +25,7 @@
/*
* Per-cpu variables require a unique name although static in some
- * configurations (e.g. CONFIG_DEBUG_FORCE_WEAK_PER_CPU or alpha modules).
+ * configurations (e.g. alpha modules).
*/
#define DA_MON_NAME CONCATENATE(da_mon_, MONITOR_NAME)
diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h
new file mode 100644
index 000000000000..e3ad94444864
--- /dev/null
+++ b/include/uapi/linux/alloc_tag.h
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * alloc_tag IOCTL API definition
+ *
+ * Copyright (C) 2026 Google, LLC. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _UAPI_ALLOC_TAG_H
+#define _UAPI_ALLOC_TAG_H
+
+#include <linux/types.h>
+
+/*
+ * Function, file and module names often have the same prefixes, therefore
+ * when filtering by these criteria, we compare the last 64 characters to
+ * minimize the chances of name collisions
+ */
+#define ALLOCINFO_STR_SIZE 64
+
+struct allocinfo_content_id {
+ __u64 id;
+};
+
+struct allocinfo_tag {
+ /* Longer names are trimmed */
+ char modname[ALLOCINFO_STR_SIZE];
+ char function[ALLOCINFO_STR_SIZE];
+ char filename[ALLOCINFO_STR_SIZE];
+ __u64 lineno;
+};
+
+/* The alignment ensures 32-bit compatible interfaces are not broken */
+struct allocinfo_counter {
+ __u64 bytes;
+ __u64 calls;
+ __u8 accurate;
+} __attribute__((aligned(8)));
+
+struct allocinfo_tag_data {
+ struct allocinfo_tag tag;
+ struct allocinfo_counter counter;
+};
+
+enum {
+ ALLOCINFO_FILTER_MODNAME,
+ ALLOCINFO_FILTER_FUNCTION,
+ ALLOCINFO_FILTER_FILENAME,
+ ALLOCINFO_FILTER_LINENO,
+ ALLOCINFO_FILTER_INACCURATE,
+ ALLOCINFO_FILTER_MIN_SIZE,
+ ALLOCINFO_FILTER_MAX_SIZE,
+ __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE
+};
+
+#define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME)
+#define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION)
+#define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME)
+#define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO)
+#define ALLOCINFO_FILTER_MASK_INACCURATE (1 << ALLOCINFO_FILTER_INACCURATE)
+#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE)
+#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE)
+
+#define ALLOCINFO_FILTER_MASKS \
+ ((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1)
+
+struct allocinfo_filter {
+ __u64 mask; /* bitmask of the filter fields used */
+ struct allocinfo_tag fields;
+ __u64 min_size;
+ __u64 max_size;
+ /* filter criteria only; see allocinfo_counter.accurate for actual accuracy */
+ __u64 inaccurate;
+};
+
+struct allocinfo_get_at {
+ /* inputs */
+ __u64 pos;
+ struct allocinfo_filter filter;
+ /* output */
+ struct allocinfo_tag_data data;
+};
+
+#define _ALLOCINFO_IOC_CONTENT_ID 0
+#define _ALLOCINFO_IOC_GET_AT 1
+#define _ALLOCINFO_IOC_GET_NEXT 2
+
+#define ALLOCINFO_IOC_BASE 0xA6
+#define ALLOCINFO_IOC_CONTENT_ID _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID, \
+ struct allocinfo_content_id)
+#define ALLOCINFO_IOC_GET_AT _IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT, \
+ struct allocinfo_get_at)
+#define ALLOCINFO_IOC_GET_NEXT _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT, \
+ struct allocinfo_tag_data)
+
+#endif /* _UAPI_ALLOC_TAG_H */
generated by cgit v1.2.3 (git 2.25.1) at 2026-08-28 18:57:23 +0000