summaryrefslogtreecommitdiff
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/damon/core.c22
-rw-r--r--mm/damon/ops-common.c5
-rw-r--r--mm/damon/paddr.c20
-rw-r--r--mm/damon/sysfs-schemes.c18
-rw-r--r--mm/damon/sysfs.c23
-rw-r--r--mm/damon/tests/core-kunit.h15
-rw-r--r--mm/damon/tests/vaddr-kunit.h5
-rw-r--r--mm/damon/vaddr.c34
-rw-r--r--mm/huge_memory.c354
-rw-r--r--mm/hugetlb.c25
-rw-r--r--mm/hugetlb_cgroup.c7
-rw-r--r--mm/hugetlb_vmemmap.c296
-rw-r--r--mm/memcontrol-v1.c43
-rw-r--r--mm/memcontrol.c13
-rw-r--r--mm/mempolicy.c2
-rw-r--r--mm/migrate.c12
-rw-r--r--mm/mm_init.c1
-rw-r--r--mm/page_alloc.c6
-rw-r--r--mm/page_vma_mapped.c8
-rw-r--r--mm/rmap.c15
-rw-r--r--mm/slab.h8
-rw-r--r--mm/slab_common.c44
-rw-r--r--mm/slub.c106
-rw-r--r--mm/sparse-vmemmap.c53
-rw-r--r--mm/userfaultfd.c4
-rw-r--r--mm/vmscan.c41
26 files changed, 587 insertions, 593 deletions
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 70ac1f08753d..a9168a03b340 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -412,6 +412,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
INIT_LIST_HEAD(&scheme->filters);
INIT_LIST_HEAD(&scheme->ops_filters);
scheme->stat = (struct damos_stat){};
+ scheme->last_applied = NULL;
INIT_LIST_HEAD(&scheme->list);
scheme->quota = *(damos_quota_init(quota));
@@ -2364,17 +2365,19 @@ static void damon_merge_two_regions(struct damon_target *t,
* sz_limit size upper limit of each region
*/
static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
- unsigned long sz_limit)
+ unsigned long sz_limit, bool count_age)
{
struct damon_region *r, *prev = NULL, *next;
damon_for_each_region_safe(r, next, t) {
- if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
- r->age = 0;
- else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0))
- r->age = 0;
- else
- r->age++;
+ if (count_age) {
+ if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
+ r->age = 0;
+ else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0))
+ r->age = 0;
+ else
+ r->age++;
+ }
if (prev && prev->ar.end == r->ar.start &&
abs(prev->nr_accesses - r->nr_accesses) <= thres &&
@@ -2408,15 +2411,18 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
struct damon_target *t;
unsigned int nr_regions;
unsigned int max_thres;
+ bool count_age = true;
max_thres = c->attrs.aggr_interval /
(c->attrs.sample_interval ? c->attrs.sample_interval : 1);
do {
nr_regions = 0;
damon_for_each_target(t, c) {
- damon_merge_regions_of(t, threshold, sz_limit);
+ damon_merge_regions_of(t, threshold, sz_limit,
+ count_age);
nr_regions += damon_nr_regions(t);
}
+ count_age = false;
threshold = max(1, threshold * 2);
} while (nr_regions > c->attrs.max_nr_regions &&
threshold / 2 < max_thres);
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index ceaa2a6e1c9a..929aab600c74 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -101,8 +101,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
unsigned int age_weight = s->quota.weight_age;
int hotness;
- freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE /
- damon_max_nr_accesses(&c->attrs);
+ freq_subscore = mult_frac(r->nr_accesses_bp / 10000,
+ DAMON_MAX_SUBSCORE,
+ damon_max_nr_accesses(&c->attrs));
age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index f28cfee8351d..1854b4bb6bd2 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -67,7 +67,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
}
}
-static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
+static bool damon_pa_young(phys_addr_t paddr)
{
struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
bool accessed;
@@ -76,7 +76,6 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
return false;
accessed = damon_folio_young(folio);
- *folio_sz = folio_size(folio);
folio_put(folio);
return accessed;
}
@@ -84,23 +83,12 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
static void __damon_pa_check_access(struct damon_region *r,
struct damon_attrs *attrs, unsigned long addr_unit)
{
- static phys_addr_t last_addr;
- static unsigned long last_folio_sz = PAGE_SIZE;
- static bool last_accessed;
+ bool accessed;
phys_addr_t sampling_addr = damon_pa_phys_addr(
r->sampling_addr, addr_unit);
- /* If the region is in the last checked page, reuse the result */
- if (ALIGN_DOWN(last_addr, last_folio_sz) ==
- ALIGN_DOWN(sampling_addr, last_folio_sz)) {
- damon_update_region_access_rate(r, last_accessed, attrs);
- return;
- }
-
- last_accessed = damon_pa_young(sampling_addr, &last_folio_sz);
- damon_update_region_access_rate(r, last_accessed, attrs);
-
- last_addr = sampling_addr;
+ accessed = damon_pa_young(sampling_addr);
+ damon_update_region_access_rate(r, accessed, attrs);
}
static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index adf60f672b8b..37e0a8bef4b1 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -166,6 +166,7 @@ static void damon_sysfs_scheme_regions_rm_dirs(
list_for_each_entry_safe(r, next, &regions->regions_list, list) {
list_del(&r->list);
+ kobject_del(&r->kobj);
kobject_put(&r->kobj);
regions->nr_regions--;
}
@@ -704,8 +705,10 @@ static void damon_sysfs_scheme_filters_rm_dirs(
struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr;
int i;
- for (i = 0; i < filters->nr; i++)
+ for (i = 0; i < filters->nr; i++) {
+ kobject_del(&filters_arr[i]->kobj);
kobject_put(&filters_arr[i]->kobj);
+ }
filters->nr = 0;
kfree(filters_arr);
filters->filters_arr = NULL;
@@ -1191,8 +1194,10 @@ static void damos_sysfs_quota_goals_rm_dirs(
struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr;
int i;
- for (i = 0; i < goals->nr; i++)
+ for (i = 0; i < goals->nr; i++) {
+ kobject_del(&goals_arr[i]->kobj);
kobject_put(&goals_arr[i]->kobj);
+ }
goals->nr = 0;
kfree(goals_arr);
goals->goals_arr = NULL;
@@ -1762,8 +1767,10 @@ static void damos_sysfs_dests_rm_dirs(
struct damos_sysfs_dest **dests_arr = dests->dests_arr;
int i;
- for (i = 0; i < dests->nr; i++)
+ for (i = 0; i < dests->nr; i++) {
+ kobject_del(&dests_arr[i]->kobj);
kobject_put(&dests_arr[i]->kobj);
+ }
dests->nr = 0;
kfree(dests_arr);
dests->dests_arr = NULL;
@@ -2303,6 +2310,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes)
for (i = 0; i < schemes->nr; i++) {
damon_sysfs_scheme_rm_dirs(schemes_arr[i]);
+ kobject_del(&schemes_arr[i]->kobj);
kobject_put(&schemes_arr[i]->kobj);
}
schemes->nr = 0;
@@ -2344,13 +2352,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes,
goto out;
err = damon_sysfs_scheme_add_dirs(scheme);
if (err)
- goto out;
+ goto del_out;
schemes_arr[i] = scheme;
schemes->nr++;
}
return 0;
+del_out:
+ kobject_del(&scheme->kobj);
out:
damon_sysfs_schemes_rm_dirs(schemes);
kobject_put(&scheme->kobj);
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index a6c3e83a3dad..53f99f05eb38 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -107,8 +107,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions)
struct damon_sysfs_region **regions_arr = regions->regions_arr;
int i;
- for (i = 0; i < regions->nr; i++)
+ for (i = 0; i < regions->nr; i++) {
+ kobject_del(&regions_arr[i]->kobj);
kobject_put(&regions_arr[i]->kobj);
+ }
regions->nr = 0;
kfree(regions_arr);
regions->regions_arr = NULL;
@@ -305,6 +307,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets)
for (i = 0; i < targets->nr; i++) {
damon_sysfs_target_rm_dirs(targets_arr[i]);
+ kobject_del(&targets_arr[i]->kobj);
kobject_put(&targets_arr[i]->kobj);
}
targets->nr = 0;
@@ -343,13 +346,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets,
err = damon_sysfs_target_add_dirs(target);
if (err)
- goto out;
+ goto del_out;
targets_arr[i] = target;
targets->nr++;
}
return 0;
+del_out:
+ kobject_del(&target->kobj);
out:
damon_sysfs_targets_rm_dirs(targets);
kobject_put(&target->kobj);
@@ -1078,6 +1083,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts)
for (i = 0; i < contexts->nr; i++) {
damon_sysfs_context_rm_dirs(contexts_arr[i]);
+ kobject_del(&contexts_arr[i]->kobj);
kobject_put(&contexts_arr[i]->kobj);
}
contexts->nr = 0;
@@ -1116,13 +1122,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts,
err = damon_sysfs_context_add_dirs(context);
if (err)
- goto out;
+ goto del_out;
contexts_arr[i] = context;
contexts->nr++;
}
return 0;
+del_out:
+ kobject_del(&context->kobj);
out:
damon_sysfs_contexts_rm_dirs(contexts);
kobject_put(&context->kobj);
@@ -1437,11 +1445,11 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx,
err = damon_select_ops(ctx, sys_ctx->ops_id);
if (err)
return err;
- ctx->addr_unit = sys_ctx->addr_unit;
+ ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit);
/* addr_unit is respected by only DAMON_OPS_PADDR */
if (sys_ctx->ops_id == DAMON_OPS_PADDR)
ctx->min_sz_region = max(
- DAMON_MIN_REGION / sys_ctx->addr_unit, 1);
+ DAMON_MIN_REGION / ctx->addr_unit, 1);
err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs);
if (err)
return err;
@@ -1867,6 +1875,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds)
for (i = 0; i < kdamonds->nr; i++) {
damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]);
+ kobject_del(&kdamonds_arr[i]->kobj);
kobject_put(&kdamonds_arr[i]->kobj);
}
kdamonds->nr = 0;
@@ -1921,13 +1930,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds,
err = damon_sysfs_kdamond_add_dirs(kdamond);
if (err)
- goto out;
+ goto del_out;
kdamonds_arr[i] = kdamond;
kdamonds->nr++;
}
return 0;
+del_out:
+ kobject_del(&kdamond->kobj);
out:
damon_sysfs_kdamonds_rm_dirs(kdamonds);
kobject_put(&kdamond->kobj);
diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h
index 668777875272..0ce648131943 100644
--- a/mm/damon/tests/core-kunit.h
+++ b/mm/damon/tests/core-kunit.h
@@ -166,6 +166,10 @@ static void damon_test_split_at(struct kunit *test)
r->last_nr_accesses = 15;
damon_add_region(r, t);
damon_split_region_at(t, r, 25);
+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+ if (damon_nr_regions(t) != 2)
+ goto out;
+
KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
KUNIT_EXPECT_EQ(test, r->ar.end, 25ul);
@@ -177,6 +181,7 @@ static void damon_test_split_at(struct kunit *test)
KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses);
KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses);
+out:
damon_free_target(t);
damon_destroy_ctx(c);
}
@@ -261,14 +266,17 @@ static void damon_test_merge_regions_of(struct kunit *test)
damon_add_region(r, t);
}
- damon_merge_regions_of(t, 9, 9999);
+ damon_merge_regions_of(t, 9, 9999, true);
/* 0-112, 114-130, 130-156, 156-170 */
KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u);
+ if (damon_nr_regions(t) != 5)
+ goto out;
for (i = 0; i < 5; i++) {
r = __nth_region_of(t, i);
KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]);
KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]);
}
+out:
damon_free_target(t);
}
@@ -588,6 +596,8 @@ static void damos_test_filter_out(struct kunit *test)
KUNIT_EXPECT_EQ(test, r->ar.start, DAMON_MIN_REGION * 1);
KUNIT_EXPECT_EQ(test, r->ar.end, DAMON_MIN_REGION * 2);
KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+ if (damon_nr_regions(t) != 2)
+ goto out;
r2 = damon_next_region(r);
KUNIT_EXPECT_EQ(test, r2->ar.start, DAMON_MIN_REGION * 2);
KUNIT_EXPECT_EQ(test, r2->ar.end, DAMON_MIN_REGION * 4);
@@ -602,11 +612,14 @@ static void damos_test_filter_out(struct kunit *test)
KUNIT_EXPECT_EQ(test, r->ar.start, DAMON_MIN_REGION * 2);
KUNIT_EXPECT_EQ(test, r->ar.end, DAMON_MIN_REGION * 6);
KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+ if (damon_nr_regions(t) != 2)
+ goto out;
r2 = damon_next_region(r);
KUNIT_EXPECT_EQ(test, r2->ar.start, DAMON_MIN_REGION * 6);
KUNIT_EXPECT_EQ(test, r2->ar.end, DAMON_MIN_REGION * 8);
damon_destroy_region(r2, t);
+out:
damon_free_target(t);
damos_free_filter(f);
}
diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h
index 30dc5459f1d2..e2c36d4d8789 100644
--- a/mm/damon/tests/vaddr-kunit.h
+++ b/mm/damon/tests/vaddr-kunit.h
@@ -149,12 +149,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test,
damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION);
+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2);
+ if (damon_nr_regions(t) != nr_expected / 2)
+ goto out;
+
for (i = 0; i < nr_expected / 2; i++) {
r = __nth_region_of(t, i);
KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]);
KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]);
}
+out:
damon_destroy_target(t, NULL);
}
diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index db93ca202460..c49f494ec17c 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -431,8 +431,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx)
}
struct damon_young_walk_private {
- /* size of the folio for the access checked virtual memory address */
- unsigned long *folio_sz;
bool young;
};
@@ -468,7 +466,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
mmu_notifier_test_young(walk->mm,
addr))
priv->young = true;
- *priv->folio_sz = HPAGE_PMD_SIZE;
folio_put(folio);
huge_out:
spin_unlock(ptl);
@@ -490,7 +487,6 @@ regular_page:
if (pte_young(ptent) || !folio_test_idle(folio) ||
mmu_notifier_test_young(walk->mm, addr))
priv->young = true;
- *priv->folio_sz = folio_size(folio);
folio_put(folio);
out:
pte_unmap_unlock(pte, ptl);
@@ -519,7 +515,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask,
if (pte_young(entry) || !folio_test_idle(folio) ||
mmu_notifier_test_young(walk->mm, addr))
priv->young = true;
- *priv->folio_sz = huge_page_size(h);
folio_put(folio);
@@ -537,11 +532,9 @@ static const struct mm_walk_ops damon_young_ops = {
.walk_lock = PGWALK_RDLOCK,
};
-static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
- unsigned long *folio_sz)
+static bool damon_va_young(struct mm_struct *mm, unsigned long addr)
{
struct damon_young_walk_private arg = {
- .folio_sz = folio_sz,
.young = false,
};
@@ -558,29 +551,18 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
* r the region to be checked
*/
static void __damon_va_check_access(struct mm_struct *mm,
- struct damon_region *r, bool same_target,
+ struct damon_region *r,
struct damon_attrs *attrs)
{
- static unsigned long last_addr;
- static unsigned long last_folio_sz = PAGE_SIZE;
- static bool last_accessed;
+ bool accessed;
if (!mm) {
damon_update_region_access_rate(r, false, attrs);
return;
}
- /* If the region is in the last checked page, reuse the result */
- if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) ==
- ALIGN_DOWN(r->sampling_addr, last_folio_sz))) {
- damon_update_region_access_rate(r, last_accessed, attrs);
- return;
- }
-
- last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz);
- damon_update_region_access_rate(r, last_accessed, attrs);
-
- last_addr = r->sampling_addr;
+ accessed = damon_va_young(mm, r->sampling_addr);
+ damon_update_region_access_rate(r, accessed, attrs);
}
static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
@@ -589,16 +571,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
struct mm_struct *mm;
struct damon_region *r;
unsigned int max_nr_accesses = 0;
- bool same_target;
damon_for_each_target(t, ctx) {
mm = damon_get_mm(t);
- same_target = false;
damon_for_each_region(r, t) {
- __damon_va_check_access(mm, r, same_target,
- &ctx->attrs);
+ __damon_va_check_access(mm, r, &ctx->attrs);
max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
- same_target = true;
}
if (mm)
mmput(mm);
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 8564a92d83f0..56b85d3d3dbb 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1095,28 +1095,86 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
return pmd;
}
+static struct deferred_split *split_queue_node(int nid)
+{
+ struct pglist_data *pgdata = NODE_DATA(nid);
+
+ return &pgdata->deferred_split_queue;
+}
+
#ifdef CONFIG_MEMCG
static inline
-struct deferred_split *get_deferred_split_queue(struct folio *folio)
+struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
+ struct deferred_split *queue)
{
- struct mem_cgroup *memcg = folio_memcg(folio);
- struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));
+ if (mem_cgroup_disabled())
+ return NULL;
+ if (split_queue_node(folio_nid(folio)) == queue)
+ return NULL;
+ return container_of(queue, struct mem_cgroup, deferred_split_queue);
+}
- if (memcg)
- return &memcg->deferred_split_queue;
- else
- return &pgdat->deferred_split_queue;
+static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
+{
+ return memcg ? &memcg->deferred_split_queue : split_queue_node(nid);
}
#else
static inline
-struct deferred_split *get_deferred_split_queue(struct folio *folio)
+struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
+ struct deferred_split *queue)
{
- struct pglist_data *pgdat = NODE_DATA(folio_nid(folio));
+ return NULL;
+}
- return &pgdat->deferred_split_queue;
+static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
+{
+ return split_queue_node(nid);
}
#endif
+static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg)
+{
+ struct deferred_split *queue;
+
+ queue = memcg_split_queue(nid, memcg);
+ spin_lock(&queue->split_queue_lock);
+
+ return queue;
+}
+
+static struct deferred_split *
+split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags)
+{
+ struct deferred_split *queue;
+
+ queue = memcg_split_queue(nid, memcg);
+ spin_lock_irqsave(&queue->split_queue_lock, *flags);
+
+ return queue;
+}
+
+static struct deferred_split *folio_split_queue_lock(struct folio *folio)
+{
+ return split_queue_lock(folio_nid(folio), folio_memcg(folio));
+}
+
+static struct deferred_split *
+folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags)
+{
+ return split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags);
+}
+
+static inline void split_queue_unlock(struct deferred_split *queue)
+{
+ spin_unlock(&queue->split_queue_lock);
+}
+
+static inline void split_queue_unlock_irqrestore(struct deferred_split *queue,
+ unsigned long flags)
+{
+ spin_unlock_irqrestore(&queue->split_queue_lock, flags);
+}
+
static inline bool is_transparent_hugepage(const struct folio *folio)
{
if (!folio_test_large(folio))
@@ -3240,7 +3298,7 @@ static void remap_page(struct folio *folio, unsigned long nr, int flags)
if (!folio_test_anon(folio))
return;
for (;;) {
- remove_migration_ptes(folio, folio, RMP_LOCKED | flags);
+ remove_migration_ptes(folio, folio, TTU_RMAP_LOCKED | flags);
i += folio_nr_pages(folio);
if (i >= nr)
break;
@@ -3270,23 +3328,6 @@ static void lru_add_split_folio(struct folio *folio, struct folio *new_folio,
}
}
-/* Racy check whether the huge page can be split */
-bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins)
-{
- int extra_pins;
-
- /* Additional pins from page cache */
- if (folio_test_anon(folio))
- extra_pins = folio_test_swapcache(folio) ?
- folio_nr_pages(folio) : 0;
- else
- extra_pins = folio_nr_pages(folio);
- if (pextra_pins)
- *pextra_pins = extra_pins;
- return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins -
- caller_pins;
-}
-
static bool page_range_has_hwpoisoned(struct page *page, long nr_pages)
{
for (; nr_pages; page++, nr_pages--)
@@ -3547,15 +3588,40 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
return ret;
}
-bool folio_split_supported(struct folio *folio, unsigned int new_order,
- bool uniform_split, bool warns)
+/**
+ * folio_check_splittable() - check if a folio can be split to a given order
+ * @folio: folio to be split
+ * @new_order: the smallest order of the after split folios (since buddy
+ * allocator like split generates folios with orders from @folio's
+ * order - 1 to new_order).
+ * @uniform_split: perform uniform split or not (non-uniform split)
+ *
+ * folio_check_splittable() checks if @folio can be split to @new_order using
+ * the requested split method. The truncated folio check must come first.
+ *
+ * Context: folio must be locked.
+ *
+ * Return: 0 - @folio can be split to @new_order, otherwise an error number is
+ * returned.
+ */
+int folio_check_splittable(struct folio *folio, unsigned int new_order,
+ bool uniform_split)
{
+ VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
+ /*
+ * Folios that just got truncated cannot get split. Signal to the
+ * caller that there was a race.
+ *
+ * TODO: this will also currently refuse folios without a mapping in the
+ * swapcache (shmem or to-be-anon folios).
+ */
+ if (!folio->mapping && !folio_test_anon(folio))
+ return -EBUSY;
+
if (folio_test_anon(folio)) {
/* order-1 is not supported for anonymous THP. */
- VM_WARN_ONCE(warns && new_order == 1,
- "Cannot split to order-1 folio");
if (new_order == 1)
- return false;
+ return -EINVAL;
} else if (!uniform_split || new_order) {
if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
!mapping_large_folio_support(folio->mapping)) {
@@ -3576,9 +3642,7 @@ bool folio_split_supported(struct folio *folio, unsigned int new_order,
* case, the mapping does not actually support large
* folios properly.
*/
- VM_WARN_ONCE(warns,
- "Cannot split file folio to non-0 order");
- return false;
+ return -EINVAL;
}
}
@@ -3591,12 +3655,16 @@ bool folio_split_supported(struct folio *folio, unsigned int new_order,
* here.
*/
if ((!uniform_split || new_order) && folio_test_swapcache(folio)) {
- VM_WARN_ONCE(warns,
- "Cannot split swapcache folio to non-0 order");
- return false;
+ return -EINVAL;
}
- return true;
+ if (is_huge_zero_folio(folio))
+ return -EINVAL;
+
+ if (folio_test_writeback(folio))
+ return -EBUSY;
+
+ return 0;
}
/*
@@ -3624,50 +3692,45 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
struct page *split_at, struct page *lock_at,
struct list_head *list, bool uniform_split)
{
- struct deferred_split *ds_queue = get_deferred_split_queue(folio);
+ struct deferred_split *ds_queue;
XA_STATE(xas, &folio->mapping->i_pages, folio->index);
struct folio *end_folio = folio_next(folio);
bool is_anon = folio_test_anon(folio);
+ struct mem_cgroup *memcg, *old_memcg;
struct address_space *mapping = NULL;
struct anon_vma *anon_vma = NULL;
- int order = folio_order(folio);
+ int old_order = folio_order(folio);
struct folio *new_folio, *next;
int nr_shmem_dropped = 0;
- int remap_flags = 0;
- int extra_pins, ret;
- pgoff_t end;
- bool is_hzp;
+ enum ttu_flags ttu_flags = 0;
+ pgoff_t end = 0;
+ int ret;
VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio);
- if (folio != page_folio(split_at) || folio != page_folio(lock_at))
- return -EINVAL;
-
- /*
- * Folios that just got truncated cannot get split. Signal to the
- * caller that there was a race.
- *
- * TODO: this will also currently refuse shmem folios that are in the
- * swapcache.
- */
- if (!is_anon && !folio->mapping)
- return -EBUSY;
-
- if (new_order >= folio_order(folio))
- return -EINVAL;
+ if (folio != page_folio(split_at) || folio != page_folio(lock_at)) {
+ ret = -EINVAL;
+ goto out_no_memcg;
+ }
- if (!folio_split_supported(folio, new_order, uniform_split, /* warn = */ true))
- return -EINVAL;
+ if (new_order >= old_order) {
+ ret = -EINVAL;
+ goto out_no_memcg;
+ }
- is_hzp = is_huge_zero_folio(folio);
- if (is_hzp) {
- pr_warn_ratelimited("Called split_huge_page for huge zero page\n");
- return -EBUSY;
+ ret = folio_check_splittable(folio, new_order, uniform_split);
+ if (ret) {
+ VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio");
+ goto out_no_memcg;
}
- if (folio_test_writeback(folio))
- return -EBUSY;
+ /*
+ * switch to folio's memcg as xarray node allocation can happen and
+ * needs to charge to it.
+ */
+ memcg = get_mem_cgroup_from_folio(folio);
+ old_memcg = set_active_memcg(memcg);
if (is_anon) {
/*
@@ -3732,7 +3795,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
* Racy check if we can split the page, before unmap_folio() will
* split PMDs
*/
- if (!can_split_folio(folio, 1, &extra_pins)) {
+ if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) {
ret = -EAGAIN;
goto out_unlock;
}
@@ -3755,29 +3818,33 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
}
/* Prevent deferred_split_scan() touching ->_refcount */
- spin_lock(&ds_queue->split_queue_lock);
- if (folio_ref_freeze(folio, 1 + extra_pins)) {
+ ds_queue = folio_split_queue_lock(folio);
+ if (folio_ref_freeze(folio,
+ (folio_test_anon(folio) &&
+ !folio_test_swapcache(folio) ?
+ 0 : folio_nr_pages(folio)) + 1)) {
struct swap_cluster_info *ci = NULL;
struct lruvec *lruvec;
int expected_refs;
- if (folio_order(folio) > 1 &&
- !list_empty(&folio->_deferred_list)) {
- ds_queue->split_queue_len--;
+ if (folio_order(folio) > 1) {
+ if (!list_empty(&folio->_deferred_list)) {
+ ds_queue->split_queue_len--;
+ /*
+ * Reinitialize page_deferred_list after removing the
+ * page from the split_queue, otherwise a subsequent
+ * split will see list corruption when checking the
+ * page_deferred_list.
+ */
+ list_del_init(&folio->_deferred_list);
+ }
if (folio_test_partially_mapped(folio)) {
folio_clear_partially_mapped(folio);
mod_mthp_stat(folio_order(folio),
MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
}
- /*
- * Reinitialize page_deferred_list after removing the
- * page from the split_queue, otherwise a subsequent
- * split will see list corruption when checking the
- * page_deferred_list.
- */
- list_del_init(&folio->_deferred_list);
}
- spin_unlock(&ds_queue->split_queue_lock);
+ split_queue_unlock(ds_queue);
if (mapping) {
int nr = folio_nr_pages(folio);
@@ -3824,7 +3891,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
next = folio_next(new_folio);
- expected_refs = folio_expected_ref_count(new_folio) + 1;
+ expected_refs = (folio_test_anon(new_folio) &&
+ !folio_test_swapcache(new_folio) ?
+ 0 : folio_nr_pages(new_folio)) + 1;
folio_ref_unfreeze(new_folio, expected_refs);
lru_add_split_folio(folio, new_folio, lruvec, list);
@@ -3864,7 +3933,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
* Otherwise, a parallel folio_try_get() can grab @folio
* and its caller can see stale page cache entries.
*/
- expected_refs = folio_expected_ref_count(folio) + 1;
+ expected_refs = (folio_test_anon(folio) &&
+ !folio_test_swapcache(folio) ?
+ 0 : folio_nr_pages(folio)) + 1;
folio_ref_unfreeze(folio, expected_refs);
unlock_page_lruvec(lruvec);
@@ -3872,7 +3943,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
if (ci)
swap_cluster_unlock(ci);
} else {
- spin_unlock(&ds_queue->split_queue_lock);
+ split_queue_unlock(ds_queue);
ret = -EAGAIN;
}
fail:
@@ -3885,8 +3956,8 @@ fail:
shmem_uncharge(mapping->host, nr_shmem_dropped);
if (!ret && is_anon)
- remap_flags = RMP_USE_SHARED_ZEROPAGE;
- remap_page(folio, 1 << order, remap_flags);
+ ttu_flags = TTU_USE_SHARED_ZEROPAGE;
+ remap_page(folio, 1 << old_order, ttu_flags);
/*
* Drop the mapping while the inode is still pinned. @folio stays
@@ -3928,10 +3999,14 @@ out_unlock:
if (mapping)
i_mmap_unlock_read(mapping);
out:
+ /* restore to caller's old_memcg */
+ set_active_memcg(old_memcg);
+ mem_cgroup_put(memcg);
+out_no_memcg:
xas_destroy(&xas);
- if (order == HPAGE_PMD_ORDER)
+ if (old_order == HPAGE_PMD_ORDER)
count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
- count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED);
+ count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED);
return ret;
}
@@ -4060,8 +4135,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio)
WARN_ON_ONCE(folio_ref_count(folio));
WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio));
- ds_queue = get_deferred_split_queue(folio);
- spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
+ ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
if (!list_empty(&folio->_deferred_list)) {
ds_queue->split_queue_len--;
if (folio_test_partially_mapped(folio)) {
@@ -4072,7 +4146,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio)
list_del_init(&folio->_deferred_list);
unqueued = true;
}
- spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
+ split_queue_unlock_irqrestore(ds_queue, flags);
return unqueued; /* useful for debug warnings */
}
@@ -4080,10 +4154,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio)
/* partially_mapped=false won't clear PG_partially_mapped folio flag */
void deferred_split_folio(struct folio *folio, bool partially_mapped)
{
- struct deferred_split *ds_queue = get_deferred_split_queue(folio);
-#ifdef CONFIG_MEMCG
- struct mem_cgroup *memcg = folio_memcg(folio);
-#endif
+ struct deferred_split *ds_queue;
unsigned long flags;
/*
@@ -4106,7 +4177,7 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped)
if (folio_test_swapcache(folio))
return;
- spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
+ ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
if (partially_mapped) {
if (!folio_test_partially_mapped(folio)) {
folio_set_partially_mapped(folio);
@@ -4121,15 +4192,16 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped)
VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio);
}
if (list_empty(&folio->_deferred_list)) {
+ struct mem_cgroup *memcg;
+
+ memcg = folio_split_queue_memcg(folio, ds_queue);
list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
ds_queue->split_queue_len++;
-#ifdef CONFIG_MEMCG
if (memcg)
set_shrinker_bit(memcg, folio_nid(folio),
- deferred_split_shrinker->id);
-#endif
+ shrinker_id(deferred_split_shrinker));
}
- spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
+ split_queue_unlock_irqrestore(ds_queue, flags);
}
static unsigned long deferred_split_count(struct shrinker *shrink,
@@ -4175,43 +4247,42 @@ static bool thp_underused(struct folio *folio)
static unsigned long deferred_split_scan(struct shrinker *shrink,
struct shrink_control *sc)
{
- struct pglist_data *pgdata = NODE_DATA(sc->nid);
- struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
+ struct deferred_split *ds_queue;
unsigned long flags;
- LIST_HEAD(list);
- struct folio *folio, *next, *prev = NULL;
- int split = 0, removed = 0;
+ struct folio *folio, *next;
+ int split = 0, i;
+ struct folio_batch fbatch;
-#ifdef CONFIG_MEMCG
- if (sc->memcg)
- ds_queue = &sc->memcg->deferred_split_queue;
-#endif
+ folio_batch_init(&fbatch);
- spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
+retry:
+ ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags);
/* Take pin on all head pages to avoid freeing them under us */
list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
_deferred_list) {
if (folio_try_get(folio)) {
- list_move(&folio->_deferred_list, &list);
- } else {
+ folio_batch_add(&fbatch, folio);
+ } else if (folio_test_partially_mapped(folio)) {
/* We lost race with folio_put() */
- if (folio_test_partially_mapped(folio)) {
- folio_clear_partially_mapped(folio);
- mod_mthp_stat(folio_order(folio),
- MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
- }
- list_del_init(&folio->_deferred_list);
- ds_queue->split_queue_len--;
+ folio_clear_partially_mapped(folio);
+ mod_mthp_stat(folio_order(folio),
+ MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
}
+ list_del_init(&folio->_deferred_list);
+ ds_queue->split_queue_len--;
if (!--sc->nr_to_scan)
break;
+ if (!folio_batch_space(&fbatch))
+ break;
}
- spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
+ split_queue_unlock_irqrestore(ds_queue, flags);
- list_for_each_entry_safe(folio, next, &list, _deferred_list) {
+ for (i = 0; i < folio_batch_count(&fbatch); i++) {
bool did_split = false;
bool underused = false;
+ struct deferred_split *fqueue;
+ folio = fbatch.folios[i];
if (!folio_test_partially_mapped(folio)) {
/*
* See try_to_map_unused_to_zeropage(): we cannot
@@ -4234,38 +4305,27 @@ static unsigned long deferred_split_scan(struct shrinker *shrink,
}
folio_unlock(folio);
next:
+ if (did_split || !folio_test_partially_mapped(folio))
+ continue;
/*
- * split_folio() removes folio from list on success.
* Only add back to the queue if folio is partially mapped.
* If thp_underused returns false, or if split_folio fails
* in the case it was underused, then consider it used and
* don't add it back to split_queue.
*/
- if (did_split) {
- ; /* folio already removed from list */
- } else if (!folio_test_partially_mapped(folio)) {
- list_del_init(&folio->_deferred_list);
- removed++;
- } else {
- /*
- * That unlocked list_del_init() above would be unsafe,
- * unless its folio is separated from any earlier folios
- * left on the list (which may be concurrently unqueued)
- * by one safe folio with refcount still raised.
- */
- swap(folio, prev);
+ fqueue = folio_split_queue_lock_irqsave(folio, &flags);
+ if (list_empty(&folio->_deferred_list)) {
+ list_add_tail(&folio->_deferred_list, &fqueue->split_queue);
+ fqueue->split_queue_len++;
}
- if (folio)
- folio_put(folio);
+ split_queue_unlock_irqrestore(fqueue, flags);
}
+ folios_put(&fbatch);
- spin_lock_irqsave(&ds_queue->split_queue_lock, flags);
- list_splice_tail(&list, &ds_queue->split_queue);
- ds_queue->split_queue_len -= removed;
- spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags);
-
- if (prev)
- folio_put(prev);
+ if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) {
+ cond_resched();
+ goto retry;
+ }
/*
* Stop shrinker if we didn't split any page, but the queue is empty.
@@ -4408,7 +4468,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
* can be split or not. So skip the check here.
*/
if (!folio_test_private(folio) &&
- !can_split_folio(folio, 0, NULL))
+ folio_expected_ref_count(folio) != folio_ref_count(folio))
goto next;
if (!folio_trylock(folio))
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e9f4cbf50a2b..d969c9ed91b1 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2140,7 +2140,8 @@ retry:
if (h->surplus_huge_pages_node[folio_nid(folio)])
adjust_surplus = true;
remove_hugetlb_folio(h, folio, adjust_surplus);
- h->max_huge_pages--;
+ if (!adjust_surplus)
+ h->max_huge_pages--;
spin_unlock_irq(&hugetlb_lock);
/*
@@ -2160,7 +2161,8 @@ retry:
if (rc) {
spin_lock_irq(&hugetlb_lock);
add_hugetlb_folio(h, folio, adjust_surplus);
- h->max_huge_pages++;
+ if (!adjust_surplus)
+ h->max_huge_pages++;
goto out;
}
} else
@@ -3451,7 +3453,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start,
gather_bootmem_prealloc_node(nid);
}
-static void __init gather_bootmem_prealloc(void)
+void __init hugetlb_bootmem_struct_page_init(void)
{
struct padata_mt_job job = {
.thread_fn = gather_bootmem_prealloc_parallel,
@@ -3648,7 +3650,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
* - For gigantic pages, this is called early in the boot process and
* pages are allocated from memblock allocated or something similar.
* Gigantic pages are actually added to pools later with the routine
- * gather_bootmem_prealloc.
+ * hugetlb_bootmem_struct_page_init.
* - For non-gigantic pages, this is called later in the boot process after
* all of mm is up and functional. Pages are allocated from buddy and
* then added to hugetlb pools.
@@ -4715,7 +4717,6 @@ static int __init hugetlb_init(void)
hugetlb_cma_check();
hugetlb_init_hstates();
- gather_bootmem_prealloc();
report_hugepages();
hugetlb_sysfs_init();
@@ -7916,14 +7917,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re
* There is no need to transfer the per-node surplus state
* when we do not cross the node.
*/
- if (new_nid == old_nid)
- return;
- spin_lock_irq(&hugetlb_lock);
- if (h->surplus_huge_pages_node[old_nid]) {
- h->surplus_huge_pages_node[old_nid]--;
- h->surplus_huge_pages_node[new_nid]++;
+ if (new_nid != old_nid) {
+ spin_lock_irq(&hugetlb_lock);
+ if (h->surplus_huge_pages_node[old_nid]) {
+ h->surplus_huge_pages_node[old_nid]--;
+ h->surplus_huge_pages_node[new_nid]++;
+ }
+ spin_unlock_irq(&hugetlb_lock);
}
- spin_unlock_irq(&hugetlb_lock);
}
/*
diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c
index 58e895f3899a..b6537a3e4f62 100644
--- a/mm/hugetlb_cgroup.c
+++ b/mm/hugetlb_cgroup.c
@@ -104,6 +104,7 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup,
struct page_counter *fault, *fault_parent = NULL;
struct page_counter *rsvd, *rsvd_parent = NULL;
unsigned long limit;
+ int ret;
if (parent_h_cgroup) {
fault_parent = hugetlb_cgroup_counter_from_cgroup(
@@ -125,8 +126,10 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup,
limit = round_down(PAGE_COUNTER_MAX,
pages_per_huge_page(&hstates[idx]));
- VM_BUG_ON(page_counter_set_max(fault, limit));
- VM_BUG_ON(page_counter_set_max(rsvd, limit));
+ ret = page_counter_set_max(fault, limit);
+ VM_WARN_ON_ONCE(ret);
+ ret = page_counter_set_max(rsvd, limit);
+ VM_WARN_ON_ONCE(ret);
}
}
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 96ee2bd16ee1..795e7dbbca7d 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -24,8 +24,8 @@
*
* @remap_pte: called for each lowest-level entry (PTE).
* @nr_walked: the number of walked pte.
- * @reuse_page: the page which is reused for the tail vmemmap pages.
- * @reuse_addr: the virtual address of the @reuse_page page.
+ * @vmemmap_head: the page to be installed as first in the vmemmap range
+ * @vmemmap_tail: the page to be installed as non-first in the vmemmap range
* @vmemmap_pages: the list head of the vmemmap pages that can be freed
* or is mapped from.
* @flags: used to modify behavior in vmemmap page table walking
@@ -34,17 +34,17 @@
struct vmemmap_remap_walk {
void (*remap_pte)(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk);
+
unsigned long nr_walked;
- struct page *reuse_page;
- unsigned long reuse_addr;
+ struct page *vmemmap_head;
+ struct page *vmemmap_tail;
struct list_head *vmemmap_pages;
+
/* Skip the TLB flush when we split the PMD */
#define VMEMMAP_SPLIT_NO_TLB_FLUSH BIT(0)
/* Skip the TLB flush when we remap the PTE */
#define VMEMMAP_REMAP_NO_TLB_FLUSH BIT(1)
-/* synchronize_rcu() to avoid writes from page_ref_add_unless() */
-#define VMEMMAP_SYNCHRONIZE_RCU BIT(2)
unsigned long flags;
};
@@ -140,14 +140,7 @@ static int vmemmap_pte_entry(pte_t *pte, unsigned long addr,
{
struct vmemmap_remap_walk *vmemmap_walk = walk->private;
- /*
- * The reuse_page is found 'first' in page table walking before
- * starting remapping.
- */
- if (!vmemmap_walk->reuse_page)
- vmemmap_walk->reuse_page = pte_page(ptep_get(pte));
- else
- vmemmap_walk->remap_pte(pte, addr, vmemmap_walk);
+ vmemmap_walk->remap_pte(pte, addr, vmemmap_walk);
vmemmap_walk->nr_walked++;
return 0;
@@ -207,18 +200,14 @@ static void free_vmemmap_page_list(struct list_head *list)
static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk)
{
- /*
- * Remap the tail pages as read-only to catch illegal write operation
- * to the tail pages.
- */
- pgprot_t pgprot = PAGE_KERNEL_RO;
struct page *page = pte_page(ptep_get(pte));
pte_t entry;
/* Remapping the head page requires r/w */
- if (unlikely(addr == walk->reuse_addr)) {
- pgprot = PAGE_KERNEL;
- list_del(&walk->reuse_page->lru);
+ if (unlikely(walk->nr_walked == 0 && walk->vmemmap_head)) {
+ VM_WARN_ON_ONCE(!PageHead((const struct page *)addr));
+
+ list_del(&walk->vmemmap_head->lru);
/*
* Makes sure that preceding stores to the page contents from
@@ -226,53 +215,47 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
* write.
*/
smp_wmb();
+
+ entry = mk_pte(walk->vmemmap_head, PAGE_KERNEL);
+ } else {
+ VM_WARN_ON_ONCE(!PageTail((const struct page *)addr));
+
+ /*
+ * Remap the tail pages as read-only to catch illegal write
+ * operation to the tail pages.
+ */
+ entry = mk_pte(walk->vmemmap_tail, PAGE_KERNEL_RO);
}
- entry = mk_pte(walk->reuse_page, pgprot);
list_add(&page->lru, walk->vmemmap_pages);
set_pte_at(&init_mm, addr, pte, entry);
}
-/*
- * How many struct page structs need to be reset. When we reuse the head
- * struct page, the special metadata (e.g. page->flags or page->mapping)
- * cannot copy to the tail struct page structs. The invalid value will be
- * checked in the free_tail_page_prepare(). In order to avoid the message
- * of "corrupted mapping in tail page". We need to reset at least 4 (one
- * head struct page struct and three tail struct page structs) struct page
- * structs.
- */
-#define NR_RESET_STRUCT_PAGE 4
-
-static inline void reset_struct_pages(struct page *start)
-{
- struct page *from = start + NR_RESET_STRUCT_PAGE;
-
- BUILD_BUG_ON(NR_RESET_STRUCT_PAGE * 2 > PAGE_SIZE / sizeof(struct page));
- memcpy(start, from, sizeof(*from) * NR_RESET_STRUCT_PAGE);
-}
-
static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk)
{
- pgprot_t pgprot = PAGE_KERNEL;
- struct page *page;
- void *to;
+ struct page *src = pte_page(ptep_get(pte)), *dst;
+
+ /*
+ * When rolling back vmemmap_remap_free(), keep the copied head page
+ * mapping and restore only PTEs currently pointing at the shared tail
+ * page.
+ */
+ if (walk->vmemmap_tail && walk->vmemmap_tail != src)
+ return;
- BUG_ON(pte_page(ptep_get(pte)) != walk->reuse_page);
+ VM_WARN_ON_ONCE(PageHead((const struct page *)addr));
- page = list_first_entry(walk->vmemmap_pages, struct page, lru);
- list_del(&page->lru);
- to = page_to_virt(page);
- copy_page(to, (void *)walk->reuse_addr);
- reset_struct_pages(to);
+ dst = list_first_entry(walk->vmemmap_pages, struct page, lru);
+ list_del(&dst->lru);
+ copy_page(page_to_virt(dst), page_to_virt(src));
/*
* Makes sure that preceding stores to the page contents become visible
* before the set_pte_at() write.
*/
smp_wmb();
- set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
+ set_pte_at(&init_mm, addr, pte, mk_pte(dst, PAGE_KERNEL));
}
/**
@@ -282,33 +265,28 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
* to remap.
* @end: end address of the vmemmap virtual address range that we want to
* remap.
- * @reuse: reuse address.
- *
* Return: %0 on success, negative error code otherwise.
*/
-static int vmemmap_remap_split(unsigned long start, unsigned long end,
- unsigned long reuse)
+static int vmemmap_remap_split(unsigned long start, unsigned long end)
{
struct vmemmap_remap_walk walk = {
.remap_pte = NULL,
.flags = VMEMMAP_SPLIT_NO_TLB_FLUSH,
};
- /* See the comment in the vmemmap_remap_free(). */
- BUG_ON(start - reuse != PAGE_SIZE);
-
- return vmemmap_remap_range(reuse, end, &walk);
+ return vmemmap_remap_range(start, end, &walk);
}
/**
* vmemmap_remap_free - remap the vmemmap virtual address range [@start, @end)
- * to the page which @reuse is mapped to, then free vmemmap
- * which the range are mapped to.
+ * to use @vmemmap_head/tail, then free vmemmap which
+ * the range are mapped to.
* @start: start address of the vmemmap virtual address range that we want
* to remap.
* @end: end address of the vmemmap virtual address range that we want to
* remap.
- * @reuse: reuse address.
+ * @vmemmap_head: the page to be installed as first in the vmemmap range
+ * @vmemmap_tail: the page to be installed as non-first in the vmemmap range
* @vmemmap_pages: list to deposit vmemmap pages to be freed. It is callers
* responsibility to free pages.
* @flags: modifications to vmemmap_remap_walk flags
@@ -316,69 +294,39 @@ static int vmemmap_remap_split(unsigned long start, unsigned long end,
* Return: %0 on success, negative error code otherwise.
*/
static int vmemmap_remap_free(unsigned long start, unsigned long end,
- unsigned long reuse,
+ struct page *vmemmap_head,
+ struct page *vmemmap_tail,
struct list_head *vmemmap_pages,
unsigned long flags)
{
int ret;
struct vmemmap_remap_walk walk = {
.remap_pte = vmemmap_remap_pte,
- .reuse_addr = reuse,
+ .vmemmap_head = vmemmap_head,
+ .vmemmap_tail = vmemmap_tail,
.vmemmap_pages = vmemmap_pages,
.flags = flags,
};
- int nid = page_to_nid((struct page *)reuse);
- gfp_t gfp_mask = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
- /*
- * Allocate a new head vmemmap page to avoid breaking a contiguous
- * block of struct page memory when freeing it back to page allocator
- * in free_vmemmap_page_list(). This will allow the likely contiguous
- * struct page backing memory to be kept contiguous and allowing for
- * more allocations of hugepages. Fallback to the currently
- * mapped head page in case should it fail to allocate.
- */
- walk.reuse_page = alloc_pages_node(nid, gfp_mask, 0);
- if (walk.reuse_page) {
- copy_page(page_to_virt(walk.reuse_page),
- (void *)walk.reuse_addr);
- list_add(&walk.reuse_page->lru, vmemmap_pages);
- memmap_pages_add(1);
- }
+ ret = vmemmap_remap_range(start, end, &walk);
+ if (!ret || !walk.nr_walked)
+ return ret;
+
+ end = start + walk.nr_walked * PAGE_SIZE;
/*
- * In order to make remapping routine most efficient for the huge pages,
- * the routine of vmemmap page table walking has the following rules
- * (see more details from the vmemmap_pte_range()):
- *
- * - The range [@start, @end) and the range [@reuse, @reuse + PAGE_SIZE)
- * should be continuous.
- * - The @reuse address is part of the range [@reuse, @end) that we are
- * walking which is passed to vmemmap_remap_range().
- * - The @reuse address is the first in the complete range.
- *
- * So we need to make sure that @start and @reuse meet the above rules.
+ * vmemmap_pages contains pages from the previous vmemmap_remap_range()
+ * call which failed. These are pages which were removed from
+ * the vmemmap. They will be restored in the following call.
*/
- BUG_ON(start - reuse != PAGE_SIZE);
+ walk = (struct vmemmap_remap_walk) {
+ .remap_pte = vmemmap_restore_pte,
+ .vmemmap_tail = vmemmap_tail,
+ .vmemmap_pages = vmemmap_pages,
+ .flags = 0,
+ };
- ret = vmemmap_remap_range(reuse, end, &walk);
- if (ret && walk.nr_walked) {
- end = reuse + walk.nr_walked * PAGE_SIZE;
- /*
- * vmemmap_pages contains pages from the previous
- * vmemmap_remap_range call which failed. These
- * are pages which were removed from the vmemmap.
- * They will be restored in the following call.
- */
- walk = (struct vmemmap_remap_walk) {
- .remap_pte = vmemmap_restore_pte,
- .reuse_addr = reuse,
- .vmemmap_pages = vmemmap_pages,
- .flags = 0,
- };
-
- vmemmap_remap_range(reuse, end, &walk);
- }
+ vmemmap_remap_range(start, end, &walk);
return ret;
}
@@ -415,29 +363,24 @@ out:
* to remap.
* @end: end address of the vmemmap virtual address range that we want to
* remap.
- * @reuse: reuse address.
* @flags: modifications to vmemmap_remap_walk flags
*
* Return: %0 on success, negative error code otherwise.
*/
static int vmemmap_remap_alloc(unsigned long start, unsigned long end,
- unsigned long reuse, unsigned long flags)
+ unsigned long flags)
{
LIST_HEAD(vmemmap_pages);
struct vmemmap_remap_walk walk = {
.remap_pte = vmemmap_restore_pte,
- .reuse_addr = reuse,
.vmemmap_pages = &vmemmap_pages,
.flags = flags,
};
- /* See the comment in the vmemmap_remap_free(). */
- BUG_ON(start - reuse != PAGE_SIZE);
-
if (alloc_vmemmap_page_list(start, end, &vmemmap_pages))
return -ENOMEM;
- return vmemmap_remap_range(reuse, end, &walk);
+ return vmemmap_remap_range(start, end, &walk);
}
DEFINE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
@@ -454,8 +397,7 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
struct folio *folio, unsigned long flags)
{
int ret;
- unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end;
- unsigned long vmemmap_reuse;
+ unsigned long vmemmap_start, vmemmap_end;
VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio);
@@ -463,21 +405,18 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
if (!folio_test_hugetlb_vmemmap_optimized(folio))
return 0;
- if (flags & VMEMMAP_SYNCHRONIZE_RCU)
- synchronize_rcu();
-
+ vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
- vmemmap_reuse = vmemmap_start;
+
vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE;
/*
* The pages which the vmemmap virtual address range [@vmemmap_start,
- * @vmemmap_end) are mapped to are freed to the buddy allocator, and
- * the range is mapped to the page which @vmemmap_reuse is mapped to.
+ * @vmemmap_end) are mapped to are freed to the buddy allocator.
* When a HugeTLB page is freed to the buddy allocator, previously
* discarded vmemmap pages must be allocated and remapping.
*/
- ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse, flags);
+ ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, flags);
if (!ret) {
folio_clear_hugetlb_vmemmap_optimized(folio);
static_branch_dec(&hugetlb_optimize_vmemmap_key);
@@ -498,7 +437,7 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
*/
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio)
{
- return __hugetlb_vmemmap_restore_folio(h, folio, VMEMMAP_SYNCHRONIZE_RCU);
+ return __hugetlb_vmemmap_restore_folio(h, folio, 0);
}
/**
@@ -521,14 +460,11 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
struct folio *folio, *t_folio;
long restored = 0;
long ret = 0;
- unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU;
+ unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
list_for_each_entry_safe(folio, t_folio, folio_list, lru) {
if (folio_test_hugetlb_vmemmap_optimized(folio)) {
ret = __hugetlb_vmemmap_restore_folio(h, folio, flags);
- /* only need to synchronize_rcu() once for each batch */
- flags &= ~VMEMMAP_SYNCHRONIZE_RCU;
-
if (ret)
break;
restored++;
@@ -565,9 +501,9 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
struct list_head *vmemmap_pages,
unsigned long flags)
{
- int ret = 0;
- unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end;
- unsigned long vmemmap_reuse;
+ unsigned long vmemmap_start, vmemmap_end;
+ struct page *vmemmap_head, *vmemmap_tail;
+ int nid, ret = 0;
VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio);
@@ -577,8 +513,6 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
static_branch_inc(&hugetlb_optimize_vmemmap_key);
- if (flags & VMEMMAP_SYNCHRONIZE_RCU)
- synchronize_rcu();
/*
* Very Subtle
* If VMEMMAP_REMAP_NO_TLB_FLUSH is set, TLB flushing is not performed
@@ -592,18 +526,30 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
*/
folio_set_hugetlb_vmemmap_optimized(folio);
+ nid = folio_nid(folio);
+ vmemmap_head = alloc_pages_node(nid, GFP_KERNEL, 0);
+ if (!vmemmap_head) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ copy_page(page_to_virt(vmemmap_head), folio);
+ list_add(&vmemmap_head->lru, vmemmap_pages);
+ memmap_pages_add(1);
+
+ vmemmap_tail = vmemmap_head;
+ vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
- vmemmap_reuse = vmemmap_start;
- vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE;
/*
- * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end)
- * to the page which @vmemmap_reuse is mapped to. Add pages previously
- * mapping the range to vmemmap_pages list so that they can be freed by
- * the caller.
+ * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end).
+ * Add pages previously mapping the range to vmemmap_pages list so that
+ * they can be freed by the caller.
*/
- ret = vmemmap_remap_free(vmemmap_start, vmemmap_end, vmemmap_reuse,
+ ret = vmemmap_remap_free(vmemmap_start, vmemmap_end,
+ vmemmap_head, vmemmap_tail,
vmemmap_pages, flags);
+out:
if (ret) {
static_branch_dec(&hugetlb_optimize_vmemmap_key);
folio_clear_hugetlb_vmemmap_optimized(folio);
@@ -626,27 +572,25 @@ void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio)
{
LIST_HEAD(vmemmap_pages);
- __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, VMEMMAP_SYNCHRONIZE_RCU);
+ __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, 0);
free_vmemmap_page_list(&vmemmap_pages);
}
static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *folio)
{
- unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end;
- unsigned long vmemmap_reuse;
+ unsigned long vmemmap_start, vmemmap_end;
if (!vmemmap_should_optimize_folio(h, folio))
return 0;
+ vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
- vmemmap_reuse = vmemmap_start;
- vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE;
/*
* Split PMDs on the vmemmap virtual address range [@vmemmap_start,
* @vmemmap_end]
*/
- return vmemmap_remap_split(vmemmap_start, vmemmap_end, vmemmap_reuse);
+ return vmemmap_remap_split(vmemmap_start, vmemmap_end);
}
static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
@@ -656,7 +600,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
struct folio *folio;
int nr_to_optimize;
LIST_HEAD(vmemmap_pages);
- unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU;
+ unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
nr_to_optimize = 0;
list_for_each_entry(folio, folio_list, lru) {
@@ -664,18 +608,19 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
unsigned long spfn, epfn;
if (boot && folio_test_hugetlb_vmemmap_optimized(folio)) {
+ static_branch_inc(&hugetlb_optimize_vmemmap_key);
+
/*
* Already optimized by pre-HVO, just map the
* mirrored tail page structs RO.
*/
spfn = (unsigned long)&folio->page;
- epfn = spfn + pages_per_huge_page(h);
+ epfn = spfn + hugetlb_vmemmap_size(h);
vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
HUGETLB_VMEMMAP_RESERVE_SIZE);
- register_page_bootmem_memmap(pfn_to_section_nr(spfn),
+ register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)),
&folio->page,
- HUGETLB_VMEMMAP_RESERVE_SIZE);
- static_branch_inc(&hugetlb_optimize_vmemmap_key);
+ HUGETLB_VMEMMAP_RESERVE_PAGES);
continue;
}
@@ -709,8 +654,6 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
int ret;
ret = __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags);
- /* only need to synchronize_rcu() once for each batch */
- flags &= ~VMEMMAP_SYNCHRONIZE_RCU;
/*
* Pages to be freed may have been accumulated. If we
@@ -789,7 +732,6 @@ void __init hugetlb_vmemmap_init_early(int nid)
{
unsigned long psize, paddr, section_size;
unsigned long ns, i, pnum, pfn, nr_pages;
- unsigned long start, end;
struct huge_bootmem_page *m = NULL;
void *map;
@@ -815,14 +757,6 @@ void __init hugetlb_vmemmap_init_early(int nid)
paddr = virt_to_phys(m);
pfn = PHYS_PFN(paddr);
map = pfn_to_page(pfn);
- start = (unsigned long)map;
- end = start + nr_pages * sizeof(struct page);
-
- if (vmemmap_populate_hvo(start, end, nid,
- HUGETLB_VMEMMAP_RESERVE_SIZE) < 0)
- continue;
-
- memmap_boot_pages_add(HUGETLB_VMEMMAP_RESERVE_SIZE / PAGE_SIZE);
pnum = pfn_to_section_nr(pfn);
ns = psize / section_size;
@@ -860,28 +794,36 @@ void __init hugetlb_vmemmap_init_late(int nid)
h = m->hstate;
pfn = PHYS_PFN(phys);
nr_pages = pages_per_huge_page(h);
+ map = pfn_to_page(pfn);
+ start = (unsigned long)map;
+ end = start + nr_pages * sizeof(struct page);
if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
/*
* Oops, the hugetlb page spans multiple zones.
- * Remove it from the list, and undo HVO.
+ * Remove it from the list, and populate it normally.
*/
list_del(&m->list);
- map = pfn_to_page(pfn);
-
- start = (unsigned long)map;
- end = start + nr_pages * sizeof(struct page);
-
- vmemmap_undo_hvo(start, end, nid,
- HUGETLB_VMEMMAP_RESERVE_SIZE);
- nr_mmap = end - start - HUGETLB_VMEMMAP_RESERVE_SIZE;
+ vmemmap_populate(start, end, nid, NULL);
+ nr_mmap = end - start;
memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
memblock_phys_free(phys, huge_page_size(h));
continue;
- } else
+ }
+
+ if (vmemmap_populate_hvo(start, end, nid,
+ HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) {
+ /* Fallback if HVO population fails */
+ vmemmap_populate(start, end, nid, NULL);
+ nr_mmap = end - start;
+ } else {
m->flags |= HUGE_BOOTMEM_ZONES_VALID;
+ nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE;
+ }
+
+ memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
}
}
#endif
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 74d3a4d78ec6..af8c9289e8e0 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -95,7 +95,6 @@ enum {
RES_LIMIT,
RES_MAX_USAGE,
RES_FAILCNT,
- RES_SOFT_LIMIT,
};
#ifdef CONFIG_LOCKDEP
@@ -1547,6 +1546,30 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
return -EINVAL;
}
+static u64 mem_cgroup_soft_limit_read(struct cgroup_subsys_state *css,
+ struct cftype *cft)
+{
+ return (u64)PAGE_COUNTER_MAX * PAGE_SIZE;
+}
+
+static ssize_t mem_cgroup_soft_limit_write(struct kernfs_open_file *of,
+ char *buf, size_t nbytes, loff_t off)
+{
+ unsigned long nr_pages;
+ int ret;
+
+ ret = page_counter_memparse(strstrip(buf), "-1", &nr_pages);
+ if (ret)
+ return ret;
+
+ pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. "
+ "Writing any value to this file has no effect. "
+ "Please report your usecase to linux-mm@kvack.org if you "
+ "depend on this functionality.\n");
+
+ return nbytes;
+}
+
static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
struct cftype *cft)
{
@@ -1583,8 +1606,6 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
return (u64)counter->watermark * PAGE_SIZE;
case RES_FAILCNT:
return counter->failcnt;
- case RES_SOFT_LIMIT:
- return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE;
default:
BUG();
}
@@ -1679,17 +1700,6 @@ static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
break;
}
break;
- case RES_SOFT_LIMIT:
- if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
- ret = -EOPNOTSUPP;
- } else {
- pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. "
- "Please report your usecase to linux-mm@kvack.org if you "
- "depend on this functionality.\n");
- WRITE_ONCE(memcg->soft_limit, nr_pages);
- ret = 0;
- }
- break;
}
return ret ?: nbytes;
}
@@ -2027,9 +2037,8 @@ struct cftype mem_cgroup_legacy_files[] = {
},
{
.name = "soft_limit_in_bytes",
- .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
- .write = mem_cgroup_write,
- .read_u64 = mem_cgroup_read_u64,
+ .write = mem_cgroup_soft_limit_write,
+ .read_u64 = mem_cgroup_soft_limit_read,
},
{
.name = "failcnt",
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 898821793349..a4a3da6f241c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2352,6 +2352,19 @@ retry:
if (!gfpflags_allow_blocking(gfp_mask))
goto nomem;
+ /*
+ * OOM victim still needs to charge memory to exit. OOM reaper should
+ * help but it might fail on mmap_lock contention. If the victim is a
+ * large thread group then all exiting threads might compete on oom_lock
+ * just to learn that there is nothing really killable anymore. Bail
+ * out early and fail the charge to expedite their exit. They are
+ * considered fully reclaimed by the oom reaper and they shouldn't
+ * contribute further charges.
+ */
+ if (tsk_is_oom_victim(current) &&
+ mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm))
+ goto nomem;
+
__memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
raised_max_event = true;
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index d18cee845354..2a84cecf5435 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -2606,7 +2606,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
prev_node = node;
/* create a local copy of node weights to operate on outside rcu */
- weights = kzalloc(nr_node_ids, GFP_KERNEL);
+ weights = kmalloc(nr_node_ids, gfp & GFP_RECLAIM_MASK);
if (!weights)
return total_allocated;
diff --git a/mm/migrate.c b/mm/migrate.c
index d03b5d265e71..3159be53a00d 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -451,11 +451,12 @@ static bool remove_migration_pte(struct folio *folio,
* Get rid of all migration entries and replace them by
* references to the indicated page.
*/
-void remove_migration_ptes(struct folio *src, struct folio *dst, int flags)
+void remove_migration_ptes(struct folio *src, struct folio *dst,
+ enum ttu_flags flags)
{
struct rmap_walk_arg rmap_walk_arg = {
.folio = src,
- .map_unused_to_zeropage = flags & RMP_USE_SHARED_ZEROPAGE,
+ .map_unused_to_zeropage = flags & TTU_USE_SHARED_ZEROPAGE,
};
struct rmap_walk_control rwc = {
@@ -463,9 +464,9 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, int flags)
.arg = &rmap_walk_arg,
};
- VM_BUG_ON_FOLIO((flags & RMP_USE_SHARED_ZEROPAGE) && (src != dst), src);
+ VM_BUG_ON_FOLIO((flags & TTU_USE_SHARED_ZEROPAGE) && (src != dst), src);
- if (flags & RMP_LOCKED)
+ if (flags & TTU_RMAP_LOCKED)
rmap_walk_locked(dst, &rwc);
else
rmap_walk(dst, &rwc);
@@ -1530,8 +1531,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio,
rc = move_to_new_folio(dst, src, mode);
if (page_was_mapped)
- remove_migration_ptes(src, !rc ? dst : src,
- ttu ? RMP_LOCKED : 0);
+ remove_migration_ptes(src, !rc ? dst : src, ttu);
if (ttu & TTU_RMAP_LOCKED)
i_mmap_unlock_write(mapping);
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 922493a35a28..5789ad8ba9b6 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2357,6 +2357,7 @@ void __init page_alloc_init_late(void)
/* Reinit limits that are based on free pages after the kernel is up */
files_maxfiles_init();
#endif
+ hugetlb_bootmem_struct_page_init();
/* Accounting of total+free memory is stable at this point. */
mem_init_print_info();
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index af37e0c2d6f8..ec5ca89cbc00 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2965,8 +2965,10 @@ static void __free_frozen_pages(struct page *page, unsigned int order,
migratetype = MIGRATE_MOVABLE;
}
- if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT)
- && (in_nmi() || in_hardirq()))) {
+ if (unlikely((fpi_flags & FPI_TRYLOCK) &&
+ ((IS_ENABLED(CONFIG_PREEMPT_RT) &&
+ (in_nmi() || in_hardirq())) ||
+ (!IS_ENABLED(CONFIG_SMP) && in_nmi())))) {
add_page_to_zone_llist(zone, page, order);
return;
}
diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index 2fdd31a5a48f..41cf3192a8f6 100644
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -104,7 +104,13 @@ again:
static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr)
{
unsigned long pfn;
- pte_t ptent = ptep_get(pvmw->pte);
+ pte_t ptent;
+
+ if (is_vm_hugetlb_page(pvmw->vma))
+ ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address,
+ pvmw->pte);
+ else
+ ptent = ptep_get(pvmw->pte);
if (pvmw->flags & PVMW_MIGRATION) {
swp_entry_t entry;
diff --git a/mm/rmap.c b/mm/rmap.c
index 4348b35db465..29f93ecf3480 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2384,20 +2384,23 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
- /*
- * Handle PFN swap PTEs, such as device-exclusive ones, that
- * actually map pages.
- */
- pteval = ptep_get(pvmw.pte);
+ address = pvmw.address;
+ if (folio_test_hugetlb(folio))
+ pteval = huge_ptep_get(mm, address, pvmw.pte);
+ else
+ pteval = ptep_get(pvmw.pte);
if (likely(pte_present(pteval))) {
pfn = pte_pfn(pteval);
} else {
+ /*
+ * Handle PFN swap PTEs, such as device-exclusive ones,
+ * that actually map pages.
+ */
pfn = swp_offset_pfn(pte_to_swp_entry(pteval));
VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
}
subpage = folio_page(folio, pfn - folio_pfn(folio));
- address = pvmw.address;
anon_exclusive = folio_test_anon(folio) &&
PageAnonExclusive(subpage);
diff --git a/mm/slab.h b/mm/slab.h
index c452f2a32c80..0b49960bf07c 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -299,9 +299,12 @@ struct kmem_cache {
#define SLAB_SUPPORTS_SYSFS 1
void sysfs_slab_unlink(struct kmem_cache *s);
void sysfs_slab_release(struct kmem_cache *s);
+int sysfs_slab_alias(struct kmem_cache *s, const char *name);
#else
static inline void sysfs_slab_unlink(struct kmem_cache *s) { }
static inline void sysfs_slab_release(struct kmem_cache *s) { }
+static inline int sysfs_slab_alias(struct kmem_cache *s, const char *name)
+ { return 0; }
#endif
void *fixup_red_left(struct kmem_cache *s, void *p);
@@ -422,11 +425,6 @@ extern void create_boot_cache(struct kmem_cache *, const char *name,
unsigned int useroffset, unsigned int usersize);
int slab_unmergeable(struct kmem_cache *s);
-struct kmem_cache *find_mergeable(unsigned size, unsigned align,
- slab_flags_t flags, const char *name, void (*ctor)(void *));
-struct kmem_cache *
-__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
- slab_flags_t flags, void (*ctor)(void *));
slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name);
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 085af9a96fd9..edd49796be4c 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -175,15 +175,22 @@ int slab_unmergeable(struct kmem_cache *s)
return 0;
}
-struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
- slab_flags_t flags, const char *name, void (*ctor)(void *))
+static struct kmem_cache *find_mergeable(unsigned int size, slab_flags_t flags,
+ const char *name, struct kmem_cache_args *args)
{
struct kmem_cache *s;
+ unsigned int align;
if (slab_nomerge)
return NULL;
- if (ctor)
+ if (args->ctor)
+ return NULL;
+
+ if (IS_ENABLED(CONFIG_HARDENED_USERCOPY) && args->usersize)
+ return NULL;
+
+ if (args->sheaf_capacity)
return NULL;
flags = kmem_cache_flags(flags, name);
@@ -192,7 +199,7 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
return NULL;
size = ALIGN(size, sizeof(void *));
- align = calculate_alignment(flags, align, size);
+ align = calculate_alignment(flags, args->align, size);
size = ALIGN(size, align);
list_for_each_entry_reverse(s, &slab_caches, list) {
@@ -253,6 +260,31 @@ out:
return ERR_PTR(err);
}
+static struct kmem_cache *
+__kmem_cache_alias(const char *name, unsigned int size, slab_flags_t flags,
+ struct kmem_cache_args *args)
+{
+ struct kmem_cache *s;
+
+ s = find_mergeable(size, flags, name, args);
+ if (s) {
+ if (sysfs_slab_alias(s, name))
+ pr_err("SLUB: Unable to add cache alias %s to sysfs\n",
+ name);
+
+ s->refcount++;
+
+ /*
+ * Adjust the object sizes so that we clear
+ * the complete object on kzalloc.
+ */
+ s->object_size = max(s->object_size, size);
+ s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
+ }
+
+ return s;
+}
+
/**
* __kmem_cache_create_args - Create a kmem cache.
* @name: A string which is used in /proc/slabinfo to identify this cache.
@@ -324,9 +356,7 @@ struct kmem_cache *__kmem_cache_create_args(const char *name,
object_size - args->usersize < args->useroffset))
args->usersize = args->useroffset = 0;
- if (!args->usersize && !args->sheaf_capacity)
- s = __kmem_cache_alias(name, object_size, args->align, flags,
- args->ctor);
+ s = __kmem_cache_alias(name, object_size, flags, args);
if (s)
goto out_unlock;
diff --git a/mm/slub.c b/mm/slub.c
index 5ddd2a5c9e79..8fdbe531a0cb 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -351,17 +351,20 @@ enum track_item { TRACK_ALLOC, TRACK_FREE };
#ifdef SLAB_SUPPORTS_SYSFS
static int sysfs_slab_add(struct kmem_cache *);
-static int sysfs_slab_alias(struct kmem_cache *, const char *);
+static int __init slab_kset_init(void);
+static void __init slab_sysfs_process_aliases(void);
#else
static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
-static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
- { return 0; }
+static inline int slab_kset_init(void) { return 0; }
+static inline void slab_sysfs_process_aliases(void) { }
#endif
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
static void debugfs_slab_add(struct kmem_cache *);
+static void __init slab_debugfs_root_init(void);
#else
static inline void debugfs_slab_add(struct kmem_cache *s) { }
+static inline void slab_debugfs_root_init(void) { }
#endif
enum stat_item {
@@ -8533,31 +8536,6 @@ void __init kmem_cache_init_late(void)
#endif
}
-struct kmem_cache *
-__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
- slab_flags_t flags, void (*ctor)(void *))
-{
- struct kmem_cache *s;
-
- s = find_mergeable(size, align, flags, name, ctor);
- if (s) {
- if (sysfs_slab_alias(s, name))
- pr_err("SLUB: Unable to add cache alias %s to sysfs\n",
- name);
-
- s->refcount++;
-
- /*
- * Adjust the object sizes so that we clear
- * the complete object on kzalloc.
- */
- s->object_size = max(s->object_size, size);
- s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
- }
-
- return s;
-}
-
int do_kmem_cache_create(struct kmem_cache *s, const char *name,
unsigned int size, struct kmem_cache_args *args,
slab_flags_t flags)
@@ -9790,7 +9768,7 @@ struct saved_alias {
static struct saved_alias *alias_list;
-static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
+int sysfs_slab_alias(struct kmem_cache *s, const char *name)
{
struct saved_alias *al;
@@ -9819,28 +9797,20 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
return 0;
}
-static int __init slab_sysfs_init(void)
+static int __init slab_kset_init(void)
{
- struct kmem_cache *s;
- int err;
-
- mutex_lock(&slab_mutex);
-
slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
if (!slab_kset) {
- mutex_unlock(&slab_mutex);
pr_err("Cannot register slab subsystem.\n");
return -ENOMEM;
}
- slab_state = FULL;
+ return 0;
+}
- list_for_each_entry(s, &slab_caches, list) {
- err = sysfs_slab_add(s);
- if (err)
- pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
- s->name);
- }
+static void __init slab_sysfs_process_aliases(void)
+{
+ int err;
while (alias_list) {
struct saved_alias *al = alias_list;
@@ -9852,12 +9822,41 @@ static int __init slab_sysfs_init(void)
al->name);
kfree(al);
}
+}
+#endif /* SLAB_SUPPORTS_SYSFS */
+
+#if defined(SLAB_SUPPORTS_SYSFS) || \
+ (defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS))
+static int __init slab_late_init(void)
+{
+ struct kmem_cache *s;
+ int err;
+
+ mutex_lock(&slab_mutex);
+
+ err = slab_kset_init();
+ if (err)
+ goto out;
+
+ slab_debugfs_root_init();
+ slab_state = FULL;
+
+ list_for_each_entry(s, &slab_caches, list) {
+ if (sysfs_slab_add(s))
+ pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
+ s->name);
+
+ if (s->flags & SLAB_STORE_USER)
+ debugfs_slab_add(s);
+ }
+ slab_sysfs_process_aliases();
+out:
mutex_unlock(&slab_mutex);
- return 0;
+ return err;
}
-late_initcall(slab_sysfs_init);
-#endif /* SLAB_SUPPORTS_SYSFS */
+late_initcall(slab_late_init);
+#endif
#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
static int slab_debugfs_show(struct seq_file *seq, void *v)
@@ -10049,23 +10048,16 @@ static void debugfs_slab_add(struct kmem_cache *s)
void debugfs_slab_release(struct kmem_cache *s)
{
+ if (unlikely(!slab_debugfs_root))
+ return;
+
debugfs_lookup_and_remove(s->name, slab_debugfs_root);
}
-static int __init slab_debugfs_init(void)
+static void __init slab_debugfs_root_init(void)
{
- struct kmem_cache *s;
-
slab_debugfs_root = debugfs_create_dir("slab", NULL);
-
- list_for_each_entry(s, &slab_caches, list)
- if (s->flags & SLAB_STORE_USER)
- debugfs_slab_add(s);
-
- return 0;
-
}
-__initcall(slab_debugfs_init);
#endif
/*
* The /proc/slabinfo ABI
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 37522d6cb398..032a81450838 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -303,59 +303,6 @@ int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
}
/*
- * Undo populate_hvo, and replace it with a normal base page mapping.
- * Used in memory init in case a HVO mapping needs to be undone.
- *
- * This can happen when it is discovered that a memblock allocated
- * hugetlb page spans multiple zones, which can only be verified
- * after zones have been initialized.
- *
- * We know that:
- * 1) The first @headsize / PAGE_SIZE vmemmap pages were individually
- * allocated through memblock, and mapped.
- *
- * 2) The rest of the vmemmap pages are mirrors of the last head page.
- */
-int __meminit vmemmap_undo_hvo(unsigned long addr, unsigned long end,
- int node, unsigned long headsize)
-{
- unsigned long maddr, pfn;
- pte_t *pte;
- int headpages;
-
- /*
- * Should only be called early in boot, so nothing will
- * be accessing these page structures.
- */
- WARN_ON(!early_boot_irqs_disabled);
-
- headpages = headsize >> PAGE_SHIFT;
-
- /*
- * Clear mirrored mappings for tail page structs.
- */
- for (maddr = addr + headsize; maddr < end; maddr += PAGE_SIZE) {
- pte = virt_to_kpte(maddr);
- pte_clear(&init_mm, maddr, pte);
- }
-
- /*
- * Clear and free mappings for head page and first tail page
- * structs.
- */
- for (maddr = addr; headpages-- > 0; maddr += PAGE_SIZE) {
- pte = virt_to_kpte(maddr);
- pfn = pte_pfn(ptep_get(pte));
- pte_clear(&init_mm, maddr, pte);
- memblock_phys_free(PFN_PHYS(pfn), PAGE_SIZE);
- }
-
- flush_tlb_kernel_range(addr, end);
-
- return vmemmap_populate(addr, end, node, NULL);
-}
-
-/*
* Write protect the mirrored tail page structs for HVO. This will be
* called from the hugetlb code when gathering and initializing the
* memblock allocated gigantic pages. The write protect can't be
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 1b0fa239aa75..34798ce783e6 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -1956,8 +1956,10 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start,
}
if (err) {
- if (err == -EAGAIN)
+ if (err == -EAGAIN) {
+ err = 0;
continue;
+ }
break;
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 05027b320eda..2c66fbcd6e84 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1312,7 +1312,8 @@ retry:
goto keep_locked;
if (folio_test_large(folio)) {
/* cannot split folio, skip it */
- if (!can_split_folio(folio, 1, NULL))
+ if (folio_expected_ref_count(folio) !=
+ folio_ref_count(folio) - 1)
goto activate_locked;
/*
* Split partially mapped folios right away.
@@ -4468,8 +4469,6 @@ void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
int tier_idx)
{
- bool success;
- bool dirty, writeback;
int gen = folio_lru_gen(folio);
int type = folio_is_file_lru(folio);
int zone = folio_zonenum(folio);
@@ -4481,15 +4480,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
- /* unevictable */
- if (!folio_evictable(folio)) {
- success = lru_gen_del_folio(lruvec, folio, true);
- VM_WARN_ON_ONCE_FOLIO(!success, folio);
- folio_set_unevictable(folio);
- lruvec_add_folio(lruvec, folio);
- __count_vm_events(UNEVICTABLE_PGCULLED, delta);
- return true;
- }
+ /* unevictable: let it through and the generic path will cull it */
+ if (!folio_evictable(folio))
+ return false;
/* promoted */
if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
@@ -4519,21 +4512,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
return true;
}
- dirty = folio_test_dirty(folio);
- writeback = folio_test_writeback(folio);
- if (type == LRU_GEN_FILE && dirty) {
- sc->nr.file_taken += delta;
- if (!writeback)
- sc->nr.unqueued_dirty += delta;
- }
-
- /* waiting for writeback */
- if (writeback || (type == LRU_GEN_FILE && dirty)) {
- gen = folio_inc_gen(lruvec, folio, true);
- list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
- return true;
- }
-
return false;
}
@@ -4561,9 +4539,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
if (!folio_test_referenced(folio))
set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
- /* for shrink_folio_list() */
- folio_clear_reclaim(folio);
-
success = lru_gen_del_folio(lruvec, folio, true);
VM_WARN_ON_ONCE_FOLIO(!success, folio);
@@ -4757,11 +4732,9 @@ retry:
list_for_each_entry_safe_reverse(folio, next, &list, lru) {
DEFINE_MIN_SEQ(lruvec);
- if (!folio_evictable(folio)) {
- list_del(&folio->lru);
- folio_putback_lru(folio);
+ /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
+ if (!folio_evictable(folio))
continue;
- }
/* retry folios that may have missed folio_rotate_reclaimable() */
if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-11 15:18:30 +0000