summaryrefslogtreecommitdiff
path: root/mm/damon
diff options
context:
space:
mode:
Diffstat (limited to 'mm/damon')
-rw-r--r--mm/damon/core.c22
-rw-r--r--mm/damon/ops-common.c5
-rw-r--r--mm/damon/paddr.c20
-rw-r--r--mm/damon/sysfs-schemes.c18
-rw-r--r--mm/damon/sysfs.c23
-rw-r--r--mm/damon/tests/core-kunit.h15
-rw-r--r--mm/damon/tests/vaddr-kunit.h5
-rw-r--r--mm/damon/vaddr.c34
8 files changed, 77 insertions, 65 deletions
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 70ac1f08753d..a9168a03b340 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -412,6 +412,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
INIT_LIST_HEAD(&scheme->filters);
INIT_LIST_HEAD(&scheme->ops_filters);
scheme->stat = (struct damos_stat){};
+ scheme->last_applied = NULL;
INIT_LIST_HEAD(&scheme->list);
scheme->quota = *(damos_quota_init(quota));
@@ -2364,17 +2365,19 @@ static void damon_merge_two_regions(struct damon_target *t,
* sz_limit size upper limit of each region
*/
static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
- unsigned long sz_limit)
+ unsigned long sz_limit, bool count_age)
{
struct damon_region *r, *prev = NULL, *next;
damon_for_each_region_safe(r, next, t) {
- if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
- r->age = 0;
- else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0))
- r->age = 0;
- else
- r->age++;
+ if (count_age) {
+ if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
+ r->age = 0;
+ else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0))
+ r->age = 0;
+ else
+ r->age++;
+ }
if (prev && prev->ar.end == r->ar.start &&
abs(prev->nr_accesses - r->nr_accesses) <= thres &&
@@ -2408,15 +2411,18 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
struct damon_target *t;
unsigned int nr_regions;
unsigned int max_thres;
+ bool count_age = true;
max_thres = c->attrs.aggr_interval /
(c->attrs.sample_interval ? c->attrs.sample_interval : 1);
do {
nr_regions = 0;
damon_for_each_target(t, c) {
- damon_merge_regions_of(t, threshold, sz_limit);
+ damon_merge_regions_of(t, threshold, sz_limit,
+ count_age);
nr_regions += damon_nr_regions(t);
}
+ count_age = false;
threshold = max(1, threshold * 2);
} while (nr_regions > c->attrs.max_nr_regions &&
threshold / 2 < max_thres);
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index ceaa2a6e1c9a..929aab600c74 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -101,8 +101,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
unsigned int age_weight = s->quota.weight_age;
int hotness;
- freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE /
- damon_max_nr_accesses(&c->attrs);
+ freq_subscore = mult_frac(r->nr_accesses_bp / 10000,
+ DAMON_MAX_SUBSCORE,
+ damon_max_nr_accesses(&c->attrs));
age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index f28cfee8351d..1854b4bb6bd2 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -67,7 +67,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
}
}
-static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
+static bool damon_pa_young(phys_addr_t paddr)
{
struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
bool accessed;
@@ -76,7 +76,6 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
return false;
accessed = damon_folio_young(folio);
- *folio_sz = folio_size(folio);
folio_put(folio);
return accessed;
}
@@ -84,23 +83,12 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
static void __damon_pa_check_access(struct damon_region *r,
struct damon_attrs *attrs, unsigned long addr_unit)
{
- static phys_addr_t last_addr;
- static unsigned long last_folio_sz = PAGE_SIZE;
- static bool last_accessed;
+ bool accessed;
phys_addr_t sampling_addr = damon_pa_phys_addr(
r->sampling_addr, addr_unit);
- /* If the region is in the last checked page, reuse the result */
- if (ALIGN_DOWN(last_addr, last_folio_sz) ==
- ALIGN_DOWN(sampling_addr, last_folio_sz)) {
- damon_update_region_access_rate(r, last_accessed, attrs);
- return;
- }
-
- last_accessed = damon_pa_young(sampling_addr, &last_folio_sz);
- damon_update_region_access_rate(r, last_accessed, attrs);
-
- last_addr = sampling_addr;
+ accessed = damon_pa_young(sampling_addr);
+ damon_update_region_access_rate(r, accessed, attrs);
}
static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index adf60f672b8b..37e0a8bef4b1 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -166,6 +166,7 @@ static void damon_sysfs_scheme_regions_rm_dirs(
list_for_each_entry_safe(r, next, &regions->regions_list, list) {
list_del(&r->list);
+ kobject_del(&r->kobj);
kobject_put(&r->kobj);
regions->nr_regions--;
}
@@ -704,8 +705,10 @@ static void damon_sysfs_scheme_filters_rm_dirs(
struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr;
int i;
- for (i = 0; i < filters->nr; i++)
+ for (i = 0; i < filters->nr; i++) {
+ kobject_del(&filters_arr[i]->kobj);
kobject_put(&filters_arr[i]->kobj);
+ }
filters->nr = 0;
kfree(filters_arr);
filters->filters_arr = NULL;
@@ -1191,8 +1194,10 @@ static void damos_sysfs_quota_goals_rm_dirs(
struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr;
int i;
- for (i = 0; i < goals->nr; i++)
+ for (i = 0; i < goals->nr; i++) {
+ kobject_del(&goals_arr[i]->kobj);
kobject_put(&goals_arr[i]->kobj);
+ }
goals->nr = 0;
kfree(goals_arr);
goals->goals_arr = NULL;
@@ -1762,8 +1767,10 @@ static void damos_sysfs_dests_rm_dirs(
struct damos_sysfs_dest **dests_arr = dests->dests_arr;
int i;
- for (i = 0; i < dests->nr; i++)
+ for (i = 0; i < dests->nr; i++) {
+ kobject_del(&dests_arr[i]->kobj);
kobject_put(&dests_arr[i]->kobj);
+ }
dests->nr = 0;
kfree(dests_arr);
dests->dests_arr = NULL;
@@ -2303,6 +2310,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes)
for (i = 0; i < schemes->nr; i++) {
damon_sysfs_scheme_rm_dirs(schemes_arr[i]);
+ kobject_del(&schemes_arr[i]->kobj);
kobject_put(&schemes_arr[i]->kobj);
}
schemes->nr = 0;
@@ -2344,13 +2352,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes,
goto out;
err = damon_sysfs_scheme_add_dirs(scheme);
if (err)
- goto out;
+ goto del_out;
schemes_arr[i] = scheme;
schemes->nr++;
}
return 0;
+del_out:
+ kobject_del(&scheme->kobj);
out:
damon_sysfs_schemes_rm_dirs(schemes);
kobject_put(&scheme->kobj);
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index a6c3e83a3dad..53f99f05eb38 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -107,8 +107,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions)
struct damon_sysfs_region **regions_arr = regions->regions_arr;
int i;
- for (i = 0; i < regions->nr; i++)
+ for (i = 0; i < regions->nr; i++) {
+ kobject_del(&regions_arr[i]->kobj);
kobject_put(&regions_arr[i]->kobj);
+ }
regions->nr = 0;
kfree(regions_arr);
regions->regions_arr = NULL;
@@ -305,6 +307,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets)
for (i = 0; i < targets->nr; i++) {
damon_sysfs_target_rm_dirs(targets_arr[i]);
+ kobject_del(&targets_arr[i]->kobj);
kobject_put(&targets_arr[i]->kobj);
}
targets->nr = 0;
@@ -343,13 +346,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets,
err = damon_sysfs_target_add_dirs(target);
if (err)
- goto out;
+ goto del_out;
targets_arr[i] = target;
targets->nr++;
}
return 0;
+del_out:
+ kobject_del(&target->kobj);
out:
damon_sysfs_targets_rm_dirs(targets);
kobject_put(&target->kobj);
@@ -1078,6 +1083,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts)
for (i = 0; i < contexts->nr; i++) {
damon_sysfs_context_rm_dirs(contexts_arr[i]);
+ kobject_del(&contexts_arr[i]->kobj);
kobject_put(&contexts_arr[i]->kobj);
}
contexts->nr = 0;
@@ -1116,13 +1122,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts,
err = damon_sysfs_context_add_dirs(context);
if (err)
- goto out;
+ goto del_out;
contexts_arr[i] = context;
contexts->nr++;
}
return 0;
+del_out:
+ kobject_del(&context->kobj);
out:
damon_sysfs_contexts_rm_dirs(contexts);
kobject_put(&context->kobj);
@@ -1437,11 +1445,11 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx,
err = damon_select_ops(ctx, sys_ctx->ops_id);
if (err)
return err;
- ctx->addr_unit = sys_ctx->addr_unit;
+ ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit);
/* addr_unit is respected by only DAMON_OPS_PADDR */
if (sys_ctx->ops_id == DAMON_OPS_PADDR)
ctx->min_sz_region = max(
- DAMON_MIN_REGION / sys_ctx->addr_unit, 1);
+ DAMON_MIN_REGION / ctx->addr_unit, 1);
err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs);
if (err)
return err;
@@ -1867,6 +1875,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds)
for (i = 0; i < kdamonds->nr; i++) {
damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]);
+ kobject_del(&kdamonds_arr[i]->kobj);
kobject_put(&kdamonds_arr[i]->kobj);
}
kdamonds->nr = 0;
@@ -1921,13 +1930,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds,
err = damon_sysfs_kdamond_add_dirs(kdamond);
if (err)
- goto out;
+ goto del_out;
kdamonds_arr[i] = kdamond;
kdamonds->nr++;
}
return 0;
+del_out:
+ kobject_del(&kdamond->kobj);
out:
damon_sysfs_kdamonds_rm_dirs(kdamonds);
kobject_put(&kdamond->kobj);
diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h
index 668777875272..0ce648131943 100644
--- a/mm/damon/tests/core-kunit.h
+++ b/mm/damon/tests/core-kunit.h
@@ -166,6 +166,10 @@ static void damon_test_split_at(struct kunit *test)
r->last_nr_accesses = 15;
damon_add_region(r, t);
damon_split_region_at(t, r, 25);
+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+ if (damon_nr_regions(t) != 2)
+ goto out;
+
KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
KUNIT_EXPECT_EQ(test, r->ar.end, 25ul);
@@ -177,6 +181,7 @@ static void damon_test_split_at(struct kunit *test)
KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses);
KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses);
+out:
damon_free_target(t);
damon_destroy_ctx(c);
}
@@ -261,14 +266,17 @@ static void damon_test_merge_regions_of(struct kunit *test)
damon_add_region(r, t);
}
- damon_merge_regions_of(t, 9, 9999);
+ damon_merge_regions_of(t, 9, 9999, true);
/* 0-112, 114-130, 130-156, 156-170 */
KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u);
+ if (damon_nr_regions(t) != 5)
+ goto out;
for (i = 0; i < 5; i++) {
r = __nth_region_of(t, i);
KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]);
KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]);
}
+out:
damon_free_target(t);
}
@@ -588,6 +596,8 @@ static void damos_test_filter_out(struct kunit *test)
KUNIT_EXPECT_EQ(test, r->ar.start, DAMON_MIN_REGION * 1);
KUNIT_EXPECT_EQ(test, r->ar.end, DAMON_MIN_REGION * 2);
KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+ if (damon_nr_regions(t) != 2)
+ goto out;
r2 = damon_next_region(r);
KUNIT_EXPECT_EQ(test, r2->ar.start, DAMON_MIN_REGION * 2);
KUNIT_EXPECT_EQ(test, r2->ar.end, DAMON_MIN_REGION * 4);
@@ -602,11 +612,14 @@ static void damos_test_filter_out(struct kunit *test)
KUNIT_EXPECT_EQ(test, r->ar.start, DAMON_MIN_REGION * 2);
KUNIT_EXPECT_EQ(test, r->ar.end, DAMON_MIN_REGION * 6);
KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+ if (damon_nr_regions(t) != 2)
+ goto out;
r2 = damon_next_region(r);
KUNIT_EXPECT_EQ(test, r2->ar.start, DAMON_MIN_REGION * 6);
KUNIT_EXPECT_EQ(test, r2->ar.end, DAMON_MIN_REGION * 8);
damon_destroy_region(r2, t);
+out:
damon_free_target(t);
damos_free_filter(f);
}
diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h
index 30dc5459f1d2..e2c36d4d8789 100644
--- a/mm/damon/tests/vaddr-kunit.h
+++ b/mm/damon/tests/vaddr-kunit.h
@@ -149,12 +149,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test,
damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION);
+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2);
+ if (damon_nr_regions(t) != nr_expected / 2)
+ goto out;
+
for (i = 0; i < nr_expected / 2; i++) {
r = __nth_region_of(t, i);
KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]);
KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]);
}
+out:
damon_destroy_target(t, NULL);
}
diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index db93ca202460..c49f494ec17c 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -431,8 +431,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx)
}
struct damon_young_walk_private {
- /* size of the folio for the access checked virtual memory address */
- unsigned long *folio_sz;
bool young;
};
@@ -468,7 +466,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
mmu_notifier_test_young(walk->mm,
addr))
priv->young = true;
- *priv->folio_sz = HPAGE_PMD_SIZE;
folio_put(folio);
huge_out:
spin_unlock(ptl);
@@ -490,7 +487,6 @@ regular_page:
if (pte_young(ptent) || !folio_test_idle(folio) ||
mmu_notifier_test_young(walk->mm, addr))
priv->young = true;
- *priv->folio_sz = folio_size(folio);
folio_put(folio);
out:
pte_unmap_unlock(pte, ptl);
@@ -519,7 +515,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask,
if (pte_young(entry) || !folio_test_idle(folio) ||
mmu_notifier_test_young(walk->mm, addr))
priv->young = true;
- *priv->folio_sz = huge_page_size(h);
folio_put(folio);
@@ -537,11 +532,9 @@ static const struct mm_walk_ops damon_young_ops = {
.walk_lock = PGWALK_RDLOCK,
};
-static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
- unsigned long *folio_sz)
+static bool damon_va_young(struct mm_struct *mm, unsigned long addr)
{
struct damon_young_walk_private arg = {
- .folio_sz = folio_sz,
.young = false,
};
@@ -558,29 +551,18 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
* r the region to be checked
*/
static void __damon_va_check_access(struct mm_struct *mm,
- struct damon_region *r, bool same_target,
+ struct damon_region *r,
struct damon_attrs *attrs)
{
- static unsigned long last_addr;
- static unsigned long last_folio_sz = PAGE_SIZE;
- static bool last_accessed;
+ bool accessed;
if (!mm) {
damon_update_region_access_rate(r, false, attrs);
return;
}
- /* If the region is in the last checked page, reuse the result */
- if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) ==
- ALIGN_DOWN(r->sampling_addr, last_folio_sz))) {
- damon_update_region_access_rate(r, last_accessed, attrs);
- return;
- }
-
- last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz);
- damon_update_region_access_rate(r, last_accessed, attrs);
-
- last_addr = r->sampling_addr;
+ accessed = damon_va_young(mm, r->sampling_addr);
+ damon_update_region_access_rate(r, accessed, attrs);
}
static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
@@ -589,16 +571,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
struct mm_struct *mm;
struct damon_region *r;
unsigned int max_nr_accesses = 0;
- bool same_target;
damon_for_each_target(t, ctx) {
mm = damon_get_mm(t);
- same_target = false;
damon_for_each_region(r, t) {
- __damon_va_check_access(mm, r, same_target,
- &ctx->attrs);
+ __damon_va_check_access(mm, r, &ctx->attrs);
max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
- same_target = true;
}
if (mm)
mmput(mm);
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-11 17:25:02 +0000