From 4febfe7d98948bf6693f5c6a0a7e198e8fb4e584 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 14 Aug 2026 09:56:39 -0700 Subject: block: skip blkcg walk in blk_cgroup_congested() when nothing throttled blk_cgroup_congested() walks the current task's blkcg ancestor chain on every readahead decision and, once swap is in use, on every anonymous and shmem folio allocation. The answer is almost always "no", but finding that out costs two loads per level on two cold cache lines, plus an out-of-line kthread_blkcg() and an RCU read-side pair. On a fleet profile of hosts running containers with 5-10 level hierarchies it costs about as much as all of mutex_lock(), 99.4% of it under __folio_throttle_swaprate(). Gate the walk on a global count of blkcgs with a non-zero congestion_count. The counter only moves on the 0 <-> 1 transitions of each blkcg's congestion_count, so the extra atomic stays in the throttle arm/disarm paths and never appears in steady state. When something is throttled the counter is non-zero and the walk runs as before. Signed-off-by: Usama Arif Acked-by: Tejun Heo Link: https://patch.msgid.link/20260814165712.510132-4-usama.arif@linux.dev Signed-off-by: Jens Axboe --- include/linux/blk-cgroup.h | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index dd5841a42c33..58abde49f8c5 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -14,6 +14,8 @@ * Nauman Rafique */ +#include +#include #include struct bio; @@ -24,10 +26,29 @@ struct gendisk; #ifdef CONFIG_BLK_CGROUP extern struct cgroup_subsys_state * const blkcg_root_css; +extern atomic_t blkcg_nr_congested; void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay); void blkcg_maybe_throttle_current(void); -bool blk_cgroup_congested(void); +bool __blk_cgroup_congested(void); + +/** + * blk_cgroup_congested - is the current task in a throttled blkcg? + * + * Called from mm hot paths where the answer is almost always false, so keep + * that case to a load and a branch and only walk the hierarchy out of line + * when something in the system really is throttled. + * + * Return: %true if the current task's blkcg or any of its ancestors is + * throttled, %false otherwise. + */ +static inline bool blk_cgroup_congested(void) +{ + if (likely(!atomic_read(&blkcg_nr_congested))) + return false; + return __blk_cgroup_congested(); +} + void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css); void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css); struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css); -- cgit v1.2.3