From 080b5c6d95034e46f5ed1abe98c06218a1386aef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:35 +0200 Subject: sched/cputime: Remove superfluous and error prone kcpustat_field() parameter The first parameter to kcpustat_field() is a pointer to the cpu kcpustat to be fetched from. This parameter is error prone because a copy to a kcpustat could be passed by accident instead of the original one. Also the kcpustat structure can already be retrieved with the help of the mandatory CPU argument. Remove the needless parameter. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Reviewed-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-4-frederic@kernel.org --- include/linux/kernel_stat.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index b97ce2df376f..dd020ecaf67b 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -100,14 +100,12 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) } #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -extern u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu); +extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); #else -static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - return kcpustat->cpustat[usage]; + return kcpustat_cpu(cpu).cpustat[usage]; } static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) -- cgit v1.2.3 From cf6444c3e1bb7dd5974441bbd74840e9821d36f9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:39 +0200 Subject: tick/sched: Unify idle cputime accounting The non-vtime dynticks-idle cputime accounting is a big mess that accumulates within two concurrent statistics, each having their own shortcomings: * The accounting for online CPUs which is based on the delta between tick_nohz_start_idle() and tick_nohz_stop_idle(). Pros: - Works when the tick is off - Has nsecs granularity Cons: - Account idle steal time but doesn't substract it from idle cputime. - Assumes CONFIG_IRQ_TIME_ACCOUNTING by not accounting IRQs but the IRQ time is simply ignored when CONFIG_IRQ_TIME_ACCOUNTING=n - The windows between 1) idle task scheduling and the first call to tick_nohz_start_idle() and 2) idle task between the last tick_nohz_stop_idle() and the rest of the idle time are blindspots wrt. cputime accounting (though mostly insignificant amount) - Relies on private fields outside of kernel stats, with specific accessors. * The accounting for offline CPUs which is based on ticks and the jiffies delta during which the tick was stopped. Pros: - Handles steal time correctly - Handle CONFIG_IRQ_TIME_ACCOUNTING=y and CONFIG_IRQ_TIME_ACCOUNTING=n correctly. - Handles the whole idle task - Accounts directly to kernel stats, without midlayer accumulator. Cons: - Doesn't elapse when the tick is off, which doesn't make it suitable for online CPUs. - Has TICK_NSEC granularity (jiffies) - Needs to track the dyntick-idle ticks that were accounted and substract them from the total jiffies time spent while the tick was stopped. This is an ugly workaround. Having two different accounting for a single context is not the only problem: since those accountings are of different natures, it is possible to observe the global idle time going backward after a CPU goes offline. Clean up the situation with introducing a hybrid approach that stays coherent and works for both online and offline CPUs: * Tick based or native vtime accounting operate before the idle loop is entered and resume once the idle loop prepares to exit. * When the idle loop starts, switch to dynticks-idle accounting as is done currently, except that the statistics accumulate directly to the relevant kernel stat fields. * Private dyntick cputime accounting fields are removed. * Works on both online and offline case. Further improvement will include: * Only switch to dynticks-idle cputime accounting when the tick actually goes in dynticks mode. * Handle CONFIG_IRQ_TIME_ACCOUNTING=n correctly such that the dynticks-idle accounting still elapses while on IRQs. * Correctly substract idle steal cputime from idle time Reported-by: Xin Zhao Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-8-frederic@kernel.org --- include/linux/kernel_stat.h | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index dd020ecaf67b..ba65aad308a1 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -34,6 +34,9 @@ enum cpu_usage_stat { }; struct kernel_cpustat { +#ifdef CONFIG_NO_HZ_COMMON + int idle_dyntick; +#endif u64 cpustat[NR_STATS]; }; @@ -99,6 +102,20 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) return kstat_cpu(cpu).irqs_sum; } +#ifdef CONFIG_NO_HZ_COMMON +extern void kcpustat_dyntick_start(void); +extern void kcpustat_dyntick_stop(void); +static inline bool kcpustat_idle_dyntick(void) +{ + return __this_cpu_read(kernel_cpustat.idle_dyntick); +} +#else +static inline bool kcpustat_idle_dyntick(void) +{ + return false; +} +#endif /* CONFIG_NO_HZ_COMMON */ + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); @@ -113,7 +130,7 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) *dst = kcpustat_cpu(cpu); } -#endif +#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); @@ -127,14 +144,13 @@ extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick(struct task_struct *tsk, int user) { - vtime_flush(tsk); + if (!kcpustat_idle_dyntick()) + vtime_flush(tsk); } #else extern void account_process_tick(struct task_struct *, int user); #endif -extern void account_idle_ticks(unsigned long ticks); - #ifdef CONFIG_SCHED_CORE extern void __account_forceidle_time(struct task_struct *tsk, u64 delta); #endif -- cgit v1.2.3 From a5fe724e206ec7ff3ceb15b285d94316c7fe6c41 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:41 +0200 Subject: tick/sched: Move dyntick-idle cputime accounting to cputime code Although the dynticks-idle cputime accounting is necessarily tied to the tick subsystem, the actual related accounting code has no business residing there and should be part of the scheduler cputime code. Move away the relevant pieces and state machine to where they belong. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-10-frederic@kernel.org --- include/linux/kernel_stat.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index ba65aad308a1..9343353ac7a3 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -35,9 +35,12 @@ enum cpu_usage_stat { struct kernel_cpustat { #ifdef CONFIG_NO_HZ_COMMON - int idle_dyntick; + bool idle_dyntick; + bool idle_elapse; + seqcount_t idle_sleeptime_seq; + u64 idle_entrytime; #endif - u64 cpustat[NR_STATS]; + u64 cpustat[NR_STATS]; }; struct kernel_stat { @@ -103,8 +106,11 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) } #ifdef CONFIG_NO_HZ_COMMON -extern void kcpustat_dyntick_start(void); -extern void kcpustat_dyntick_stop(void); +extern void kcpustat_dyntick_start(u64 now); +extern void kcpustat_dyntick_stop(u64 now); +extern void kcpustat_irq_enter(u64 now); +extern void kcpustat_irq_exit(u64 now); + static inline bool kcpustat_idle_dyntick(void) { return __this_cpu_read(kernel_cpustat.idle_dyntick); -- cgit v1.2.3 From 127b2eb44f36d5d7059f1af425b5800cb27440f9 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:44 +0200 Subject: tick/sched: Consolidate idle time fetching APIs Fetching the idle cputime is available through a variety of accessors all over the place depending on the different accounting flavours and needs: - idle vtime generic accounting can be accessed by kcpustat_field(), kcpustat_cpu_fetch(), get_idle/iowait_time() and get_cpu_idle/iowait_time_us() - dynticks-idle accounting can only be accessed by get_idle/iowait_time() or get_cpu_idle/iowait_time_us() - CONFIG_NO_HZ_COMMON=n idle accounting can be accessed by kcpustat_field() kcpustat_cpu_fetch(), or get_idle/iowait_time() but not by get_cpu_idle/iowait_time_us() Moreover get_idle/iowait_time() relies on get_cpu_idle/iowait_time_us() with a non-sensical conversion to microseconds and back to nanoseconds on the way. Start consolidating the APIs with removing get_idle/iowait_time() and make kcpustat_field() and kcpustat_cpu_fetch() work for all cases. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-13-frederic@kernel.org --- include/linux/kernel_stat.h | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 9343353ac7a3..3680519d7b2c 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -110,32 +110,59 @@ extern void kcpustat_dyntick_start(u64 now); extern void kcpustat_dyntick_stop(u64 now); extern void kcpustat_irq_enter(u64 now); extern void kcpustat_irq_exit(u64 now); +extern u64 kcpustat_field_idle(int cpu); +extern u64 kcpustat_field_iowait(int cpu); static inline bool kcpustat_idle_dyntick(void) { return __this_cpu_read(kernel_cpustat.idle_dyntick); } #else +static inline u64 kcpustat_field_idle(int cpu) +{ + return kcpustat_cpu(cpu).cpustat[CPUTIME_IDLE]; +} +static inline u64 kcpustat_field_iowait(int cpu) +{ + return kcpustat_cpu(cpu).cpustat[CPUTIME_IOWAIT]; +} + static inline bool kcpustat_idle_dyntick(void) { return false; } #endif /* CONFIG_NO_HZ_COMMON */ +/* Fetch cputime values when vtime is disabled on a CPU */ +static inline u64 kcpustat_field_default(enum cpu_usage_stat usage, int cpu) +{ + if (usage == CPUTIME_IDLE) + return kcpustat_field_idle(cpu); + if (usage == CPUTIME_IOWAIT) + return kcpustat_field_iowait(cpu); + return kcpustat_cpu(cpu).cpustat[usage]; +} + +static inline void kcpustat_cpu_fetch_default(struct kernel_cpustat *dst, int cpu) +{ + *dst = kcpustat_cpu(cpu); + dst->cpustat[CPUTIME_IDLE] = kcpustat_field_idle(cpu); + dst->cpustat[CPUTIME_IOWAIT] = kcpustat_field_iowait(cpu); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); #else static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - return kcpustat_cpu(cpu).cpustat[usage]; + return kcpustat_field_default(usage, cpu); } static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) { - *dst = kcpustat_cpu(cpu); + kcpustat_cpu_fetch_default(dst, cpu); } - #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ extern void account_user_time(struct task_struct *, u64); @@ -145,7 +172,6 @@ extern void account_system_index_time(struct task_struct *, u64, enum cpu_usage_stat); extern void account_steal_time(u64); extern void account_idle_time(u64); -extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick(struct task_struct *tsk, int user) -- cgit v1.2.3 From 3b45b4f188f3a0ebd16ab71efd2ffcc7a16ad861 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:45 +0200 Subject: sched/cputime: Provide get_cpu_[idle|iowait]_time_us() off-case The last reason why get_cpu_idle/iowait_time_us() may return -1 now is if the config doesn't support nohz. The ad-hoc replacement solution by cpufreq is to compute jiffies minus the whole busy cputime. Although the intention should provide a coherent low resolution estimation of the idle and iowait time, the implementation is buggy because jiffies don't start at 0. Just provide instead a real get_cpu_[idle|iowait]_time_us() offcase. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-14-frederic@kernel.org --- include/linux/kernel_stat.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 3680519d7b2c..512104b0ff49 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -133,6 +133,9 @@ static inline bool kcpustat_idle_dyntick(void) } #endif /* CONFIG_NO_HZ_COMMON */ +extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); +extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); + /* Fetch cputime values when vtime is disabled on a CPU */ static inline u64 kcpustat_field_default(enum cpu_usage_stat usage, int cpu) { -- cgit v1.2.3 From 6199f9999a9b62b2b84a1bf5b52a9fd0bb8de5af Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 8 May 2026 15:16:47 +0200 Subject: sched/cputime: Handle dyntick-idle steal time correctly The dyntick-idle steal time is currently accounted when the tick restarts but the stolen idle time is not subtracted from the idle time that was already accounted. This is to avoid observing the idle time going backward as the dyntick-idle cputime accessors can't reliably know in advance the stolen idle time. In order to maintain a forward progressing idle cputime while subtracting idle steal time from it, keep track of the previously accounted idle stolen time and substract it from _later_ idle cputime accounting. Signed-off-by: Frederic Weisbecker Signed-off-by: Thomas Gleixner Tested-by: Shrikanth Hegde Link: https://patch.msgid.link/20260508131647.43868-16-frederic@kernel.org --- include/linux/kernel_stat.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 512104b0ff49..fce1392e2140 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -39,6 +39,7 @@ struct kernel_cpustat { bool idle_elapse; seqcount_t idle_sleeptime_seq; u64 idle_entrytime; + u64 idle_stealtime[2]; #endif u64 cpustat[NR_STATS]; }; -- cgit v1.2.3 From 670e057744e0cc8047bf96d15d18c46e16ae2e93 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 13 May 2026 16:01:28 +0200 Subject: s390/idle: Provide arch specific kcpustat_field_idle()/kcpustat_field_iowait() The former s390 specific arch_cpu_idle_time() implementation was removed, since its implementation was racy and reported idle time could go backwards [1]. However this removal was not necessary, since independently of the s390 architecture specific races there exists the iowait counter update race, which can also lead to reported idle time going backwards [2]. With Frederic Weisbecker's recent cpu idle time accounting refactoring kernel_cpustat got a sequence counter. Use this to implement s390 specific variants of kcpustat_field_idle() and kcpustat_field_iowait(). This is logically a revert of [1] and moves cpu idle time accounting back into s390 architecture code, which is also more precise than the dyntick idle time accounting by nohz/scheduler. For comparing cross cpu time stamps it is necessary to use the stcke instead of the stckf instruction in irq entry path. Furthermore this open-codes a sequence lock in assembler and C code, which is required to update the irq entry time stamp to the per cpu idle_data structure in a race free manner. [1] commit be76ea614460 ("s390/idle: remove arch_cpu_idle_time() and corresponding code") [2] commit ead70b752373 ("timers/nohz: Add a comment about broken iowait counter update race") Signed-off-by: Heiko Carstens Acked-by: Frederic Weisbecker Signed-off-by: Alexander Gordeev --- include/linux/kernel_stat.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'include/linux/kernel_stat.h') diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index fce1392e2140..9ca6c2259dfe 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -107,6 +107,30 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) } #ifdef CONFIG_NO_HZ_COMMON + +#ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE + +static inline void kcpustat_dyntick_start(u64 now) { } +static inline void kcpustat_dyntick_stop(u64 now) { } +static inline void kcpustat_irq_enter(u64 now) { } +static inline void kcpustat_irq_exit(u64 now) { } +static inline bool kcpustat_idle_dyntick(void) { return false; } + +extern u64 arch_kcpustat_field_idle(int cpu); +extern u64 arch_kcpustat_field_iowait(int cpu); + +static inline u64 kcpustat_field_idle(int cpu) +{ + return arch_kcpustat_field_idle(cpu); +} + +static inline u64 kcpustat_field_iowait(int cpu) +{ + return arch_kcpustat_field_iowait(cpu); +} + +#else /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ + extern void kcpustat_dyntick_start(u64 now); extern void kcpustat_dyntick_stop(u64 now); extern void kcpustat_irq_enter(u64 now); @@ -118,6 +142,9 @@ static inline bool kcpustat_idle_dyntick(void) { return __this_cpu_read(kernel_cpustat.idle_dyntick); } + +#endif /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ + #else static inline u64 kcpustat_field_idle(int cpu) { -- cgit v1.2.3