summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-08-18 16:23:56 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-08-18 16:23:56 -0700
commit3b4128b9f374b4219eb716f4ad8a307bc7eb3d84 (patch)
treee9489db7a4ab524e7849fb70a3cc47a08700e1ad /kernel
parentb0239dd672306ad242545f793132938847f17e53 (diff)
parent4fa377c19e111c539a530a8200996b911ceff9ff (diff)
Merge tag 'timers-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timer and timekeeping core updates from Thomas Gleixner: - Fix a subtly inconsistency in the timekeeping code, which fails to account for the monotonicity adjustment in ntp_error. For small changes of the clocksource multiplicator (+/-1) which are typically used by the NTP PLL this is hard to observe. But for larger adjustments, e.g. caused by a direct frequency setting through adjtimex() the one-time uncompensated offset is significant. Cure this by adjusting ntp_error with the resulting offset so that the discrepancy is smoothed away over time - Make tick length calculations correct in NTP. The timekeeping core takes the quantisation of the clocksource into account when calculating the tick length to compensate for the deviation of the nominal NTP_INTERVAL_LENGTH. While timekeeping gets this right, NTP is not aware of that, which means it operates on the nominal value and not on the actual value which is determined by the clock source frequency. The rounding of a coarse clocksource like the ACPI PM timer results in a +127 PPM deviation. Cure this by exposing the deviation to the NTP code so that it can operate on the same data as the timekeeping core. This is purely kernel internal. User space still sees the nominal tick lenght via adjtimex(). - The accuracy of the NTP adjustments is fairly approximate as the code assumes that the invocations are precisely in NTP interval frequency ticks and the final adjustment can over and under-run. Cure this by adjusting ntp_error by the intended skew on each tick to achieve the desired rate. - Handle the two competing skews of time offset and time adjustment correctly by calculating the conflict portion between the skews and adjusting both accordingly. - A set of updates and improvements for the selftests - The usual small fixes and improvements all over the place * tag 'timers-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (58 commits) selftests: timers: nsleep-lat: Check all calls to clock_nanosleep() and clock_gettime() selftests: timers: nsleep-lat: Reuse kselftest error numbers selftests: timers: nsleep-lat: Explicitly list the tested clocks selftests: timers: nsleep-lat: Use NSEC_PER_MSEC define for unreasonable latency selftests: timers: nanosleep: Report each test separately selftests: timers: nanosleep: Explicitly handle timer_delete() failure selftests: timers: nanosleep: Move all single clock tests out of the loop in main() selftests: timers: nanosleep: Reuse kselftest error numbers selftests: timers: nanosleep: Explicitly list the tested clocks selftests: timers: nanosleep: Drop output alignment selftests: timers: Use clock_name() and constants from clock-helpers.h selftests: Add clock-helpers.h timer_list: Use ktime_t over nanoseconds timer_list: Use standard 'long long' format placeholders hrtimer: Add a lockdep assertion to hrtimer_update_base() timekeeping: Use u32 for clock_was_set_seq timekeeping: Rename clockid_aux_valid() to clockid_is_aux_clock() hrtimer: Account nr_retries on recovered interrupt retries timers/itimer: Zero-init old itimerval before copy to userspace nohz: Replace dead select with choice default ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched/fair.c2
-rw-r--r--kernel/time/Kconfig1
-rw-r--r--kernel/time/clocksource.c9
-rw-r--r--kernel/time/hrtimer.c64
-rw-r--r--kernel/time/itimer.c2
-rw-r--r--kernel/time/namespace.c6
-rw-r--r--kernel/time/ntp.c310
-rw-r--r--kernel/time/ntp_internal.h4
-rw-r--r--kernel/time/posix-cpu-timers.c9
-rw-r--r--kernel/time/tick-internal.h1
-rw-r--r--kernel/time/time.c1
-rw-r--r--kernel/time/timekeeping.c204
-rw-r--r--kernel/time/timekeeping.h2
-rw-r--r--kernel/time/timekeeping_internal.h21
-rw-r--r--kernel/time/timer_list.c78
-rw-r--r--kernel/time/timer_migration.c4
-rw-r--r--kernel/time/timer_migration.h2
17 files changed, 546 insertions, 174 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 635f5775cc13..001140132a7d 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -26,6 +26,8 @@
#include <linux/jiffies.h>
#include <linux/mm_api.h>
#include <linux/highmem.h>
+#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/spinlock_api.h>
#include <linux/cpumask_api.h>
#include <linux/lockdep_api.h>
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index d098ac39bde4..ddfb6bee0745 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -133,7 +133,6 @@ config NO_HZ_FULL
depends on HAVE_VIRT_CPU_ACCOUNTING_GEN
select NO_HZ_COMMON
select RCU_NOCB_CPU
- select VIRT_CPU_ACCOUNTING_GEN
select IRQ_WORK
select CPU_ISOLATION
help
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index e48c4d379a7c..f1253f5795c6 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -123,7 +123,6 @@ static atomic_t watchdog_reset_pending;
/* Watchdog interval: 0.5sec. */
#define WATCHDOG_INTERVAL (HZ >> 1)
-#define WATCHDOG_INTERVAL_NS (WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ))
/* Maximum time between two reference watchdog readouts */
#define WATCHDOG_READOUT_MAX_NS (50U * NSEC_PER_USEC)
@@ -1566,8 +1565,12 @@ static int __init init_clocksource_sysfs(void)
{
int error = subsys_system_register(&clocksource_subsys, NULL);
- if (!error)
- error = device_register(&device_clocksource);
+ if (error)
+ return error;
+
+ error = device_register(&device_clocksource);
+ if (error)
+ bus_unregister(&clocksource_subsys);
return error;
}
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 313dcea127fe..530d61257b9a 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -26,6 +26,7 @@
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/notifier.h>
#include <linux/syscalls.h>
#include <linux/interrupt.h>
@@ -676,6 +677,8 @@ static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
{
+ lockdep_assert_held(&base->lock);
+
ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
@@ -707,7 +710,7 @@ static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
tick_program_event(expires_next, 1);
}
-static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
+static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
ktime_t expires_next)
{
cpu_base->expires_next = expires_next;
@@ -743,7 +746,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip
if (skip_equal && expires_next == cpu_base->expires_next)
return;
- __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
+ __hrtimer_reprogram(cpu_base, expires_next);
}
/* High resolution timer related functions */
@@ -896,14 +899,14 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
cpu_base->next_timer = timer;
- __hrtimer_reprogram(cpu_base, timer, expires);
+ __hrtimer_reprogram(cpu_base, expires);
}
static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
{
struct hrtimer_clock_base *base;
- unsigned int seq;
ktime_t expires;
+ u32 seq;
/*
* Update the base offsets unconditionally so the following
@@ -1040,6 +1043,30 @@ static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned lon
}
/**
+ * hrtimer_update_function - Update the timer's callback function
+ * @timer: Timer to update
+ * @function: New callback function
+ *
+ * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
+ * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
+ */
+void hrtimer_update_function(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *))
+{
+#ifdef CONFIG_PROVE_LOCKING
+ guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
+
+ if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
+ return;
+
+ if (WARN_ON_ONCE(!function))
+ return;
+#endif
+ ACCESS_PRIVATE(timer, function) = function;
+}
+EXPORT_SYMBOL_GPL(hrtimer_update_function);
+
+/**
* hrtimer_forward() - forward the timer expiry
* @timer: hrtimer to forward
* @now: forward past this time
@@ -1786,13 +1813,21 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
ktime_t hrtimer_get_next_event(void)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
- ktime_t expires = KTIME_MAX;
- guard(raw_spinlock_irqsave)(&cpu_base->lock);
- if (!hrtimer_hres_active(cpu_base))
- expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
+ /*
+ * When HRES is active cmp_next_hrtimer_event() expects KTIME_MAX.
+ *
+ * cpu_base->hres_active is written only by the local CPU in
+ * hrtimer_switch_to_hres() from hard interrupt context and in
+ * hrtimers_cpu_starting() during CPU bring-up, and all callers reach
+ * this with interrupts disabled on the same CPU, so an unlocked read is
+ * stable without holding the lock.
+ */
+ if (hrtimer_hres_active(cpu_base))
+ return KTIME_MAX;
- return expires;
+ guard(raw_spinlock_irqsave)(&cpu_base->lock);
+ return __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
}
/**
@@ -2060,13 +2095,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc
base->running = NULL;
}
-static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base)
-{
- struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
-
- return next ? hrtimer_from_timerqueue_node(next) : NULL;
-}
-
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
unsigned long flags, unsigned int active_mask)
{
@@ -2228,8 +2256,10 @@ retry:
expires_next = hrtimer_update_next_event(cpu_base);
cpu_base->hang_detected = false;
if (expires_next < now) {
- if (++retries < 3)
+ if (++retries < 3) {
+ cpu_base->nr_retries++;
goto retry;
+ }
delta = ktime_sub(now, entry_time);
cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c
index 7c6110e964e7..03a32dffc56c 100644
--- a/kernel/time/itimer.c
+++ b/kernel/time/itimer.c
@@ -100,7 +100,7 @@ static int do_getitimer(int which, struct itimerspec64 *value)
static int put_itimerval(struct __kernel_old_itimerval __user *o,
const struct itimerspec64 *i)
{
- struct __kernel_old_itimerval v;
+ struct __kernel_old_itimerval v = {};
v.it_interval.tv_sec = i->it_interval.tv_sec;
v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC;
diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c
index 5fa0af66cf3f..3aff27bb0a15 100644
--- a/kernel/time/namespace.c
+++ b/kernel/time/namespace.c
@@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
return -EINVAL;
}
- if (off->val.tv_sec > KTIME_SEC_MAX ||
- off->val.tv_sec < -KTIME_SEC_MAX)
+ if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX)
return -ERANGE;
+ if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC)
+ return -EINVAL;
+
tp = timespec64_add(tp, off->val);
/*
* KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 97fa99b96dd0..d22b532ec536 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -26,11 +26,13 @@
/**
* struct ntp_data - Structure holding all NTP related state
* @tick_usec: USER_HZ period in microseconds
- * @tick_length: Adjusted tick length
- * @tick_length_base: Base value for @tick_length
+ * @tick_length: Tick length in ns << NTP_SCALE_SHIFT
* @time_state: State of the clock synchronization
* @time_status: Clock status bits
* @time_offset: Time adjustment in nanoseconds
+ * @skew_delta: Per-tick phase slew rate for the coming second, in
+ * @time_offset units (shifted-ns / HZ). Set by
+ * second_overflow().
* @time_constant: PLL time constant
* @time_maxerror: Maximum error in microseconds holding the NTP sync distance
* (NTP dispersion + delay / 2)
@@ -38,7 +40,13 @@
* @time_freq: Frequency offset scaled nsecs/secs
* @time_reftime: Time at last adjustment in seconds
* @time_adjust: Adjustment value
+ * @time_adjust_frac: Sub-microsecond remainder of @time_adjust being
+ * delivered, in ns << NTP_SCALE_SHIFT (not divided by HZ).
* @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled)
+ * @cs_tick_adj: Fixed per-second adjustment compensating for the difference
+ * between the nominal NTP interval and the real time taken
+ * by the clocksource's integer @cycle_interval (upscaled).
+ * Set by the timekeeping core via ntp_clear().
* @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap
*
* @pps_valid: PPS signal watchdog counter
@@ -59,17 +67,19 @@
struct ntp_data {
unsigned long tick_usec;
u64 tick_length;
- u64 tick_length_base;
int time_state;
int time_status;
s64 time_offset;
+ s64 skew_delta;
long time_constant;
long time_maxerror;
long time_esterror;
s64 time_freq;
time64_t time_reftime;
long time_adjust;
+ s64 time_adjust_frac;
s64 ntp_tick_adj;
+ s64 cs_tick_adj;
time64_t ntp_next_leap_sec;
#ifdef CONFIG_NTP_PPS
int pps_valid;
@@ -101,6 +111,9 @@ static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = {
#define SECS_PER_DAY 86400
#define MAX_TICKADJ 500LL /* usecs */
+/* One microsecond of phase, in plain shifted-ns (ns << NTP_SCALE_SHIFT) */
+#define ONE_US_NS ((s64)NSEC_PER_USEC << NTP_SCALE_SHIFT)
+/* Per-tick MAX_TICKADJ slew, in plain shifted-ns */
#define MAX_TICKADJ_SCALED \
(((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
#define MAX_TAI_OFFSET 100000
@@ -245,8 +258,7 @@ static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_time
#endif /* CONFIG_NTP_PPS */
/*
- * Update tick_length and tick_length_base, based on tick_usec, ntp_tick_adj and
- * time_freq:
+ * Update tick_length based on tick_usec, ntp_tick_adj and time_freq:
*/
static void ntp_update_frequency(struct ntp_data *ntpdata)
{
@@ -255,6 +267,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
second_length += ntpdata->ntp_tick_adj;
+ second_length += ntpdata->cs_tick_adj;
second_length += ntpdata->time_freq;
new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
@@ -263,8 +276,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
* Don't wait for the next second_overflow, apply the change to the
* tick length immediately:
*/
- ntpdata->tick_length += new_base - ntpdata->tick_length_base;
- ntpdata->tick_length_base = new_base;
+ ntpdata->tick_length = new_base;
}
static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs)
@@ -335,14 +347,15 @@ static void __ntp_clear(struct ntp_data *ntpdata)
{
/* Stop active adjtime() */
ntpdata->time_adjust = 0;
+ ntpdata->time_adjust_frac = 0;
ntpdata->time_status |= STA_UNSYNC;
ntpdata->time_maxerror = NTP_PHASE_LIMIT;
ntpdata->time_esterror = NTP_PHASE_LIMIT;
ntp_update_frequency(ntpdata);
- ntpdata->tick_length = ntpdata->tick_length_base;
ntpdata->time_offset = 0;
+ ntpdata->skew_delta = 0;
ntpdata->ntp_next_leap_sec = TIME64_MAX;
/* Clear PPS state variables */
@@ -350,11 +363,26 @@ static void __ntp_clear(struct ntp_data *ntpdata)
}
/**
- * ntp_clear - Clears the NTP state variables
- * @tkid: Timekeeper ID to be able to select proper ntp data array member
+ * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
+ * @tkid: Timekeeper ID
+ * @cs_tick_adj: Per-second adjustment in ns << NTP_SCALE_SHIFT
+ *
+ * The timekeeping core uses an integer number of cycles (@cycle_interval)
+ * per NTP interval, so the real time that interval represents differs from
+ * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
+ * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
+ * tick_length computation in ntp.c, instead of being applied during
+ * timekeeping accumulation where the NTP code never saw it. Like
+ * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
+ * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
+ * around the corrected base rate.
+ *
+ * Called whenever the clocksource is (re)configured, which is also when the
+ * rest of the NTP state must be cleared, so the two are done together.
*/
-void ntp_clear(unsigned int tkid)
+void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
{
+ tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
__ntp_clear(&tk_ntp_data[tkid]);
}
@@ -364,6 +392,186 @@ u64 ntp_tick_length(unsigned int tkid)
return tk_ntp_data[tkid].tick_length;
}
+s64 ntp_get_skew_delta(unsigned int tkid)
+{
+ return tk_ntp_data[tkid].skew_delta;
+}
+
+/* Sign of @x as +1 or -1 (zero counts as positive; callers pass nonzero). */
+static inline int signof(s64 x)
+{
+ return x < 0 ? -1 : 1;
+}
+
+static s64 ntp_drain_time_offset(unsigned int tkid, s64 amount)
+{
+ struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+
+ /* Only drain if amount and time_offset have the same sign */
+ if (!amount || signof(amount) != signof(ntpdata->time_offset))
+ return amount;
+
+ /* Clamp: don't overshoot zero */
+ if (abs(amount) > abs(ntpdata->time_offset)) {
+ s64 undrained = amount - ntpdata->time_offset;
+
+ ntpdata->time_offset = 0;
+ return undrained;
+ }
+
+ ntpdata->time_offset -= amount;
+ return 0;
+}
+
+/*
+ * Drain the legacy adjtime() correction (time_adjust) as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ); it covers
+ * both the exponential time_offset slew and the linear adjtime slew. This
+ * function claims only the adjtime share — capped at the MAX_TICKADJ rate —
+ * and returns the remainder for ntp_drain_time_offset().
+ *
+ * time_adjust is in whole µs. The sub-µs remainder being delivered lives in
+ * time_adjust_frac (plain shifted-ns, i.e. ns << NTP_SCALE_SHIFT -- unlike
+ * time_offset these are NOT pre-divided by HZ); we top it up by borrowing
+ * whole microseconds from time_adjust as the drain consumes it.
+ */
+static s64 ntp_drain_time_adjust(unsigned int tkid, s64 amount, unsigned int shift)
+{
+ struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+ /* Sign reference: time_adjust if any whole us remain, else the drawer */
+ s64 ref = ntpdata->time_adjust ? (s64)ntpdata->time_adjust
+ : ntpdata->time_adjust_frac;
+ s64 deliver, deficit, claimed;
+
+ if (!amount || !ref || signof(amount) != signof(ref))
+ return amount;
+
+ /*
+ * Phase to deliver this accumulation, in plain shifted-ns. The drain
+ * @amount is in ÷HZ units, so multiply by HZ first, then clamp to the
+ * MAX_TICKADJ rate (MAX_TICKADJ_SCALED is the per-tick slew in
+ * shifted-ns). Multiply-then-clamp avoids an s64 divide for the cap.
+ */
+ deliver = min(abs(amount) * NTP_INTERVAL_FREQ,
+ (s64)MAX_TICKADJ_SCALED << shift);
+
+ /* Top up the sub-µs drawer from whole-µs time_adjust as needed */
+ deficit = deliver - abs(ntpdata->time_adjust_frac);
+ if (deficit > 0 && ntpdata->time_adjust) {
+ long borrow = div64_u64(deficit + ONE_US_NS - 1, ONE_US_NS);
+
+ if (ntpdata->time_adjust > 0) {
+ borrow = min(borrow, ntpdata->time_adjust);
+ ntpdata->time_adjust -= borrow;
+ ntpdata->time_adjust_frac += (s64)borrow * ONE_US_NS;
+ } else {
+ /* Clamp without negating time_adjust (UB for LONG_MIN) */
+ if (ntpdata->time_adjust > -borrow)
+ borrow = -ntpdata->time_adjust;
+ ntpdata->time_adjust += borrow;
+ ntpdata->time_adjust_frac -= (s64)borrow * ONE_US_NS;
+ }
+ }
+
+ /* Never deliver more than the drawer holds */
+ deliver = min(deliver, abs(ntpdata->time_adjust_frac));
+ if (ntpdata->time_adjust_frac > 0)
+ ntpdata->time_adjust_frac -= deliver;
+ else
+ ntpdata->time_adjust_frac += deliver;
+
+ /* Return the unclaimed remainder in ÷HZ drain units for time_offset */
+ claimed = div_s64(deliver, NTP_INTERVAL_FREQ);
+ return amount - signof(amount) * claimed;
+}
+
+/*
+ * Drain one accumulation's worth of intentional skew as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ). The
+ * adjtime() linear share is taken from time_adjust first (capped at the
+ * MAX_TICKADJ rate, hence @shift), then the exponential remainder from
+ * time_offset. Returns the amount actually claimed (same ÷HZ units).
+ */
+s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift)
+{
+ s64 unclaimed = ntp_drain_time_adjust(tkid, amount, shift);
+
+ unclaimed = ntp_drain_time_offset(tkid, unclaimed);
+
+ /*
+ * Return the amount actually drained from the intentional
+ * phase offset in time_offset and/or time_adjust.
+ */
+ return amount - unclaimed;
+}
+
+/*
+ * time_offset (drained exponentially) and time_adjust (drained linearly at the
+ * MAX_TICKADJ rate) can be asked to slew the clock in opposite directions.
+ * second_overflow() only folds their *net* into skew_delta, so the cancelling
+ * part would never be drained from either tracker via the per-tick code -- and
+ * if they cancel exactly, skew_delta is zero and neither converges at all.
+ *
+ * Settle that cancelling phase directly between the two here. No clock motion
+ * results (the opposing slews annihilate), but both move toward zero so neither
+ * stalls. @amount is the phase to take off time_offset, in its (÷HZ) units and
+ * with its sign; the same real magnitude comes off time_adjust in the opposite
+ * direction. Clamped so neither tracker is driven past zero.
+ */
+static void ntp_transfer_offset_adjust(struct ntp_data *ntpdata, s64 amount)
+{
+ s64 frac_delta, carry;
+
+ /*
+ * Don't drain time_offset past zero. @amount shares its sign and is
+ * normally bounded below it by ntp_offset_chunk(), but the ±1 skew_delta
+ * floor for a tiny time_offset can exceed it, so clamp.
+ */
+ if (abs(amount) > abs(ntpdata->time_offset))
+ amount = ntpdata->time_offset;
+ if (!amount)
+ return;
+
+ /*
+ * Remove the matching phase from time_adjust, in plain shifted-ns. No
+ * clamp against time_adjust's zero is needed: @amount is bounded by the
+ * adjtime chunk, which second_overflow() never lets exceed time_adjust's
+ * own pending phase, so this cannot overshoot.
+ */
+ frac_delta = amount * NTP_INTERVAL_FREQ;
+
+ ntpdata->time_offset -= amount;
+
+ /* Add the matching phase to time_adjust, carrying whole µs (O(1)). */
+ ntpdata->time_adjust_frac += frac_delta;
+ if (ntpdata->time_adjust_frac >= ONE_US_NS ||
+ ntpdata->time_adjust_frac <= -ONE_US_NS) {
+ carry = div64_s64(ntpdata->time_adjust_frac, ONE_US_NS);
+ ntpdata->time_adjust += carry;
+ ntpdata->time_adjust_frac -= carry * ONE_US_NS;
+ }
+
+ /*
+ * Keep time_adjust and its sub-µs remainder the same sign. The
+ * truncating carry above can leave them opposed (e.g. +4 µs paired
+ * with -250 ns), and ntp_drain_time_adjust() treats abs(time_adjust_frac)
+ * as same-direction drawer capacity -- an opposing remainder there makes
+ * it over-deliver phase that was never removed from the pile. Borrow or
+ * repay a single whole µs to realign; the total phase is unchanged.
+ */
+ if (ntpdata->time_adjust > 0 && ntpdata->time_adjust_frac < 0) {
+ ntpdata->time_adjust--;
+ ntpdata->time_adjust_frac += ONE_US_NS;
+ } else if (ntpdata->time_adjust < 0 && ntpdata->time_adjust_frac > 0) {
+ ntpdata->time_adjust++;
+ ntpdata->time_adjust_frac -= ONE_US_NS;
+ }
+}
+
/**
* ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
* @tkid: Timekeeper ID
@@ -398,7 +606,6 @@ ktime_t ntp_get_next_leap(unsigned int tkid)
int second_overflow(unsigned int tkid, time64_t secs)
{
struct ntp_data *ntpdata = &tk_ntp_data[tkid];
- s64 delta;
int leap = 0;
s32 rem;
@@ -458,35 +665,70 @@ int second_overflow(unsigned int tkid, time64_t secs)
}
/* Compute the phase adjustment for the next second */
- ntpdata->tick_length = ntpdata->tick_length_base;
-
- delta = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
- ntpdata->time_offset -= delta;
- ntpdata->tick_length += delta;
/* Check PPS signal */
pps_dec_valid(ntpdata);
- if (!ntpdata->time_adjust)
- goto out;
+ /*
+ * Set the per-tick skew rate for the next second. This is in
+ * the same units as time_offset: (ns << NTP_SCALE_SHIFT) / HZ.
+ * If the result is so low that the skew imparted would round
+ * to zero, pass the bare minimum ±1 to ensure that it *does*
+ * actually drain completely to zero. It won't overshoot because
+ * logarithmic_accumulation() only drains what it can from
+ * time_offset or time_adjust, and the rest ends up in ntp_error
+ * which drives the selection of 'mult' immediately each tick.
+ */
+ if (ntpdata->time_offset || ntpdata->time_adjust ||
+ ntpdata->time_adjust_frac) {
+ s64 off_chunk = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
+ s64 adj_chunk = 0, net;
- if (ntpdata->time_adjust > MAX_TICKADJ) {
- ntpdata->time_adjust -= MAX_TICKADJ;
- ntpdata->tick_length += MAX_TICKADJ_SCALED;
- goto out;
- }
+ /*
+ * Once the exponential chunk rounds to zero, deliver the last
+ * remaining offset this second so it converges to zero instead
+ * of stalling just above it.
+ */
+ if (!off_chunk)
+ off_chunk = ntpdata->time_offset;
+
+ if (ntpdata->time_adjust || ntpdata->time_adjust_frac) {
+ s64 adj;
+
+ if (ntpdata->time_adjust >= MAX_TICKADJ)
+ adj = MAX_TICKADJ * ONE_US_NS;
+ else if (ntpdata->time_adjust <= -MAX_TICKADJ)
+ adj = -MAX_TICKADJ * ONE_US_NS;
+ else
+ adj = ntpdata->time_adjust * ONE_US_NS +
+ ntpdata->time_adjust_frac;
+
+ adj_chunk = div_s64(adj, NTP_INTERVAL_FREQ);
+ if (!adj_chunk)
+ adj_chunk = signof(ntpdata->time_adjust_frac);
+ }
- if (ntpdata->time_adjust < -MAX_TICKADJ) {
- ntpdata->time_adjust += MAX_TICKADJ;
- ntpdata->tick_length -= MAX_TICKADJ_SCALED;
- goto out;
- }
+ /*
+ * If the two slews oppose, only their net would drive the
+ * per-tick drain, so the cancelling part would never drain from
+ * either tracker and an exact cancellation would stall both.
+ * Settle that overlap directly between them (no clock motion).
+ */
+ if (off_chunk && adj_chunk && signof(off_chunk) != signof(adj_chunk)) {
+ s64 conflict = min(abs(off_chunk), abs(adj_chunk));
- ntpdata->tick_length += (s64)(ntpdata->time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
- << NTP_SCALE_SHIFT;
- ntpdata->time_adjust = 0;
+ ntp_transfer_offset_adjust(ntpdata, signof(off_chunk) * conflict);
+ }
+
+ /* Net is what the clock delivers; reduce to per-tick, then floor. */
+ net = off_chunk + adj_chunk;
+ ntpdata->skew_delta = div_s64(net, NTP_INTERVAL_FREQ);
+ if (!ntpdata->skew_delta && net)
+ ntpdata->skew_delta = signof(net);
+ } else {
+ ntpdata->skew_delta = 0;
+ }
-out:
return leap;
}
@@ -779,6 +1021,7 @@ int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct tim
if (!(txc->modes & ADJ_OFFSET_READONLY)) {
/* adjtime() is independent from ntp_adjtime() */
ntpdata->time_adjust = txc->offset;
+ ntpdata->time_adjust_frac = 0;
ntp_update_frequency(ntpdata);
audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust);
@@ -1020,6 +1263,7 @@ static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
NTP_INTERVAL_FREQ);
/* Cancel running adjtime() */
ntpdata->time_adjust = 0;
+ ntpdata->time_adjust_frac = 0;
}
/* Update jitter */
ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN;
diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h
index 7084d839c207..0474a761bafc 100644
--- a/kernel/time/ntp_internal.h
+++ b/kernel/time/ntp_internal.h
@@ -3,9 +3,11 @@
#define _LINUX_NTP_INTERNAL_H
extern void ntp_init(void);
-extern void ntp_clear(unsigned int tkid);
+extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj);
/* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
extern u64 ntp_tick_length(unsigned int tkid);
+extern s64 ntp_get_skew_delta(unsigned int tkid);
+extern s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift);
extern ktime_t ntp_get_next_leap(unsigned int tkid);
extern int second_overflow(unsigned int tkid, time64_t secs);
extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index a7d3e8229c4b..d73d31c7994f 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -1357,8 +1357,11 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
unsigned long flags, start;
LIST_HEAD(firing);
- if (!lock_task_sighand(tsk, &flags))
- return;
+ /*
+ * tsk is current and ->sighand is stable, see the
+ * tsk->exit_state check in run_posix_cpu_timers()
+ */
+ spin_lock_irqsave(&tsk->sighand->siglock, flags);
do {
/*
@@ -1418,7 +1421,7 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
* that gets the timer lock before we do will give it up and
* spin until we've taken care of that timer below.
*/
- unlock_task_sighand(tsk, &flags);
+ spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
/*
* Now that all the timers on our list have the firing flag,
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 597d816d22e8..182974c4f21b 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -3,6 +3,7 @@
* tick internal variable and functions used by low/high res code
*/
#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/tick.h>
#include "timekeeping.h"
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 0dd63a91e7c5..d1a7efd80bf5 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -42,6 +42,7 @@
#include <generated/timeconst.h>
#include "timekeeping.h"
+#include "timekeeping_internal.h"
/*
* The timezone where the local system is located. Used as a default by some
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index b1b5ec43c0f2..ea2e6e55f37b 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { }
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
{
u64 interval;
- u64 tmp, ntpinterval;
struct clocksource *old_clock;
++tk->cs_was_changed_seq;
@@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
/* Do the ns -> cycle conversion first, using original mult */
- tmp = NTP_INTERVAL_LENGTH;
- tmp <<= clock->shift;
- ntpinterval = tmp;
- tmp += clock->mult/2;
- do_div(tmp, clock->mult);
- if (tmp == 0)
- tmp = 1;
-
- interval = (u64) tmp;
+ interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
+ interval += clock->mult / 2;
+ do_div(interval, clock->mult);
+ if (interval == 0)
+ interval = 1;
+
tk->cycle_interval = interval;
/* Go back from cycles -> shifted ns */
tk->xtime_interval = interval * clock->mult;
- tk->xtime_remainder = ntpinterval - tk->xtime_interval;
tk->raw_interval = interval * clock->mult;
/* if changing clocks, convert xtime_nsec shift units */
@@ -386,7 +381,38 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->ntp_error = 0;
tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
- tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
+
+ /*
+ * ntp_tick is the tick length that NTP disciplines (its ±500 PPM
+ * scales only this part), in NTP-shifted ns: the real interval of
+ * a whole number of counter cycles. Because cycle_interval is
+ * rounded to an integer number of cycles, this ntp_tick differs
+ * from the true intended 1/HZ tick length by up to half a cycle
+ * period.
+ */
+ tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift;
+
+ /*
+ * cs_tick_adj is the constant difference between the disciplined
+ * ntp_tick above and the true 1/HZ tick, expressed per-second to
+ * match the ntp_update_frequency() addends and handed to NTP via
+ * ntp_clear() to be explicitly included in its tick_length.
+ *
+ * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which
+ * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.
+ *
+ * So ntp_tick is actually 1.000127ms, as that is the amount of
+ * time that 3580 cycles will take at the nominal frequency. This
+ * is the part that NTP disciplines, causing each 3580 counts to
+ * advance the clock by up to NTP's ±500PPM of that amount.
+ *
+ * The "extra" 127ns/tick is what's stored in cs_tick_adj and
+ * applied as a constant correction by ntp_update_frequency() so
+ * that NTP *believes* it's disciplining a 1ms tick.
+ */
+ tk->cs_tick_adj = (s64)tk->ntp_tick -
+ ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);
+ tk->cs_tick_adj *= NTP_INTERVAL_FREQ;
/*
* The timekeeper keeps its own mult values for the currently
@@ -397,6 +423,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->tkr_raw.mult = clock->mult;
tk->ntp_err_mult = 0;
tk->skip_second_overflow = 0;
+ tk->skew_delta = 0;
tk->cs_id = clock->id;
@@ -803,7 +830,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
if (action & TK_CLEAR_NTP) {
tk->ntp_error = 0;
- ntp_clear(tk->id);
+ ntp_clear(tk->id, tk->cs_tick_adj);
}
tk_update_leap_state(tk);
@@ -831,7 +858,11 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
* the downside that the reader side does not longer benefit from
* the cacheline optimized data layout of the timekeeper and requires
* another indirection.
+ *
+ * Write xtime_sec first so that even if the memcpy() tears the store
+ * data integrity is provided for ktime_get_real_seconds().
*/
+ WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec);
memcpy(&tkd->timekeeper, tk, sizeof(*tk));
write_seqcount_end(&tkd->seq);
}
@@ -1159,11 +1190,11 @@ time64_t ktime_get_real_seconds(void)
unsigned int seq;
if (IS_ENABLED(CONFIG_64BIT))
- return tk->xtime_sec;
+ return READ_ONCE(tk->xtime_sec);
do {
seq = read_seqcount_begin(&tk_core.seq);
- seconds = tk->xtime_sec;
+ seconds = READ_ONCE(tk->xtime_sec);
} while (read_seqcount_retry(&tk_core.seq, seq));
@@ -1185,7 +1216,7 @@ noinstr time64_t __ktime_get_real_seconds(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
- return tk->xtime_sec;
+ return READ_ONCE(tk->xtime_sec);
}
static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
@@ -1202,10 +1233,21 @@ static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
/**
* ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with
- * CLOCK_MONOTONIC_RAW and the underlying
+ * the corresponding monotonic raw and the underlying
* clocksource counter value.
* @clock_id: The clock ID to snapshot
* @systime_snapshot: Pointer to struct receiving the system time snapshot
+ *
+ * For the system time keeping clocks (REALTIME, MONOTONIC and BOOTTIME) the
+ * monotonic raw clock is CLOCK_MONOTONIC_RAW. For AUX clocks this is the
+ * monotonic raw clock related to the AUX clock. These AUX clock related
+ * monotonic raw clocks have a strict linear offset to the system time
+ * CLOCK_MONOTONIC_RAW:
+ *
+ * MONOTONIC_RAW(AUX$N) = CLOCK_MONOTONIC_RAW(system) + offset(AUX$N)
+ *
+ * The offset is established when a AUX clock is initialized, but it is
+ * currently not accessible.
*/
void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot)
{
@@ -1512,6 +1554,9 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);
* @xtstamp: Receives simultaneously captured system and device time
*
* Reads a timestamp from a device and correlates it to system time
+ *
+ * See documentation for ktime_get_snapshot_id() for information about the raw
+ * monotonic time stamp which is used here.
*/
int get_device_system_crosststamp(int (*get_time_fn)
(ktime_t *device_time,
@@ -1522,10 +1567,11 @@ int get_device_system_crosststamp(int (*get_time_fn)
struct system_device_crosststamp *xtstamp)
{
u64 syscnt_cycles, cycles, now, interval_start;
- unsigned int seq, clock_was_set_seq = 0;
ktime_t base_sys, base_raw, *offs;
+ u32 clock_was_set_seq = 0;
u64 nsec_sys, nsec_raw;
u8 cs_was_changed_seq;
+ unsigned int seq;
bool do_interp;
struct timekeeper *tk;
struct tk_data *tkd;
@@ -1897,40 +1943,6 @@ void ktime_get_raw_ts64(struct timespec64 *ts)
EXPORT_SYMBOL(ktime_get_raw_ts64);
/**
- * ktime_get_clock_ts64 - Returns time of a clock in a timespec
- * @id: POSIX clock ID of the clock to read
- * @ts: Pointer to the timespec64 to be set
- *
- * The timestamp is invalidated (@ts->sec is set to -1) if the
- * clock @id is not available.
- */
-void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts)
-{
- /* Invalidate time stamp */
- ts->tv_sec = -1;
- ts->tv_nsec = 0;
-
- switch (id) {
- case CLOCK_REALTIME:
- ktime_get_real_ts64(ts);
- return;
- case CLOCK_MONOTONIC:
- ktime_get_ts64(ts);
- return;
- case CLOCK_MONOTONIC_RAW:
- ktime_get_raw_ts64(ts);
- return;
- case CLOCK_AUX ... CLOCK_AUX_LAST:
- if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
- ktime_get_aux_ts64(id, ts);
- return;
- default:
- WARN_ON_ONCE(1);
- }
-}
-EXPORT_SYMBOL_GPL(ktime_get_clock_ts64);
-
-/**
* timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
*/
int timekeeping_valid_for_hres(void)
@@ -2076,7 +2088,12 @@ void __init timekeeping_init(void)
tk_set_wall_to_mono(tks, wall_to_mono);
- timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
+ /*
+ * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's
+ * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise
+ * redundant as ntp_init() already initialised it above.
+ */
+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
}
/* time in seconds when suspend began for persistent clock */
@@ -2390,6 +2407,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
* xtime_nsec_2 = xtime_nsec_1 - offset
* Which simplifies to:
* xtime_nsec -= offset
+ *
+ * When subtracting offset from xtime_nsec, the same amount
+ * (in appropriate units) has to be added to ntp_error, in
+ * order to correctly track the delta between the time
+ * reported in xtime_nsec, and the intended time.
*/
if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
/* NTP adjustment caused clocksource mult overflow */
@@ -2400,6 +2422,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
tk->tkr_mono.mult += mult_adj;
tk->xtime_interval += interval;
tk->tkr_mono.xtime_nsec -= offset;
+ tk->ntp_error += offset << tk->ntp_error_shift;
}
/*
@@ -2409,18 +2432,27 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
u64 ntp_tl = ntp_tick_length(tk->id);
+ s64 skew = ntp_get_skew_delta(tk->id);
u32 mult;
/*
- * Determine the multiplier from the current NTP tick length.
- * Avoid expensive division when the tick length doesn't change.
+ * Determine the multiplier from the current NTP tick length plus
+ * skew_delta. The skew biases mult so that ±1 dithering can deliver
+ * the time_offset slew rate. Recompute when either changes.
*/
- if (likely(tk->ntp_tick == ntp_tl)) {
+ if (likely(tk->ntp_tick == ntp_tl && tk->skew_delta == skew)) {
+ /* Revert to the base mult rate. */
mult = tk->tkr_mono.mult - tk->ntp_err_mult;
} else {
tk->ntp_tick = ntp_tl;
- mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
- tk->xtime_remainder, tk->cycle_interval);
+ tk->skew_delta = skew;
+ /*
+ * skew_delta is stored pre-divided by HZ (matching time_offset);
+ * scale it back up to the full per-tick rate for the mult bias.
+ */
+ skew *= NTP_INTERVAL_FREQ;
+ mult = div64_u64((tk->ntp_tick + skew) >> tk->ntp_error_shift,
+ tk->cycle_interval);
}
/*
@@ -2545,8 +2577,25 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
/* Accumulate error between NTP and clock interval */
tk->ntp_error += tk->ntp_tick << shift;
- tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
- (tk->ntp_error_shift + shift);
+ tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift);
+
+ /*
+ * When skewing, do so by adjusting ntp_error to impart an extra
+ * target delta into ntp_error per tick, limited to what can be
+ * drained from time_offset / time_adjust to avoid overshoot.
+ *
+ * The base 'mult' value was calculated with the skew taken into
+ * account, such that the per-tick choice of 'mult' vs. 'mult+1'
+ * allows for the desired effective rate and ntp_error does not
+ * grow unbounded.
+ *
+ * Once the full desired phase offset is delivered, any remaining
+ * skew imparted by the adjusted 'mult', accounted above, remains
+ * in ntp_error and will be compensated by the dithering over time.
+ */
+ if (tk->skew_delta)
+ tk->ntp_error += ntp_drain_skew(tk->id, tk->skew_delta << shift,
+ shift) * NTP_INTERVAL_FREQ;
return offset;
}
@@ -2795,7 +2844,7 @@ void do_timer(unsigned long ticks)
*
* Called from hrtimer_interrupt() or retrigger_next_event()
*/
-ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
+ktime_t ktime_get_update_offsets_now(u32 *cwsseq, ktime_t *offs_real,
ktime_t *offs_boot, ktime_t *offs_tai)
{
struct timekeeper *tk = &tk_core.timekeeper;
@@ -2943,10 +2992,12 @@ static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc,
return ret;
add_device_randomness(txc, sizeof(*txc));
- if (!aux_clock)
+ if (!aux_clock) {
ktime_get_real_ts64(&ts);
- else
- tk_get_aux_ts64(tkd->timekeeper.id, &ts);
+ } else {
+ if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts))
+ return -ENODEV;
+ }
add_device_randomness(&ts, sizeof(ts));
@@ -3051,7 +3102,7 @@ static inline unsigned int clockid_to_tkid(unsigned int id)
static inline struct tk_data *aux_get_tk_data(clockid_t id)
{
- if (!clockid_aux_valid(id))
+ if (!clockid_is_aux_clock(id))
return NULL;
return &timekeeper_data[clockid_to_tkid(id)];
}
@@ -3146,7 +3197,7 @@ EXPORT_SYMBOL_GPL(ktime_get_aux_ts64);
static int aux_get_res(clockid_t id, struct timespec64 *tp)
{
- if (!clockid_aux_valid(id))
+ if (!clockid_is_aux_clock(id))
return -ENODEV;
tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC;
@@ -3313,7 +3364,9 @@ static const struct attribute_group aux_clock_enable_attr_group = {
static int __init tk_aux_sysfs_init(void)
{
struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj);
+ struct kobject *clks[MAX_AUX_CLOCKS];
int ret = -ENOMEM;
+ int i;
if (!tko)
return ret;
@@ -3322,21 +3375,28 @@ static int __init tk_aux_sysfs_init(void)
if (!auxo)
goto err_clean;
- for (int i = 0; i < MAX_AUX_CLOCKS; i++) {
+ for (i = 0; i < MAX_AUX_CLOCKS; i++) {
char id[2] = { [0] = '0' + i, };
- struct kobject *clk = kobject_create_and_add(id, auxo);
+ clks[i] = kobject_create_and_add(id, auxo);
- if (!clk) {
+ if (!clks[i]) {
ret = -ENOMEM;
- goto err_clean;
+ goto err_clks;
}
- ret = sysfs_create_group(clk, &aux_clock_enable_attr_group);
+ ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group);
if (ret)
- goto err_clean;
+ goto err_clk;
}
return 0;
+err_clk:
+ kobject_put(clks[i]);
+err_clks:
+ while (--i >= 0) {
+ sysfs_remove_group(clks[i], &aux_clock_enable_attr_group);
+ kobject_put(clks[i]);
+ }
err_clean:
kobject_put(auxo);
kobject_put(tko);
diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h
index 198d0608db74..4201f9e90813 100644
--- a/kernel/time/timekeeping.h
+++ b/kernel/time/timekeeping.h
@@ -4,7 +4,7 @@
/*
* Internal interfaces for kernel/time/
*/
-extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq,
+extern ktime_t ktime_get_update_offsets_now(u32 *cwsseq,
ktime_t *offs_real,
ktime_t *offs_boot,
ktime_t *offs_tai);
diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h
index 973ede670a36..6d719b8e5ea2 100644
--- a/kernel/time/timekeeping_internal.h
+++ b/kernel/time/timekeeping_internal.h
@@ -6,6 +6,8 @@
#include <linux/spinlock.h>
#include <linux/time.h>
+struct timekeeper;
+
/*
* timekeeping debug functions
*/
@@ -48,4 +50,23 @@ void timekeeper_unlock_irqrestore(unsigned long flags);
/* NTP specific interface to access the current seconds value */
long ktime_get_ntp_seconds(unsigned int id);
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
+
+extern void update_vsyscall(struct timekeeper *tk);
+extern void update_vsyscall_tz(void);
+extern void vdso_time_update_aux(struct timekeeper *tk);
+
+#else
+
+static inline void update_vsyscall(struct timekeeper *tk)
+{
+}
+static inline void update_vsyscall_tz(void)
+{
+}
+static inline void vdso_time_update_aux(struct timekeeper *tk)
+{
+}
+#endif
+
#endif /* _TIMEKEEPING_INTERNAL_H */
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 514802def1e0..0406bf4488e9 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -20,7 +20,7 @@
struct timer_list_iter {
int cpu;
bool second_pass;
- u64 now;
+ ktime_t now;
};
/*
@@ -44,19 +44,19 @@ static void SEQ_printf(struct seq_file *m, const char *fmt, ...)
static void
print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
- int idx, u64 now)
+ int idx, ktime_t now)
{
SEQ_printf(m, " #%d: <%p>, %ps", idx, taddr, ACCESS_PRIVATE(timer, function));
SEQ_printf(m, ", S:%02x", timer->is_queued);
SEQ_printf(m, "\n");
- SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n",
- (unsigned long long)ktime_to_ns(hrtimer_get_softexpires(timer)),
- (unsigned long long)ktime_to_ns(hrtimer_get_expires(timer)),
- (long long)(ktime_to_ns(hrtimer_get_softexpires(timer)) - now),
- (long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now));
+ SEQ_printf(m, " # expires at %lld-%lld nsecs [in %lld to %lld nsecs]\n",
+ (long long)hrtimer_get_softexpires(timer),
+ (long long)hrtimer_get_expires(timer),
+ (long long)ktime_sub(hrtimer_get_softexpires(timer), now),
+ (long long)ktime_sub(hrtimer_get_expires(timer), now));
}
-static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
+static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, ktime_t now)
{
struct timerqueue_linked_node *curr;
struct hrtimer *timer, tmp;
@@ -94,21 +94,21 @@ next_one:
}
static void
-print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
+print_base(struct seq_file *m, struct hrtimer_clock_base *base, ktime_t now)
{
SEQ_printf(m, " .base: %p\n", base);
SEQ_printf(m, " .index: %d\n", base->index);
SEQ_printf(m, " .resolution: %u nsecs\n", hrtimer_resolution);
#ifdef CONFIG_HIGH_RES_TIMERS
- SEQ_printf(m, " .offset: %Ld nsecs\n",
+ SEQ_printf(m, " .offset: %lld nsecs\n",
(long long) base->offset);
#endif
SEQ_printf(m, "active timers:\n");
- print_active_timers(m, base, now + ktime_to_ns(base->offset));
+ print_active_timers(m, base, ktime_add(now, base->offset));
}
-static void print_cpu(struct seq_file *m, int cpu, u64 now)
+static void print_cpu(struct seq_file *m, int cpu, ktime_t now)
{
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
int i;
@@ -118,15 +118,17 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
SEQ_printf(m, " clock %d:\n", i);
print_base(m, cpu_base->clock_base + i, now);
}
-#define P(x) \
- SEQ_printf(m, " .%-15s: %Lu\n", #x, \
- (unsigned long long)(cpu_base->x))
-#define P_ns(x) \
- SEQ_printf(m, " .%-15s: %Lu nsecs\n", #x, \
- (unsigned long long)(ktime_to_ns(cpu_base->x)))
+
+#define DIAG_READ(x) data_race(READ_ONCE(x))
+
+#define P(x) \
+ SEQ_printf(m, " .%-15s: %llu\n", #x, \
+ (unsigned long long)DIAG_READ(cpu_base->x))
+#define P_ktime(x) \
+ SEQ_printf(m, " .%-15s: %lld nsecs\n", #x, (long long)DIAG_READ(cpu_base->x))
#ifdef CONFIG_HIGH_RES_TIMERS
- P_ns(expires_next);
+ P_ktime(expires_next);
P(hres_active);
P(nr_events);
P(nr_retries);
@@ -134,38 +136,39 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
P(max_hang_time);
#endif
#undef P
-#undef P_ns
+#undef P_ktime
#ifdef CONFIG_TICK_ONESHOT
# define P(x) \
- SEQ_printf(m, " .%-15s: %Lu\n", #x, \
- (unsigned long long)(ts->x))
-# define P_ns(x) \
- SEQ_printf(m, " .%-15s: %Lu nsecs\n", #x, \
- (unsigned long long)(ktime_to_ns(ts->x)))
+ SEQ_printf(m, " .%-15s: %llu\n", #x, \
+ (unsigned long long)DIAG_READ(ts->x))
+# define P_ktime(x) \
+ SEQ_printf(m, " .%-15s: %lld nsecs\n", #x, (long long)DIAG_READ(ts->x))
# define P_flag(x, f) \
- SEQ_printf(m, " .%-15s: %d\n", #x, !!(ts->flags & (f)))
+ SEQ_printf(m, " .%-15s: %d\n", #x, !!(DIAG_READ(ts->flags) & (f)))
{
struct tick_sched *ts = tick_get_tick_sched(cpu);
P_flag(nohz, TS_FLAG_NOHZ);
P_flag(highres, TS_FLAG_HIGHRES);
- P_ns(last_tick);
+ P_ktime(last_tick);
P_flag(tick_stopped, TS_FLAG_STOPPED);
P(idle_calls);
P(idle_sleeps);
- P_ns(idle_entrytime);
- P_ns(idle_waketime);
+ P_ktime(idle_entrytime);
+ P_ktime(idle_waketime);
P(last_jiffies);
P(next_timer);
- P_ns(idle_expires);
- SEQ_printf(m, "jiffies: %Lu\n",
+ P_ktime(idle_expires);
+ SEQ_printf(m, "jiffies: %llu\n",
(unsigned long long)jiffies);
}
#endif
#undef P
-#undef P_ns
+#undef P_ktime
+#undef P_flag
+#undef DIAG_READ
SEQ_printf(m, "\n");
}
@@ -196,8 +199,7 @@ print_tickdevice(struct seq_file *m, struct tick_device *td, int cpu)
SEQ_printf(m, " mult: %u\n", dev->mult);
SEQ_printf(m, " shift: %u\n", dev->shift);
SEQ_printf(m, " mode: %d\n", clockevent_get_state(dev));
- SEQ_printf(m, " next_event: %Ld nsecs\n",
- (unsigned long long) ktime_to_ns(dev->next_event));
+ SEQ_printf(m, " next_event: %lld nsecs\n", (long long)dev->next_event);
SEQ_printf(m, " set_next_event: %ps\n", dev->set_next_event);
@@ -250,17 +252,17 @@ static void timer_list_show_tickdevices_header(struct seq_file *m)
}
#endif
-static inline void timer_list_header(struct seq_file *m, u64 now)
+static inline void timer_list_header(struct seq_file *m, ktime_t now)
{
SEQ_printf(m, "Timer List Version: v0.11\n");
SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES);
- SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now);
+ SEQ_printf(m, "now at %lld nsecs\n", (long long)now);
SEQ_printf(m, "\n");
}
void sysrq_timer_list_show(void)
{
- u64 now = ktime_to_ns(ktime_get());
+ ktime_t now = ktime_get();
int cpu;
timer_list_header(NULL, now);
@@ -318,7 +320,7 @@ static void *timer_list_start(struct seq_file *file, loff_t *offset)
struct timer_list_iter *iter = file->private;
if (!*offset)
- iter->now = ktime_to_ns(ktime_get());
+ iter->now = ktime_get();
iter->cpu = -1;
iter->second_pass = false;
return move_iter(iter, *offset);
diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c
index 806c23cf71fc..059d43355e65 100644
--- a/kernel/time/timer_migration.c
+++ b/kernel/time/timer_migration.c
@@ -1847,8 +1847,10 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu,
}
/* Assert single root without parent */
- if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels))
+ if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) {
+ kfree(stack);
return -EINVAL;
+ }
for (; i >= start_lvl; i--) {
group = stack[i];
diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h
index 31735dd52327..c9c1c29f011d 100644
--- a/kernel/time/timer_migration.h
+++ b/kernel/time/timer_migration.h
@@ -103,7 +103,7 @@ struct tmigr_group {
* before the timer migration hierarchy hotplug callback is
* reached. During this phase, the CPU has to handle the
* global timers on its own and must not act as a migrator.
-
+ *
* @idle: Indicates whether the CPU is idle in the timer migration
* hierarchy
* @remote: Is set when timers of the CPU are expired remotely