diff options
| author | Uladzislau Rezki (Sony) <urezki@gmail.com> | 2026-06-02 19:45:08 +0200 |
|---|---|---|
| committer | Uladzislau Rezki (Sony) <urezki@gmail.com> | 2026-06-02 19:45:08 +0200 |
| commit | e853c1b28580ea93fda3cd729e440a3fc16fa647 (patch) | |
| tree | 49264154ae9d7fca30a2fa65df34de6a8f7e36e3 | |
| parent | 956d852943621dff1d93fa6fa303d03fc0c869f3 (diff) | |
| parent | 002668809b068c528838c1ab1ff46c87bdbb095d (diff) | |
Merge branches 'rcutorture.2026.05.24' and 'misc.2026.05.24' into rcu-merge.2026.05.24refs/merge-window/d0426043cbc2fc676d6152b911adbeaf2a2cc1f1
rcutorture.2026.05.24: Torture-test updates
misc.2026.05.24: Miscellaneous RCU updates
| -rw-r--r-- | Documentation/admin-guide/kernel-parameters.txt | 10 | ||||
| -rw-r--r-- | include/linux/rcupdate.h | 12 | ||||
| -rw-r--r-- | include/linux/srcu.h | 2 | ||||
| -rw-r--r-- | kernel/rcu/tasks.h | 3 | ||||
| -rw-r--r-- | kernel/rcu/tree.c | 56 | ||||
| -rw-r--r-- | kernel/rcu/tree_nocb.h | 2 | ||||
| -rwxr-xr-x | scripts/checkpatch.pl | 5 |
7 files changed, 60 insertions, 30 deletions
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4d0f545fb3ec..d5db2e85d551 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5862,13 +5862,13 @@ Kernel parameters use a call_rcu[_hurry]() path. Please note, this is for a normal grace period. - How to enable it: + How to disable it: - echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp - or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" + echo 0 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp + or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=0" - Default is 1 if num_possible_cpus() <= 16 and it is not explicitly - disabled by the boot parameter passing 0. + Default is 1 if it is not explicitly disabled by the boot parameter + passing 0. rcuscale.gp_async= [KNL] Measure performance of asynchronous diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index bfa765132de8..5e95acc33989 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -592,11 +592,13 @@ context_unsafe( \ * lockdep checks for being in an RCU read-side critical section. This is * useful when the value of this pointer is accessed, but the pointer is * not dereferenced, for example, when testing an RCU-protected pointer - * against NULL. Although rcu_access_pointer() may also be used in cases - * where update-side locks prevent the value of the pointer from changing, - * you should instead use rcu_dereference_protected() for this use case. - * Within an RCU read-side critical section, there is little reason to - * use rcu_access_pointer(). + * against NULL. Within an RCU read-side critical section, there is little + * reason to use rcu_access_pointer(). Although rcu_access_pointer() may + * also be used in cases where update-side locks prevent the value of the + * pointer from changing, you should instead use rcu_dereference_protected() + * for this use case. It is also permissible to use rcu_access_pointer() + * within lockless updaters to obtain the old value for an atomic operation, + * for example, for cmpxchg(). * * It is usually best to test the rcu_access_pointer() return value * directly in order to avoid accidental dereferences being introduced diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 81b1938512d5..a54ce9e808b9 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -397,7 +397,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ * * The same srcu_struct may be used concurrently by srcu_down_read_fast() * and srcu_read_lock_fast(). However, the same definition/initialization - * requirements called out for srcu_read_lock_safe() apply. + * requirements called out for srcu_read_lock_fast_updown() apply. */ static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) { diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 48f0d803c8e2..f4da5fad70f5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, // Queuing callbacks before initialization not yet supported. if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) rcu_segcblist_init(&rtpcp->cblist); - needwake = (func == wakeme_after_rcu) || + needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || + (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 55df6d37145e..afb9e7db8f78 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param int ret = kstrtoul(val, 0, &j); if (!ret) { - WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1)); + WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); adjust_jiffies_till_sched_qs(); } return ret; @@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) atomic_set_release(&sr_wn->inuse, 0); } -/* Enable rcu_normal_wake_from_gp automatically on small systems. */ -#define WAKE_FROM_GP_CPU_THRESHOLD 16 - -static int rcu_normal_wake_from_gp = -1; +static int rcu_normal_wake_from_gp = 1; module_param(rcu_normal_wake_from_gp, int, 0644); static struct workqueue_struct *sync_wq; +#define RCU_SR_NORMAL_LATCH_THR 64 + +/* Number of in-flight synchronize_rcu() calls queued on srs_next. */ +static atomic_long_t rcu_sr_normal_count; +static int rcu_sr_normal_latched; /* 0/1 */ + static void rcu_sr_normal_complete(struct llist_node *node) { struct rcu_synchronize *rs = container_of( (struct rcu_head *) node, struct rcu_synchronize, head); + long nr; WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && !poll_state_synchronize_rcu_full(&rs->oldstate), @@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) /* Finally. */ complete(&rs->completion); + nr = atomic_long_dec_return(&rcu_sr_normal_count); + WARN_ON_ONCE(nr < 0); + + /* + * Unlatch: switch back to normal path when fully + * drained and if it has been latched. + */ + if (nr == 0) + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0); } static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work) @@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) static void rcu_sr_normal_add_req(struct rcu_synchronize *rs) { + /* + * Increment before publish to avoid a complete + * vs enqueue race on latch. + */ + long nr = atomic_long_inc_return(&rcu_sr_normal_count); + + /* + * Latch when threshold is reached. Checking for an exact match + * restricts cmpxchg() to a single context. + * + * This latch is intentionally relaxed and best-effort. Concurrent + * set/clear can race and temporarily lose the latch, which is OK + * because it only selects between the fast and fallback paths. + */ + if (nr == RCU_SR_NORMAL_LATCH_THR) + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1); + + /* Publish for the GP kthread/worker. */ llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); } @@ -2584,7 +2615,7 @@ static void rcu_do_batch(struct rcu_data *rdp) const long npj = NSEC_PER_SEC / HZ; long rrn = READ_ONCE(rcu_resched_ns); - rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn; + rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC); tlimit = local_clock() + rrn; jlimit = jiffies + (rrn + npj + 1) / npj; jlimit_check = true; @@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) { struct rcu_synchronize rs; + init_rcu_head_on_stack(&rs.head); trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); - if (READ_ONCE(rcu_normal_wake_from_gp) < 1) { + if (READ_ONCE(rcu_normal_wake_from_gp) < 1 || + READ_ONCE(rcu_sr_normal_latched)) { wait_rcu_gp(call_rcu_hurry); goto trace_complete_out; } - init_rcu_head_on_stack(&rs.head); init_completion(&rs.completion); /* @@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) /* Now we can wait. */ wait_for_completion(&rs.completion); - destroy_rcu_head_on_stack(&rs.head); trace_complete_out: trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); + destroy_rcu_head_on_stack(&rs.head); } /** @@ -4904,12 +4936,6 @@ void __init rcu_init(void) sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); WARN_ON(!sync_wq); - /* Respect if explicitly disabled via a boot parameter. */ - if (rcu_normal_wake_from_gp < 0) { - if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) - rcu_normal_wake_from_gp = 1; - } - /* Fill in default value for rcutree.qovld boot parameter. */ /* -After- the rcu_node ->lock fields are initialized! */ if (qovld < 0) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 1047b30cd46b..373b877cf171 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) * No-CBs GP kthreads come here to wait for additional callbacks to show up * or for grace periods to end. */ -static void nocb_gp_wait(struct rcu_data *my_rdp) +static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 0492d6afc9a1..cc5bbd70cb84 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -865,8 +865,6 @@ our %deprecated_apis = ( "DEFINE_IDR" => "DEFINE_XARRAY", "idr_init" => "xa_init", "idr_init_base" => "xa_init_flags", - "rcu_read_lock_trace" => "rcu_read_lock_tasks_trace", - "rcu_read_unlock_trace" => "rcu_read_unlock_tasks_trace", ); #Create a search pattern for all these strings to speed up a loop below @@ -7596,12 +7594,15 @@ sub process { # Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_tasks_trace | rcu_read_lock_trace | rcu_read_lock_trace_held | rcu_read_unlock_trace | + rcu_read_unlock_tasks_trace | call_rcu_tasks_trace | synchronize_rcu_tasks_trace | rcu_barrier_tasks_trace | + rcu_tasks_trace_expedite_current | rcu_request_urgent_qs_task )}; our $rcu_trace_paths = qr{(?x: |
