summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorUladzislau Rezki (Sony) <urezki@gmail.com>2026-06-02 19:45:08 +0200
committerUladzislau Rezki (Sony) <urezki@gmail.com>2026-06-02 19:45:08 +0200
commite853c1b28580ea93fda3cd729e440a3fc16fa647 (patch)
tree49264154ae9d7fca30a2fa65df34de6a8f7e36e3
parent956d852943621dff1d93fa6fa303d03fc0c869f3 (diff)
parent002668809b068c528838c1ab1ff46c87bdbb095d (diff)
Merge branches 'rcutorture.2026.05.24' and 'misc.2026.05.24' into rcu-merge.2026.05.24refs/merge-window/d0426043cbc2fc676d6152b911adbeaf2a2cc1f1
rcutorture.2026.05.24: Torture-test updates misc.2026.05.24: Miscellaneous RCU updates
-rw-r--r--Documentation/admin-guide/kernel-parameters.txt10
-rw-r--r--include/linux/rcupdate.h12
-rw-r--r--include/linux/srcu.h2
-rw-r--r--kernel/rcu/tasks.h3
-rw-r--r--kernel/rcu/tree.c56
-rw-r--r--kernel/rcu/tree_nocb.h2
-rwxr-xr-xscripts/checkpatch.pl5
7 files changed, 60 insertions, 30 deletions
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 4d0f545fb3ec..d5db2e85d551 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5862,13 +5862,13 @@ Kernel parameters
use a call_rcu[_hurry]() path. Please note, this is for a
normal grace period.
- How to enable it:
+ How to disable it:
- echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp
- or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1"
+ echo 0 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp
+ or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=0"
- Default is 1 if num_possible_cpus() <= 16 and it is not explicitly
- disabled by the boot parameter passing 0.
+ Default is 1 if it is not explicitly disabled by the boot parameter
+ passing 0.
rcuscale.gp_async= [KNL]
Measure performance of asynchronous
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index bfa765132de8..5e95acc33989 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -592,11 +592,13 @@ context_unsafe( \
* lockdep checks for being in an RCU read-side critical section. This is
* useful when the value of this pointer is accessed, but the pointer is
* not dereferenced, for example, when testing an RCU-protected pointer
- * against NULL. Although rcu_access_pointer() may also be used in cases
- * where update-side locks prevent the value of the pointer from changing,
- * you should instead use rcu_dereference_protected() for this use case.
- * Within an RCU read-side critical section, there is little reason to
- * use rcu_access_pointer().
+ * against NULL. Within an RCU read-side critical section, there is little
+ * reason to use rcu_access_pointer(). Although rcu_access_pointer() may
+ * also be used in cases where update-side locks prevent the value of the
+ * pointer from changing, you should instead use rcu_dereference_protected()
+ * for this use case. It is also permissible to use rcu_access_pointer()
+ * within lockless updaters to obtain the old value for an atomic operation,
+ * for example, for cmpxchg().
*
* It is usually best to test the rcu_access_pointer() return value
* directly in order to avoid accidental dereferences being introduced
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 81b1938512d5..a54ce9e808b9 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -397,7 +397,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_
*
* The same srcu_struct may be used concurrently by srcu_down_read_fast()
* and srcu_read_lock_fast(). However, the same definition/initialization
- * requirements called out for srcu_read_lock_safe() apply.
+ * requirements called out for srcu_read_lock_fast_updown() apply.
*/
static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp)
{
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 48f0d803c8e2..f4da5fad70f5 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
// Queuing callbacks before initialization not yet supported.
if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))
rcu_segcblist_init(&rtpcp->cblist);
- needwake = (func == wakeme_after_rcu) ||
+ needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) ||
+ (func == wakeme_after_rcu) ||
(rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);
if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {
if (rtp->lazy_jiffies)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 55df6d37145e..afb9e7db8f78 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param
int ret = kstrtoul(val, 0, &j);
if (!ret) {
- WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
+ WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ));
adjust_jiffies_till_sched_qs();
}
return ret;
@@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node)
atomic_set_release(&sr_wn->inuse, 0);
}
-/* Enable rcu_normal_wake_from_gp automatically on small systems. */
-#define WAKE_FROM_GP_CPU_THRESHOLD 16
-
-static int rcu_normal_wake_from_gp = -1;
+static int rcu_normal_wake_from_gp = 1;
module_param(rcu_normal_wake_from_gp, int, 0644);
static struct workqueue_struct *sync_wq;
+#define RCU_SR_NORMAL_LATCH_THR 64
+
+/* Number of in-flight synchronize_rcu() calls queued on srs_next. */
+static atomic_long_t rcu_sr_normal_count;
+static int rcu_sr_normal_latched; /* 0/1 */
+
static void rcu_sr_normal_complete(struct llist_node *node)
{
struct rcu_synchronize *rs = container_of(
(struct rcu_head *) node, struct rcu_synchronize, head);
+ long nr;
WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&
!poll_state_synchronize_rcu_full(&rs->oldstate),
@@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node)
/* Finally. */
complete(&rs->completion);
+ nr = atomic_long_dec_return(&rcu_sr_normal_count);
+ WARN_ON_ONCE(nr < 0);
+
+ /*
+ * Unlatch: switch back to normal path when fully
+ * drained and if it has been latched.
+ */
+ if (nr == 0)
+ (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0);
}
static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
@@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void)
static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
{
+ /*
+ * Increment before publish to avoid a complete
+ * vs enqueue race on latch.
+ */
+ long nr = atomic_long_inc_return(&rcu_sr_normal_count);
+
+ /*
+ * Latch when threshold is reached. Checking for an exact match
+ * restricts cmpxchg() to a single context.
+ *
+ * This latch is intentionally relaxed and best-effort. Concurrent
+ * set/clear can race and temporarily lose the latch, which is OK
+ * because it only selects between the fast and fallback paths.
+ */
+ if (nr == RCU_SR_NORMAL_LATCH_THR)
+ (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1);
+
+ /* Publish for the GP kthread/worker. */
llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
}
@@ -2584,7 +2615,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
const long npj = NSEC_PER_SEC / HZ;
long rrn = READ_ONCE(rcu_resched_ns);
- rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
+ rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC);
tlimit = local_clock() + rrn;
jlimit = jiffies + (rrn + npj + 1) / npj;
jlimit_check = true;
@@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void)
{
struct rcu_synchronize rs;
+ init_rcu_head_on_stack(&rs.head);
trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));
- if (READ_ONCE(rcu_normal_wake_from_gp) < 1) {
+ if (READ_ONCE(rcu_normal_wake_from_gp) < 1 ||
+ READ_ONCE(rcu_sr_normal_latched)) {
wait_rcu_gp(call_rcu_hurry);
goto trace_complete_out;
}
- init_rcu_head_on_stack(&rs.head);
init_completion(&rs.completion);
/*
@@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void)
/* Now we can wait. */
wait_for_completion(&rs.completion);
- destroy_rcu_head_on_stack(&rs.head);
trace_complete_out:
trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));
+ destroy_rcu_head_on_stack(&rs.head);
}
/**
@@ -4904,12 +4936,6 @@ void __init rcu_init(void)
sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
WARN_ON(!sync_wq);
- /* Respect if explicitly disabled via a boot parameter. */
- if (rcu_normal_wake_from_gp < 0) {
- if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD)
- rcu_normal_wake_from_gp = 1;
- }
-
/* Fill in default value for rcutree.qovld boot parameter. */
/* -After- the rcu_node ->lock fields are initialized! */
if (qovld < 0)
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index 1047b30cd46b..373b877cf171 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)
* No-CBs GP kthreads come here to wait for additional callbacks to show up
* or for grace periods to end.
*/
-static void nocb_gp_wait(struct rcu_data *my_rdp)
+static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp)
{
bool bypass = false;
int __maybe_unused cpu = my_rdp->cpu;
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 0492d6afc9a1..cc5bbd70cb84 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -865,8 +865,6 @@ our %deprecated_apis = (
"DEFINE_IDR" => "DEFINE_XARRAY",
"idr_init" => "xa_init",
"idr_init_base" => "xa_init_flags",
- "rcu_read_lock_trace" => "rcu_read_lock_tasks_trace",
- "rcu_read_unlock_trace" => "rcu_read_unlock_tasks_trace",
);
#Create a search pattern for all these strings to speed up a loop below
@@ -7596,12 +7594,15 @@ sub process {
# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU).
our $rcu_trace_funcs = qr{(?x:
+ rcu_read_lock_tasks_trace |
rcu_read_lock_trace |
rcu_read_lock_trace_held |
rcu_read_unlock_trace |
+ rcu_read_unlock_tasks_trace |
call_rcu_tasks_trace |
synchronize_rcu_tasks_trace |
rcu_barrier_tasks_trace |
+ rcu_tasks_trace_expedite_current |
rcu_request_urgent_qs_task
)};
our $rcu_trace_paths = qr{(?x: