diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/rcu/rcu.h | 7 | ||||
| -rw-r--r-- | kernel/rcu/rcutorture.c | 24 | ||||
| -rw-r--r-- | kernel/rcu/tree_plugin.h | 35 |
3 files changed, 66 insertions, 0 deletions
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..14faa11ef23c 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index ffeca1b7e170..1a033ae1f1eb 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -80,6 +80,7 @@ MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@ /* Must be power of two minus one. */ #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) +torture_param(bool, deboost_timeliness_check, 0, "Enable checks for immediate deboosting"); torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); @@ -426,6 +427,7 @@ struct rcu_torture_ops { void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len); void (*set_gpwrap_lag)(unsigned long lag); int (*get_gpwrap_count)(int cpu); + bool (*is_task_rcu_boosted)(void); long cbflood_max; int irq_capable; int can_boost; @@ -635,6 +637,7 @@ static struct rcu_torture_ops rcu_ops = { .format_gp_seqs = rcutorture_format_gp_seqs, .set_gpwrap_lag = rcu_set_gpwrap_lag, .get_gpwrap_count = rcu_get_gpwrap_count, + .is_task_rcu_boosted = rcu_is_task_rcu_boosted, .irq_capable = 1, .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), .extendables = RCUTORTURE_MAX_EXTEND, @@ -2588,7 +2591,9 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, */ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) { + static int firsttime = 1; int newstate; + unsigned int nsegs; struct rcu_torture_one_read_state rtors; WARN_ON_ONCE(!rcu_is_watching()); @@ -2600,6 +2605,25 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + // This splat will happen on systems built with CONFIG_IRQ_WORK=n + // and on systems where arch_irq_work_has_interrupt() returns false. + // It might also happen on systems using a short-duration clock + // interrupt instead of a self-IPI (powerpc, s390) or that use + // neither a self-IPI nor a short-duration clock interrupts + // (all architectures using the generic implementation + // of arch_irq_work_raise()). On such systems, RCU cannot + // guarantee to immediately deboost RCU readers when the outermost + // rcu_read_unlock() does not end the full segmented RCU read-side + // critical section. + if (cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi() && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + WARN_ON_ONCE(deboost_timeliness_check); + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..d950722eb8f9 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1320,6 +1320,41 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + bool ret; + struct rcu_node *rnp; + struct task_struct *t = current; + + preempt_disable(); // Stabilize ->rcu_blocked_node + rnp = t->rcu_blocked_node; + if (!rnp) + ret = false; + else + ret = (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t); + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) |