1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
// SPDX-License-Identifier: GPL-2.0
//! Sleepable read-copy update (SRCU) support.
//!
//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h)
use crate::{
bindings,
error::to_result,
prelude::*,
sync::LockClassKey,
types::{
NotThreadSafe,
Opaque, //
},
};
use pin_init::pin_data;
/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class.
#[doc(hidden)]
#[macro_export]
macro_rules! new_srcu {
($($name:literal)?) => {
$crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!())
};
}
pub use new_srcu;
/// Sleepable read-copy update primitive.
///
/// SRCU readers may sleep while holding the read-side guard.
///
/// The destructor waits for active readers and callbacks, so it may sleep.
/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return.
///
/// # Invariants
///
/// This represents a valid `struct srcu_struct` initialized by the C SRCU API
/// and it remains pinned and valid until the pinned destructor runs.
#[repr(transparent)]
#[pin_data(PinnedDrop)]
pub struct Srcu {
#[pin]
inner: Opaque<bindings::srcu_struct>,
}
impl Srcu {
/// Creates a new SRCU instance.
#[inline]
pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self, Error> {
try_pin_init!(Self {
// INVARIANT: On success, the C initializer creates a valid `srcu_struct` and
// it remains pinned until `PinnedDrop` runs.
inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| {
// SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`.
to_result(unsafe {
bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr())
})
}),
})
}
/// Enters an SRCU read-side critical section.
///
/// Leaking the returned [`Guard`] leaves the SRCU read-side critical
/// section active and makes `drop` sleep forever.
#[inline]
pub fn read_lock(&self) -> Guard<'_> {
// SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`.
let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) };
// INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`.
Guard {
srcu: self,
idx,
_not_send: NotThreadSafe,
}
}
/// Waits until all pre-existing SRCU readers have completed.
#[inline]
pub fn synchronize(&self) {
// SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`.
unsafe { bindings::synchronize_srcu(self.inner.get()) };
}
/// Waits until all pre-existing SRCU readers have completed, expedited.
///
/// This requests a lower-latency grace period than [`Srcu::synchronize`] typically
/// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default
/// and use this variant only when reducing reset or teardown latency is more important
/// than the extra cost.
#[inline]
pub fn synchronize_expedited(&self) {
// SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`.
unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) };
}
}
#[pinned_drop]
impl PinnedDrop for Srcu {
fn drop(self: Pin<&mut Self>) {
let ptr = self.inner.get();
if crate::warn_on!(
// SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`
// and `srcu_readers_active()` only checks the active reader count.
unsafe { bindings::srcu_readers_active(ptr) }
) {
// `cleanup_srcu_struct()` may return early if there are still active readers.
// This should only happen if a guard was leaked with `mem::forget`, which is
// "WRONG" code and may cause a UAF because Rust will free the `srcu_struct`
// while it is still referenced from the C side (e.g. by `call_srcu()` callbacks).
//
// Another consequence of leaking guards is that `call_srcu()` callbacks will
// never run because the grace period can never complete due to permanently
// active readers (i.e. leaked guards).
//
// If this ever happens, that means the guard was leaked by mistake and the
// caller must fix the bug. Sleeping here is intentional and less harmful
// than risking a UAF.
//
// SAFETY: By the type invariants, `self` contains a valid and pinned
// `struct srcu_struct`.
unsafe { bindings::synchronize_srcu(ptr) };
}
// Ensure all SRCU callbacks have been finished before freeing.
// SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`.
unsafe { bindings::srcu_barrier(ptr) };
// SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`.
unsafe { bindings::cleanup_srcu_struct(ptr) };
}
}
// SAFETY: `srcu_struct` may be shared and used across threads.
unsafe impl Send for Srcu {}
// SAFETY: `srcu_struct` may be shared and used concurrently.
unsafe impl Sync for Srcu {}
/// Guard for an active SRCU read-side critical section on a particular [`Srcu`].
///
/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side
/// critical section active and makes dropping the associated [`Srcu`] sleep forever.
///
/// # Invariants
///
/// `idx` is the index returned by `srcu_read_lock()` for `srcu`.
#[must_use = "if unused, the lock will be immediately unlocked"]
pub struct Guard<'a> {
srcu: &'a Srcu,
idx: i32,
_not_send: NotThreadSafe,
}
impl Guard<'_> {
/// Explicitly releases the SRCU read-side critical section.
#[inline]
pub fn unlock(self) {}
}
impl Drop for Guard<'_> {
#[inline]
fn drop(&mut self) {
// SAFETY: `Guard` is only constructible through `Srcu::read_lock()`,
// which returns a valid index for the SRCU instance.
unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) };
}
}
|
