diff options
Diffstat (limited to 'rust/kernel')
35 files changed, 1351 insertions, 392 deletions
diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test new file mode 100644 index 000000000000..e6a5c7a795f0 --- /dev/null +++ b/rust/kernel/Kconfig.test @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig RUST_KUNIT_TESTS + bool "Rust KUnit tests" + depends on KUNIT && RUST + default KUNIT_ALL_TESTS + help + This menu collects all options for Rust KUnit tests. + See Documentation/rust/testing.rst for how to protect + unit tests with these options. + + Say Y here to enable Rust KUnit tests. + + If unsure, say N. + +if RUST_KUNIT_TESTS +config RUST_ALLOCATOR_KUNIT_TEST + bool "KUnit tests for Rust allocator API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust allocator API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KVEC_KUNIT_TEST + bool "KUnit tests for Rust KVec API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust KVec API. + These are only for development and testing, not for + regular kernel use cases. + + If unsure, say N. + +config RUST_BITMAP_KUNIT_TEST + bool "KUnit tests for Rust bitmap API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust bitmap API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_KUNIT_SELFTEST + bool "KUnit selftests for Rust" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit selftests. These are only + for development and testing, not for regular kernel + use cases. + + If unsure, say N. + +config RUST_STR_KUNIT_TEST + bool "KUnit tests for Rust strings API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust strings API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_ATOMICS_KUNIT_TEST + bool "KUnit tests for Rust atomics API" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust atomics API. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +config RUST_BITFIELD_KUNIT_TEST + bool "KUnit tests for the Rust `bitfield!` macro" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + help + This option enables KUnit tests for the Rust `bitfield!` macro. + These are only for development and testing, not for regular + kernel use cases. + + If unsure, say N. + +endif diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index e38720349dcf..21067bde6860 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -22,8 +22,12 @@ pub use self::kvec::Vec; #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; -use crate::error::{code::EINVAL, Result}; -use core::{alloc::Layout, ptr::NonNull}; +use crate::prelude::*; + +use core::{ + alloc::Layout, + ptr::NonNull, // +}; /// Flags to be used when allocating memory. /// diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 63bfb91b3671..cd4203f27aed 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -8,14 +8,25 @@ //! //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html> -use super::Flags; -use core::alloc::Layout; -use core::ptr; -use core::ptr::NonNull; - -use crate::alloc::{AllocError, Allocator, NumaNode}; -use crate::bindings; -use crate::page; +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + bindings, + page, // +}; + +use core::{ + alloc::Layout, + ptr::{ + self, + NonNull, // + }, // +}; const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN; @@ -163,8 +174,11 @@ impl Vmalloc { /// # Examples /// /// ``` - /// # use core::ptr::{NonNull, from_mut}; - /// # use kernel::{page, prelude::*}; + /// # use core::ptr::{ + /// # from_mut, + /// # NonNull, // + /// # }; + /// # use kernel::page; /// use kernel::alloc::allocator::Vmalloc; /// /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?; @@ -251,6 +265,7 @@ unsafe impl Allocator for KVmalloc { } } +#[cfg(CONFIG_RUST_ALLOCATOR_KUNIT_TEST)] #[macros::kunit_tests(rust_allocator)] mod tests { use super::*; diff --git a/rust/kernel/alloc/allocator/iter.rs b/rust/kernel/alloc/allocator/iter.rs index e0a70b7a744a..02fda3ea5cae 100644 --- a/rust/kernel/alloc/allocator/iter.rs +++ b/rust/kernel/alloc/allocator/iter.rs @@ -1,9 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 use super::Vmalloc; + use crate::page; -use core::marker::PhantomData; -use core::ptr::NonNull; + +use core::{ + marker::PhantomData, + ptr::NonNull, // +}; /// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation. /// diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index bd6da02c7ab8..80eb39364e86 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -3,24 +3,47 @@ //! Implementation of [`Box`]. #[allow(unused_imports)] // Used in doc comments. -use super::allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter}; -use super::{AllocError, Allocator, Flags, NumaNode}; -use core::alloc::Layout; -use core::borrow::{Borrow, BorrowMut}; -use core::marker::PhantomData; -use core::mem::ManuallyDrop; -use core::mem::MaybeUninit; -use core::ops::{Deref, DerefMut}; -use core::pin::Pin; -use core::ptr::NonNull; -use core::result::Result; - -use crate::ffi::c_void; -use crate::fmt; -use crate::init::InPlaceInit; -use crate::page::AsPageIter; -use crate::types::ForeignOwnable; -use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption}; +use super::allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // +}; + +use super::{ + AllocError, + Allocator, + Flags, + NumaNode, // +}; + +use crate::{ + fmt, + page::AsPageIter, + prelude::*, + types::ForeignOwnable, // +}; + +use core::{ + alloc::Layout, + borrow::{ + Borrow, + BorrowMut, // + }, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, + result::Result, // +}; + +use pin_init::ZeroableOption; /// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`. /// @@ -274,7 +297,10 @@ where /// # Examples /// /// ``` - /// use kernel::sync::{new_spinlock, SpinLock}; + /// use kernel::sync::{ + /// new_spinlock, + /// SpinLock, // + /// }; /// /// struct Inner { /// a: u32, @@ -411,6 +437,7 @@ where { type Initialized = Box<T, A>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -420,6 +447,7 @@ where Ok(unsafe { Box::assume_init(self) }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -567,7 +595,6 @@ where /// /// ``` /// # use core::borrow::Borrow; -/// # use kernel::alloc::KBox; /// struct Foo<B: Borrow<u32>>(B); /// /// // Owned instance. @@ -595,7 +622,6 @@ where /// /// ``` /// # use core::borrow::BorrowMut; -/// # use kernel::alloc::KBox; /// struct Foo<B: BorrowMut<u32>>(B); /// /// // Owned instance. @@ -660,9 +686,13 @@ where /// # Examples /// /// ``` -/// # use kernel::prelude::*; -/// use kernel::alloc::allocator::VmallocPageIter; -/// use kernel::page::{AsPageIter, PAGE_SIZE}; +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; /// /// let mut vbox = VBox::new((), GFP_KERNEL)?; /// diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 6438385e4322..f7af62835aa8 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -3,29 +3,52 @@ //! Implementation of [`Vec`]. use super::{ - allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter}, + allocator::{ + KVmalloc, + Kmalloc, + Vmalloc, + VmallocPageIter, // + }, layout::ArrayLayout, - AllocError, Allocator, Box, Flags, NumaNode, + AllocError, + Allocator, + Box, + Flags, + NumaNode, // }; + use crate::{ fmt, page::{ AsPageIter, PAGE_SIZE, // - }, + }, // }; + use core::{ - borrow::{Borrow, BorrowMut}, + borrow::{ + Borrow, + BorrowMut, // + }, marker::PhantomData, - mem::{ManuallyDrop, MaybeUninit}, - ops::Deref, - ops::DerefMut, - ops::Index, - ops::IndexMut, - ptr, - ptr::NonNull, - slice, - slice::SliceIndex, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Index, + IndexMut, // + }, + ptr::{ + self, + NonNull, // + }, + slice::{ + self, + SliceIndex, // + }, // }; mod errors; @@ -614,7 +637,7 @@ where /// /// v.reserve(10, GFP_KERNEL)?; /// let cap = v.capacity(); - /// assert!(cap >= 10); + /// assert!(cap >= v.len() + 10); /// /// v.reserve(10, GFP_KERNEL)?; /// let new_cap = v.capacity(); @@ -849,6 +872,24 @@ impl<T> Vec<T, KVmalloc> { impl<T: Clone, A: Allocator> Vec<T, A> { /// Extend the vector by `n` clones of `value`. + /// + /// # Examples + /// + /// ``` + /// let mut v = KVec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_with(3, 5, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5]); + /// + /// v.extend_with(2, 8, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// v.extend_with(0, 3, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]); + /// + /// # Ok::<(), Error>(()) + /// ``` pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> { if n == 0 { return Ok(()); @@ -866,7 +907,7 @@ impl<T: Clone, A: Allocator> Vec<T, A> { spare[n - 1].write(value); // SAFETY: - // - `self.len() + n < self.capacity()` due to the call to reserve above, + // - `self.len() + n <= self.capacity()` due to the call to reserve above, // - the loop and the line above initialized the next `n` elements. unsafe { self.inc_len(n) }; @@ -1146,9 +1187,13 @@ where /// # Examples /// /// ``` -/// # use kernel::prelude::*; -/// use kernel::alloc::allocator::VmallocPageIter; -/// use kernel::page::{AsPageIter, PAGE_SIZE}; +/// use kernel::{ +/// alloc::allocator::VmallocPageIter, +/// page::{ +/// AsPageIter, +/// PAGE_SIZE, // +/// }, // +/// }; /// /// let mut vec = VVec::<u8>::new(); /// @@ -1463,6 +1508,7 @@ impl<'vec, T> Drop for DrainAll<'vec, T> { } } +#[cfg(CONFIG_RUST_KVEC_KUNIT_TEST)] #[macros::kunit_tests(rust_kvec)] mod tests { use super::*; diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs index 985c5f2c3962..aaca6446516a 100644 --- a/rust/kernel/alloc/kvec/errors.rs +++ b/rust/kernel/alloc/kvec/errors.rs @@ -2,8 +2,10 @@ //! Errors for the [`Vec`] type. -use kernel::fmt; -use kernel::prelude::*; +use crate::{ + fmt, + prelude::*, // +}; /// Error type for [`Vec::push_within_capacity`]. pub struct PushError<T>(pub T); diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs index 9f8be72feb7a..62a459c66baf 100644 --- a/rust/kernel/alloc/layout.rs +++ b/rust/kernel/alloc/layout.rs @@ -4,7 +4,10 @@ //! //! Custom layout types extending or improving [`Layout`]. -use core::{alloc::Layout, marker::PhantomData}; +use core::{ + alloc::Layout, + marker::PhantomData, // +}; /// Error when constructing an [`ArrayLayout`]. pub struct LayoutError; @@ -47,7 +50,10 @@ impl<T> ArrayLayout<T> { /// # Examples /// /// ``` - /// # use kernel::alloc::layout::{ArrayLayout, LayoutError}; + /// # use kernel::alloc::layout::{ + /// # ArrayLayout, + /// # LayoutError, // + /// # }; /// let layout = ArrayLayout::<i32>::new(15)?; /// assert_eq!(layout.len(), 15); /// diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs new file mode 100644 index 000000000000..554a5a2ff0ab --- /dev/null +++ b/rust/kernel/bitfield.rs @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Support for defining bitfields as Rust structures. +//! +//! The [`bitfield!`](kernel::bitfield!) macro declares integer types that are split into distinct +//! bit fields of arbitrary length. Each field is typed using [`Bounded`](kernel::num::Bounded) to +//! ensure values are properly validated and to avoid implicit data loss. +//! +//! # Example +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! bitfield! { +//! pub struct Rgb(u16) { +//! 15:11 blue; +//! 10:5 green; +//! 4:0 red; +//! } +//! } +//! +//! // Valid value for the `blue` field. +//! let blue = Bounded::<u16, 5>::new::<0x18>(); +//! +//! // Setters can be chained. Values ranges are checked at compile-time. +//! let color = Rgb::zeroed() +//! // Compile-time bounds check of constant value. +//! .with_const_red::<0x10>() +//! .with_const_green::<0x1f>() +//! // A `Bounded` can also be passed. +//! .with_blue(blue); +//! +//! assert_eq!(color.red(), 0x10); +//! assert_eq!(color.green(), 0x1f); +//! assert_eq!(color.blue(), 0x18); +//! assert_eq!( +//! color.into_raw(), +//! (0x18 << Rgb::BLUE_SHIFT) + (0x1f << Rgb::GREEN_SHIFT) + 0x10, +//! ); +//! +//! // Convert to/from the backing storage type. +//! let raw: u16 = color.into(); +//! assert_eq!(Rgb::from(raw), color); +//! ``` +//! +//! # Syntax +//! +//! ```text +//! bitfield! { +//! #[attributes] +//! // Documentation for `Name`. +//! pub struct Name(storage_type) { +//! // `field_1` documentation. +//! hi:lo field_1; +//! // `field_2` documentation. +//! hi:lo field_2 => ConvertedType; +//! // `field_3` documentation. +//! hi:lo field_3 ?=> ConvertedType; +//! ... +//! } +//! } +//! ``` +//! +//! - `storage_type`: The underlying unsigned integer type ([`u8`], [`u16`], [`u32`], [`u64`]). +//! Signed integer storage types are not supported. +//! - `hi:lo`: Bit range (inclusive), where `hi >= lo`. +//! - `=> Type`: Optional infallible conversion (see [below](#infallible-conversion-)). +//! - `?=> Type`: Optional fallible conversion (see [below](#fallible-conversion-)). +//! - Documentation strings and attributes are optional. +//! +//! # Generated code +//! +//! Each field is internally represented as a [`Bounded`] parameterized by its bit width. Field +//! values can either be set/retrieved directly, or converted from/to another type. +//! +//! The use of [`Bounded`] for each field enforces bounds-checking (at build time or runtime) of +//! every value assigned to a field. This ensures that data is never accidentally truncated. +//! +//! The macro generates the bitfield type, [`From`] and [`Into`] implementations for its storage +//! type, as well as [`Debug`] and [`Zeroable`](pin_init::Zeroable) implementations. +//! +//! For each field, it also generates: +//! +//! - `field()`: Getter method for the field value. +//! - `with_field(value)`: Infallible setter; the argument type must fit within the field's width. +//! - `with_const_field::<VALUE>()`: `const` setter; the value is validated at compile time. +//! Usually shorter to use than `with_field` for constant values as it doesn't require +//! constructing a [`Bounded`]. +//! - `try_with_field(value)`: Fallible setter. Returns an error if the value is out of range. +//! - `FIELD_MASK`, `FIELD_SHIFT`, `FIELD_RANGE`: Constants for manual bit manipulation. +//! +//! # Reserved names for field identifiers +//! +//! Field identifiers are used to generate methods and associated constants on the bitfield type. +//! For a field named `field`, the macro may generate methods named `field`, `with_field`, +//! `with_const_field`, `try_with_field`, `__field` and `__with_field`, as well as constants named +//! `FIELD_MASK`, `FIELD_SHIFT` and `FIELD_RANGE`. +//! +//! Therefore, field identifiers must not use names that would collide with generated items for +//! any field in the same bitfield. The following prefixes are thus reserved for field identifiers: +//! +//! - `with_` +//! - `const_` +//! - `try_with_` +//! - `__` +//! +//! The field identifiers `from_raw`, `into_raw`, and `into` are also reserved. +//! +//! In addition, field identifiers should follow Rust `snake_case` conventions, since the associated +//! constants are generated by uppercasing the field name. +//! +//! # Implicit conversions +//! +//! Types that fit entirely within a field's bit width can be used directly with setters. For +//! example, [`bool`] works with single-bit fields, and [`u8`] works with 8-bit fields: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Flags(u32) { +//! 15:8 byte_field; +//! 0:0 flag; +//! } +//! } +//! +//! let flags = Flags::zeroed() +//! .with_byte_field(0x42_u8) +//! .with_flag(true); +//! +//! assert_eq!(flags.into_raw(), (0x42 << Flags::BYTE_FIELD_SHIFT) | 1); +//! ``` +//! +//! # Runtime bounds checking +//! +//! When a value is not known at compile time, use `try_with_field()` to check bounds at runtime: +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Config(u8) { +//! 3:0 nibble; +//! } +//! } +//! +//! fn set_nibble(config: Config, value: u8) -> Result<Config, Error> { +//! // Returns `EOVERFLOW` if `value > 0xf`. +//! config.try_with_nibble(value) +//! } +//! # Ok::<(), Error>(()) +//! ``` +//! +//! # Type conversion +//! +//! Fields can be automatically converted to/from a custom type using `=>` (infallible) or `?=>` +//! (fallible). The custom type must implement the appropriate [`From`] or [`TryFrom`] traits with +//! [`Bounded`]. +//! +//! ## Infallible conversion (`=>`) +//! +//! Use this when all possible bit patterns of a field map to valid values: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Power { +//! Off, +//! On, +//! } +//! +//! impl From<Bounded<u32, 1>> for Power { +//! fn from(v: Bounded<u32, 1>) -> Self { +//! match *v { +//! 0 => Power::Off, +//! _ => Power::On, +//! } +//! } +//! } +//! +//! impl From<Power> for Bounded<u32, 1> { +//! fn from(p: Power) -> Self { +//! (p as u32 != 0).into() +//! } +//! } +//! +//! bitfield! { +//! pub struct Control(u32) { +//! 0:0 power => Power; +//! } +//! } +//! +//! let ctrl = Control::zeroed().with_power(Power::On); +//! assert_eq!(ctrl.power(), Power::On); +//! ``` +//! +//! ## Fallible conversion (`?=>`) +//! +//! Use this when some bit patterns of a field are invalid. The getter returns a [`Result`]: +//! +//! ```rust +//! use kernel::bitfield; +//! use kernel::num::Bounded; +//! +//! #[derive(Debug, Clone, Copy, PartialEq)] +//! enum Mode { +//! Low = 0, +//! High = 1, +//! Auto = 2, +//! // 3 is invalid +//! } +//! +//! impl TryFrom<Bounded<u32, 2>> for Mode { +//! type Error = u32; +//! +//! fn try_from(v: Bounded<u32, 2>) -> Result<Self, u32> { +//! match *v { +//! 0 => Ok(Mode::Low), +//! 1 => Ok(Mode::High), +//! 2 => Ok(Mode::Auto), +//! n => Err(n), +//! } +//! } +//! } +//! +//! impl From<Mode> for Bounded<u32, 2> { +//! fn from(m: Mode) -> Self { +//! match m { +//! Mode::Low => Bounded::<u32, _>::new::<0>(), +//! Mode::High => Bounded::<u32, _>::new::<1>(), +//! Mode::Auto => Bounded::<u32, _>::new::<2>(), +//! } +//! } +//! } +//! +//! bitfield! { +//! pub struct Config(u32) { +//! 1:0 mode ?=> Mode; +//! } +//! } +//! +//! let cfg = Config::zeroed().with_mode(Mode::Auto); +//! assert_eq!(cfg.mode(), Ok(Mode::Auto)); +//! +//! // Invalid bit pattern returns an error. +//! assert_eq!(Config::from(0b11).mode(), Err(3)); +//! ``` +//! +//! # Bits outside of declared fields +//! +//! Bits of the storage type that are not part of any declared field are preserved by the setter +//! methods, and can only be modified through `from_raw` or the [`From`] implementation from the +//! storage type. +//! +//! ```rust +//! use kernel::bitfield; +//! +//! bitfield! { +//! pub struct Sparse(u8) { +//! 7:6 high; +//! // Bits 5:1 are not covered by any field. +//! 0:0 low; +//! } +//! } +//! +//! // Set the gap bits via `from_raw`, then mutate the declared fields. +//! let val = Sparse::from_raw(0b0010_1010) +//! .with_const_high::<0b11>() +//! .with_low(true); +//! +//! // Bits 5:1 are unchanged. +//! assert_eq!(val.into_raw(), 0b1110_1011); +//! ``` +//! +//! # Signed field values +//! +//! Bitfield storage types are unsigned. Since field getter methods return a [`Bounded`] of the +//! storage type, fields are also unsigned by default. +//! +//! If a field needs to encode a signed value, use a custom conversion type with `=>` or `?=>` to +//! perform the sign interpretation explicitly. +//! +//! [`Bounded`]: kernel::num::Bounded + +/// Defines a bitfield struct with bounds-checked accessors for individual bit ranges. +/// +/// See the [`mod@kernel::bitfield`] module for full documentation and examples. +#[macro_export] +macro_rules! bitfield { + // Entry point defining the bitfield struct, its implementations and its field accessors. + ( + $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } + ) => { + $crate::bitfield!(@core + #[allow(non_camel_case_types)] + $(#[$attr])* $vis $name $storage + ); + $crate::bitfield!(@fields $vis $name $storage { $($fields)* }); + }; + + // All rules below are helpers. + + // Defines the wrapper `$name` type and its conversions from/to the storage type. + (@core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => { + $(#[$attr])* + #[repr(transparent)] + #[derive(Clone, Copy, PartialEq, Eq)] + $vis struct $name { + inner: $storage, + } + + #[allow(dead_code)] + impl $name { + /// Creates a bitfield from a raw value. + #[inline(always)] + $vis const fn from_raw(value: $storage) -> Self { + Self{ inner: value } + } + + /// Turns this bitfield into its raw value. + /// + /// This is similar to the [`From`] implementation, but is shorter to invoke in + /// most cases. + #[inline(always)] + $vis const fn into_raw(self) -> $storage { + self.inner + } + } + + // SAFETY: `$storage` is `Zeroable` and `$name` is transparent. + unsafe impl ::pin_init::Zeroable for $name {} + + impl ::core::convert::From<$name> for $storage { + #[inline(always)] + fn from(val: $name) -> $storage { + val.into_raw() + } + } + + impl ::core::convert::From<$storage> for $name { + #[inline(always)] + fn from(val: $storage) -> $name { + Self::from_raw(val) + } + } + }; + + // Definitions requiring knowledge of individual fields: private and public field accessors, + // and `Debug` implementation. + (@fields $vis:vis $name:ident $storage:ty { + $($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident + $(?=> $try_into_type:ty)? + $(=> $into_type:ty)? + ; + )* + } + ) => { + #[allow(dead_code)] + impl $name { + $( + $crate::bitfield!(@private_field_accessors $vis $name $storage : $hi:$lo $field); + $crate::bitfield!( + @public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field + $(?=> $try_into_type)? + $(=> $into_type)? + ); + )* + } + + $crate::bitfield!(@debug $name { $($field;)* }); + }; + + // Private field accessors working with the exact `Bounded` type for the field. + ( + @private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + $vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive<u8> = $lo..=$hi; + $vis const [<$field:upper _MASK>]: $storage = + ((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1); + $vis const [<$field:upper _SHIFT>]: u32 = $lo; + ); + + ::kernel::macros::paste!( + #[inline(always)] + fn [<__ $field>](self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> { + // Left shift to align the field's MSB with the storage MSB. + const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1); + // Right shift to move the top-aligned field to bit 0 of the storage. + const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo; + + // Extract the field using two shifts. `Bounded::shr` produces the correctly-sized + // output type. + let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from( + self.inner << ALIGN_TOP + ); + val.shr::<ALIGN_BOTTOM, { $hi + 1 - $lo } >() + } + + #[inline(always)] + const fn [<__with_ $field>]( + mut self, + value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>, + ) -> Self + { + const MASK: $storage = <$name>::[<$field:upper _MASK>]; + const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>]; + + let value = value.get() << SHIFT; + self.inner = (self.inner & !MASK) | value; + + self + } + ); + }; + + // Public accessors for fields infallibly (`=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:literal:$lo:literal $field:ident => $into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> $into_type + { + self.[<__ $field>]().into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [<with_ $field>](self, value: $into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields fallibly (`?=>`) converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::core::result::Result< + $try_into_type, + <$try_into_type as ::core::convert::TryFrom< + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + >>::Error + > + { + self.[<__ $field>]().try_into() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [<with_ $field>](self, value: $try_into_type) -> Self + { + self.[<__with_ $field>](value.into()) + } + + ); + }; + + // Public accessors for fields not converted to a type. + ( + @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : + $hi:tt:$lo:tt $field:ident + ) => { + ::kernel::macros::paste!( + + $(#[doc = $doc])* + #[doc = "Returns the value of this field."] + #[inline(always)] + $vis fn $field(self) -> + ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> + { + self.[<__ $field>]() + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the compile-time constant `VALUE`."] + #[inline(always)] + $vis const fn [<with_const_ $field>]<const VALUE: $storage>(self) -> Self { + self.[<__with_ $field>]( + ::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::<VALUE>() + ) + } + + $(#[doc = $doc])* + #[doc = "Sets this field to the given `value`."] + #[inline(always)] + $vis fn [<with_ $field>]<T>( + self, + value: T, + ) -> Self + where T: ::core::convert::Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>, + { + self.[<__with_ $field>](value.into()) + } + + $(#[doc = $doc])* + #[doc = "Tries to set this field to `value`, returning an error if it is out of range."] + #[inline(always)] + $vis fn [<try_with_ $field>]<T>( + self, + value: T, + ) -> ::kernel::error::Result<Self> + where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>, + { + Ok( + self.[<__with_ $field>]( + value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)? + ) + ) + } + + ); + }; + + // `Debug` implementation. + (@debug $name:ident { $($field:ident;)* }) => { + impl ::kernel::fmt::Debug for $name { + fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { + f.debug_struct(stringify!($name)) + .field("<raw>", &::kernel::prelude::fmt!("{:#x}", self.inner)) + $( + .field(stringify!($field), &self.$field()) + )* + .finish() + } + } + }; +} + +#[cfg(CONFIG_RUST_BITFIELD_KUNIT_TEST)] +#[::kernel::macros::kunit_tests(rust_kernel_bitfield)] +mod tests { + use core::convert::TryFrom; + + use pin_init::Zeroable; + + use kernel::num::Bounded; + + // Enum types for testing `=>` and `?=>` conversions. + + #[derive(Debug, Clone, Copy, PartialEq)] + enum MemoryType { + Unmapped = 0, + Normal = 1, + Device = 2, + Reserved = 3, + } + + impl TryFrom<Bounded<u64, 4>> for MemoryType { + type Error = u64; + fn try_from(value: Bounded<u64, 4>) -> Result<Self, Self::Error> { + match value.get() { + 0 => Ok(MemoryType::Unmapped), + 1 => Ok(MemoryType::Normal), + 2 => Ok(MemoryType::Device), + 3 => Ok(MemoryType::Reserved), + _ => Err(value.get()), + } + } + } + + impl From<MemoryType> for Bounded<u64, 4> { + fn from(mt: MemoryType) -> Bounded<u64, 4> { + Bounded::from_expr(mt as u64) + } + } + + #[derive(Debug, Clone, Copy, PartialEq)] + enum Priority { + Low = 0, + Medium = 1, + High = 2, + Critical = 3, + } + + impl From<Bounded<u16, 2>> for Priority { + fn from(value: Bounded<u16, 2>) -> Self { + match value & 0x3 { + 0 => Priority::Low, + 1 => Priority::Medium, + 2 => Priority::High, + _ => Priority::Critical, + } + } + } + + impl From<Priority> for Bounded<u16, 2> { + fn from(p: Priority) -> Bounded<u16, 2> { + Bounded::from_expr(p as u16) + } + } + + bitfield! { + struct TestU64(u64) { + 63:63 field_63; + 61:52 field_61_52; + 51:16 field_51_16; + 15:12 field_15_12 ?=> MemoryType; + 11:9 field_11_9; + 1:1 field_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU16(u16) { + 15:8 field_15_8; + 7:4 field_7_4; // Partial overlap with `field_5_4`. + 5:4 field_5_4 => Priority; + 3:1 field_3_1; + 0:0 field_0; + } + } + + bitfield! { + struct TestU8(u8) { + 7:0 field_7_0; // Full byte overlap. + 7:4 field_7_4; + 3:2 field_3_2; + 1:1 field_1; + 0:0 field_0; + } + } + + // Single and multi-bit fields basic access. + #[test] + fn test_basic_access() { + // `TestU64`. + let mut val = TestU64::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + val = val.with_field_1(false); + assert!(!val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_11_9::<0x5>(); + assert_eq!(val.field_11_9(), 0x5); + assert_eq!(val.into_raw(), 0xA01); + + val = val.with_const_field_51_16::<0x123456>(); + assert_eq!(val.field_51_16(), 0x123456); + assert_eq!(val.into_raw(), 0x0012_3456_0A01); + + const MAX_FIELD_51_16: u64 = ::kernel::bits::genmask_u64(0..=35); + val = val.with_const_field_51_16::<{ MAX_FIELD_51_16 }>(); + assert_eq!(val.field_51_16(), MAX_FIELD_51_16); + + val = val.with_const_field_61_52::<0x3FF>(); + assert_eq!(val.field_61_52(), 0x3FF); + + val = val.with_field_63(true); + assert!(val.field_63().into_bool()); + + // `TestU16`. + let mut val = TestU16::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_const_field_3_1::<0x5>(); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.into_raw(), 0xB); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAB); + + val = val.with_const_field_15_8::<0x42>(); + assert_eq!(val.field_15_8(), 0x42); + assert_eq!(val.into_raw(), 0x42AB); + + // `TestU8`. + let mut val = TestU8::zeroed(); + assert_eq!(val.into_raw(), 0x0); + + val = val.with_field_0(true); + assert!(val.field_0().into_bool()); + assert_eq!(val.into_raw(), 0x1); + + val = val.with_field_1(true); + assert!(val.field_1().into_bool()); + assert_eq!(val.into_raw(), 0x3); + + val = val.with_const_field_3_2::<0x3>(); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.into_raw(), 0xF); + + val = val.with_const_field_7_4::<0xA>(); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.into_raw(), 0xAF); + } + + // `=>` infallible conversion. + #[test] + fn test_infallible_conversion() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::Low); + assert_eq!(val.field_5_4(), Priority::Low); + assert_eq!(val.into_raw() & 0x30, 0x00); + + val = val.with_field_5_4(Priority::Medium); + assert_eq!(val.field_5_4(), Priority::Medium); + assert_eq!(val.into_raw() & 0x30, 0x10); + + val = val.with_field_5_4(Priority::High); + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.into_raw() & 0x30, 0x20); + + val = val.with_field_5_4(Priority::Critical); + assert_eq!(val.field_5_4(), Priority::Critical); + assert_eq!(val.into_raw() & 0x30, 0x30); + } + + // `?=>` fallible conversion. + #[test] + fn test_fallible_conversion() { + let mut val = TestU64::zeroed(); + + val = val.with_field_15_12(MemoryType::Unmapped); + assert_eq!(val.field_15_12(), Ok(MemoryType::Unmapped)); + val = val.with_field_15_12(MemoryType::Normal); + assert_eq!(val.field_15_12(), Ok(MemoryType::Normal)); + val = val.with_field_15_12(MemoryType::Device); + assert_eq!(val.field_15_12(), Ok(MemoryType::Device)); + val = val.with_field_15_12(MemoryType::Reserved); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + + // `field_15_12` is 4 bits wide (0-15); `MemoryType` only covers 0-3, so 4-15 return `Err`. + let raw = (val.into_raw() & !::kernel::bits::genmask_u64(12..=15)) | (0x7 << 12); + assert_eq!(TestU64::from_raw(raw).field_15_12(), Err(0x7)); + } + + // Test that setting an overlapping field affects the overlapped one as expected. + #[test] + fn test_overlapping_fields() { + let mut val = TestU16::zeroed(); + + val = val.with_field_5_4(Priority::High); // High == 2 == 0b10. + assert_eq!(val.field_5_4(), Priority::High); + assert_eq!(val.field_7_4(), 0x2); // Bits 7:6 == 0, bits 5:4 == 0b10. + + val = val.with_const_field_7_4::<0xF>(); + assert_eq!(val.field_7_4(), 0xF); + assert_eq!(val.field_5_4(), Priority::Critical); // Bits 5:4 == 0b11. + + // `field_7_0` should encompass all other fields. + let mut val = TestU8::zeroed() + .with_field_0(true) + .with_field_1(true) + .with_const_field_3_2::<0x3>() + .with_const_field_7_4::<0xA>(); + assert_eq!(val.into_raw(), 0xAF); + + val = val.with_field_7_0(0x55); + assert_eq!(val.field_7_0(), 0x55); + assert!(val.field_0().into_bool()); + assert!(!val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x1); + assert_eq!(val.field_7_4(), 0x5); + } + + // Checks that bits not mapped to any field are left untouched. + #[test] + fn test_unallocated_bits() { + let gap_bits = (1u64 << 62) | 0x1FC; + + let set_all_fields = |val: TestU64| { + val.with_field_63(true) + .with_const_field_61_52::<0x155>() + .with_const_field_51_16::<0x123456>() + .with_field_15_12(MemoryType::Device) + .with_const_field_11_9::<0x5>() + .with_field_1(true) + .with_field_0(true) + }; + + // Gap bits to 0. + let val = set_all_fields(TestU64::from_raw(0)); + assert_eq!(val.into_raw() & gap_bits, 0); + + // Gap bits to 1. + let val = set_all_fields(TestU64::from_raw(gap_bits)); + assert_eq!(val.into_raw() & gap_bits, gap_bits); + } + + #[test] + fn test_try_with() { + let val = TestU64::zeroed().try_with_field_51_16(0x123456).unwrap(); + assert_eq!(val.field_51_16(), 0x123456); + + let err = TestU64::zeroed().try_with_field_51_16(u64::MAX); + assert_eq!(err, Err(::kernel::error::code::EOVERFLOW)); + + let val = TestU64::zeroed() + .try_with_field_51_16(0xABCDEF) + .and_then(|p| p.try_with_field_0(1)) + .unwrap(); + assert_eq!(val.field_51_16(), 0xABCDEF); + assert!(val.field_0().into_bool()); + } + + // `from_raw`/`into_raw` and `From`/`Into` round-trips. + #[test] + fn test_raw() { + let raw: u64 = 0xBFF0_0000_3123_3E03; + let val = TestU64::from_raw(raw); + assert_eq!(u64::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_11_9(), 0x7); + assert_eq!(val.field_51_16(), 0x3123); + assert_eq!(val.field_15_12(), Ok(MemoryType::Reserved)); + assert_eq!(val.field_61_52(), 0x3FF); + assert!(val.field_63().into_bool()); + + let raw: u16 = 0x42AB; + let val = TestU16::from_raw(raw); + assert_eq!(u16::from(val), raw); + assert!(val.field_0().into_bool()); + assert_eq!(val.field_3_1(), 0x5); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_15_8(), 0x42); + + let raw: u8 = 0xAF; + let val = TestU8::from_raw(raw); + assert_eq!(u8::from(val), raw); + assert!(val.field_0().into_bool()); + assert!(val.field_1().into_bool()); + assert_eq!(val.field_3_2(), 0x3); + assert_eq!(val.field_7_4(), 0xA); + assert_eq!(val.field_7_0(), 0xAF); + } +} diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index 83d7dea99137..b27e0ec80d64 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -499,9 +499,8 @@ impl Bitmap { } } -use macros::kunit_tests; - -#[kunit_tests(rust_kernel_bitmap)] +#[cfg(CONFIG_RUST_BITMAP_KUNIT_TEST)] +#[macros::kunit_tests(rust_kernel_bitmap)] mod tests { use super::*; use kernel::alloc::flags::GFP_KERNEL; diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs index 2ea2154ec30c..c3acb9b68a65 100644 --- a/rust/kernel/build_assert.rs +++ b/rust/kernel/build_assert.rs @@ -61,15 +61,16 @@ //! undefined symbols and linker errors, it is not developer friendly to debug, so it is recommended //! to avoid it and prefer other two assertions where possible. +#[doc(inline)] pub use crate::{ - build_assert, + build_assert_macro as build_assert, build_error, const_assert, static_assert, // }; #[doc(hidden)] -pub use build_error::build_error; +pub use build_error::build_error as build_error_fn; /// Static assert (i.e. compile-time assert). /// @@ -105,6 +106,7 @@ pub use build_error::build_error; /// static_assert!(f(40) == 42, "f(x) must add 2 to the given input."); /// ``` #[macro_export] +#[doc(hidden)] macro_rules! static_assert { ($condition:expr $(,$arg:literal)?) => { const _: () = ::core::assert!($condition $(,$arg)?); @@ -133,6 +135,7 @@ macro_rules! static_assert { /// } /// ``` #[macro_export] +#[doc(hidden)] macro_rules! const_assert { ($condition:expr $(,$arg:literal)?) => { const { ::core::assert!($condition $(,$arg)?) }; @@ -157,12 +160,13 @@ macro_rules! const_assert { /// // foo(usize::MAX); // Fails to compile. /// ``` #[macro_export] +#[doc(hidden)] macro_rules! build_error { () => {{ - $crate::build_assert::build_error("") + $crate::build_assert::build_error_fn("") }}; ($msg:expr) => {{ - $crate::build_assert::build_error($msg) + $crate::build_assert::build_error_fn($msg) }}; } @@ -200,15 +204,16 @@ macro_rules! build_error { /// const _: () = const_bar(2); /// ``` #[macro_export] -macro_rules! build_assert { +#[doc(hidden)] +macro_rules! build_assert_macro { ($cond:expr $(,)?) => {{ if !$cond { - $crate::build_assert::build_error(concat!("assertion failed: ", stringify!($cond))); + $crate::build_assert::build_error_fn(concat!("assertion failed: ", stringify!($cond))); } }}; ($cond:expr, $msg:expr) => {{ if !$cond { - $crate::build_assert::build_error($msg); + $crate::build_assert::build_error_fn($msg); } }}; } diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index d8d26870bea2..a20bd5006f38 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -1323,7 +1323,7 @@ impl<T: Driver> Registration<T> { // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; - PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).map_or(0, |f| f)) + PolicyCpu::from_cpu(cpu_id).map_or(0, |mut policy| T::get(&mut policy).unwrap_or(0)) } /// Driver's `update_limit` callback. diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 4995ee5dc689..642ccff465c8 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -1152,8 +1152,8 @@ unsafe impl Sync for CoherentHandle {} /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// /// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// let whole = kernel::dma_read!(alloc, [2]?); -/// let field = kernel::dma_read!(alloc, [1]?.field); +/// let whole = kernel::dma_read!(alloc, [try: 2]); +/// let field = kernel::dma_read!(alloc, [panic: 1].field); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] @@ -1189,8 +1189,8 @@ macro_rules! dma_read { /// unsafe impl kernel::transmute::AsBytes for MyStruct{}; /// /// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// kernel::dma_write!(alloc, [2]?.member, 0xf); -/// kernel::dma_write!(alloc, [1]?, MyStruct { member: 0xf }); +/// kernel::dma_write!(alloc, [try: 2].member, 0xf); +/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf }); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] @@ -1207,11 +1207,8 @@ macro_rules! dma_write { (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) }; - (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr]? $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$index]?] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$index:expr] $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$index]] [$($rest)*]) + (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*]) }; ($dma:expr, $($rest:tt)*) => { $crate::dma_write!(@parse [$dma] [] [$($rest)*]) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 05cf869ac090..a56ba6309594 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -25,10 +25,8 @@ pub mod code { #[doc = $doc] )* pub const $err: super::Error = - match super::Error::try_from_errno(-(crate::bindings::$err as i32)) { - Some(err) => err, - None => panic!("Invalid errno in `declare_err!`"), - }; + super::Error::try_from_errno(-(crate::bindings::$err as i32)) + .expect("Invalid errno in `declare_err!`"); }; } diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs index 1e8725eb44ed..73afbc51ba33 100644 --- a/rust/kernel/fmt.rs +++ b/rust/kernel/fmt.rs @@ -4,7 +4,14 @@ //! //! This module is intended to be used in place of `core::fmt` in kernel code. -pub use core::fmt::{Arguments, Debug, Error, Formatter, Result, Write}; +pub use core::fmt::{ + Arguments, + Debug, + Error, + Formatter, + Result, + Write, // +}; /// Internal adapter used to route and allow implementations of formatting traits for foreign types. /// @@ -27,7 +34,15 @@ macro_rules! impl_fmt_adapter_forward { }; } -use core::fmt::{Binary, LowerExp, LowerHex, Octal, Pointer, UpperExp, UpperHex}; +use core::fmt::{ + Binary, + LowerExp, + LowerHex, + Octal, + Pointer, + UpperExp, + UpperHex, // +}; impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp); /// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 7a0d4559d7b5..05a12e869a57 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -151,6 +151,7 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf> where Error: From<E>, @@ -168,6 +169,7 @@ pub trait InPlaceInit<T>: Sized { E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index abc49926abfe..f924c7c7c1db 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -108,9 +108,10 @@ use core::marker::PhantomData; -use crate::io::IoLoc; - -use kernel::build_assert; +use crate::{ + build_assert::build_assert, + io::IoLoc, // +}; /// Trait implemented by all registers. pub trait Register: Sized { @@ -872,7 +873,7 @@ macro_rules! register { @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } ) => { - ::kernel::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); @@ -895,7 +896,9 @@ macro_rules! register { @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident [ $idx:expr ] { $($fields:tt)* } ) => { - ::kernel::static_assert!($idx < <$alias as $crate::io::register::RegisterArray>::SIZE); + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!( @@ -912,7 +915,7 @@ macro_rules! register { [ $size:expr, stride = $stride:expr ] @ $base:ident + $offset:literal { $($fields:tt)* } ) => { - ::kernel::static_assert!(::core::mem::size_of::<$storage>() <= $stride); + $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); @@ -938,7 +941,9 @@ macro_rules! register { @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } ) => { - ::kernel::static_assert!($idx < <$alias as $crate::io::register::RegisterArray>::SIZE); + $crate::build_assert::static_assert!( + $idx < <$alias as $crate::io::register::RegisterArray>::SIZE + ); $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!( @@ -956,11 +961,10 @@ macro_rules! register { ( @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } ) => { - $crate::register!(@bitfield_core + $crate::bitfield!( #[allow(non_camel_case_types)] - $(#[$attr])* $vis $name $storage + $(#[$attr])* $vis struct $name($storage) { $($fields)* } ); - $crate::register!(@bitfield_fields $vis $name $storage { $($fields)* }); }; // Implementations shared by all registers types. @@ -1016,245 +1020,4 @@ macro_rules! register { impl $crate::io::register::RelativeRegisterArray for $name {} }; - - // Defines the wrapper `$name` type and its conversions from/to the storage type. - (@bitfield_core $(#[$attr:meta])* $vis:vis $name:ident $storage:ty) => { - $(#[$attr])* - #[repr(transparent)] - #[derive(Clone, Copy, PartialEq, Eq)] - $vis struct $name { - inner: $storage, - } - - #[allow(dead_code)] - impl $name { - /// Creates a bitfield from a raw value. - #[inline(always)] - $vis const fn from_raw(value: $storage) -> Self { - Self{ inner: value } - } - - /// Turns this bitfield into its raw value. - /// - /// This is similar to the [`From`] implementation, but is shorter to invoke in - /// most cases. - #[inline(always)] - $vis const fn into_raw(self) -> $storage { - self.inner - } - } - - // SAFETY: `$storage` is `Zeroable` and `$name` is transparent. - unsafe impl ::pin_init::Zeroable for $name {} - - impl ::core::convert::From<$name> for $storage { - #[inline(always)] - fn from(val: $name) -> $storage { - val.into_raw() - } - } - - impl ::core::convert::From<$storage> for $name { - #[inline(always)] - fn from(val: $storage) -> $name { - Self::from_raw(val) - } - } - }; - - // Definitions requiring knowledge of individual fields: private and public field accessors, - // and `Debug` implementation. - (@bitfield_fields $vis:vis $name:ident $storage:ty { - $($(#[doc = $doc:expr])* $hi:literal:$lo:literal $field:ident - $(?=> $try_into_type:ty)? - $(=> $into_type:ty)? - ; - )* - } - ) => { - #[allow(dead_code)] - impl $name { - $( - $crate::register!(@private_field_accessors $vis $name $storage : $hi:$lo $field); - $crate::register!( - @public_field_accessors $(#[doc = $doc])* $vis $name $storage : $hi:$lo $field - $(?=> $try_into_type)? - $(=> $into_type)? - ); - )* - } - - $crate::register!(@debug $name { $($field;)* }); - }; - - // Private field accessors working with the exact `Bounded` type for the field. - ( - @private_field_accessors $vis:vis $name:ident $storage:ty : $hi:tt:$lo:tt $field:ident - ) => { - ::kernel::macros::paste!( - $vis const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive<u8> = $lo..=$hi; - $vis const [<$field:upper _MASK>]: $storage = - ((((1 << $hi) - 1) << 1) + 1) - ((1 << $lo) - 1); - $vis const [<$field:upper _SHIFT>]: u32 = $lo; - ); - - ::kernel::macros::paste!( - fn [<__ $field>](self) -> - ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> { - // Left shift to align the field's MSB with the storage MSB. - const ALIGN_TOP: u32 = $storage::BITS - ($hi + 1); - // Right shift to move the top-aligned field to bit 0 of the storage. - const ALIGN_BOTTOM: u32 = ALIGN_TOP + $lo; - - // Extract the field using two shifts. `Bounded::shr` produces the correctly-sized - // output type. - let val = ::kernel::num::Bounded::<$storage, { $storage::BITS }>::from( - self.inner << ALIGN_TOP - ); - val.shr::<ALIGN_BOTTOM, { $hi + 1 - $lo } >() - } - - const fn [<__with_ $field>]( - mut self, - value: ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>, - ) -> Self - { - const MASK: $storage = <$name>::[<$field:upper _MASK>]; - const SHIFT: u32 = <$name>::[<$field:upper _SHIFT>]; - - let value = value.get() << SHIFT; - self.inner = (self.inner & !MASK) | value; - - self - } - ); - }; - - // Public accessors for fields infallibly (`=>`) converted to a type. - ( - @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : - $hi:literal:$lo:literal $field:ident => $into_type:ty - ) => { - ::kernel::macros::paste!( - - $(#[doc = $doc])* - #[doc = "Returns the value of this field."] - #[inline(always)] - $vis fn $field(self) -> $into_type - { - self.[<__ $field>]().into() - } - - $(#[doc = $doc])* - #[doc = "Sets this field to the given `value`."] - #[inline(always)] - $vis fn [<with_ $field>](self, value: $into_type) -> Self - { - self.[<__with_ $field>](value.into()) - } - - ); - }; - - // Public accessors for fields fallibly (`?=>`) converted to a type. - ( - @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : - $hi:tt:$lo:tt $field:ident ?=> $try_into_type:ty - ) => { - ::kernel::macros::paste!( - - $(#[doc = $doc])* - #[doc = "Returns the value of this field."] - #[inline(always)] - $vis fn $field(self) -> - Result< - $try_into_type, - <$try_into_type as ::core::convert::TryFrom< - ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> - >>::Error - > - { - self.[<__ $field>]().try_into() - } - - $(#[doc = $doc])* - #[doc = "Sets this field to the given `value`."] - #[inline(always)] - $vis fn [<with_ $field>](self, value: $try_into_type) -> Self - { - self.[<__with_ $field>](value.into()) - } - - ); - }; - - // Public accessors for fields not converted to a type. - ( - @public_field_accessors $(#[doc = $doc:expr])* $vis:vis $name:ident $storage:ty : - $hi:tt:$lo:tt $field:ident - ) => { - ::kernel::macros::paste!( - - $(#[doc = $doc])* - #[doc = "Returns the value of this field."] - #[inline(always)] - $vis fn $field(self) -> - ::kernel::num::Bounded<$storage, { $hi + 1 - $lo }> - { - self.[<__ $field>]() - } - - $(#[doc = $doc])* - #[doc = "Sets this field to the compile-time constant `VALUE`."] - #[inline(always)] - $vis const fn [<with_const_ $field>]<const VALUE: $storage>(self) -> Self { - self.[<__with_ $field>]( - ::kernel::num::Bounded::<$storage, { $hi + 1 - $lo }>::new::<VALUE>() - ) - } - - $(#[doc = $doc])* - #[doc = "Sets this field to the given `value`."] - #[inline(always)] - $vis fn [<with_ $field>]<T>( - self, - value: T, - ) -> Self - where T: Into<::kernel::num::Bounded<$storage, { $hi + 1 - $lo }>>, - { - self.[<__with_ $field>](value.into()) - } - - $(#[doc = $doc])* - #[doc = "Tries to set this field to `value`, returning an error if it is out of range."] - #[inline(always)] - $vis fn [<try_with_ $field>]<T>( - self, - value: T, - ) -> ::kernel::error::Result<Self> - where T: ::kernel::num::TryIntoBounded<$storage, { $hi + 1 - $lo }>, - { - Ok( - self.[<__with_ $field>]( - value.try_into_bounded().ok_or(::kernel::error::code::EOVERFLOW)? - ) - ) - } - - ); - }; - - // `Debug` implementation. - (@debug $name:ident { $($field:ident;)* }) => { - impl ::kernel::fmt::Debug for $name { - fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { - f.debug_struct(stringify!($name)) - .field("<raw>", &::kernel::prelude::fmt!("{:#x}", self.inner)) - $( - .field(stringify!($field), &self.$field()) - )* - .finish() - } - } - }; } diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index b7ac9faf141d..17b0c174cfc5 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -229,7 +229,7 @@ impl Flags { // Always inline to optimize out error path of `build_assert`. #[inline(always)] const fn new(value: u32) -> Self { - crate::build_assert!(value as u64 <= c_ulong::MAX as u64); + build_assert!(value as u64 <= c_ulong::MAX as u64); Flags(value as c_ulong) } } diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs index 2fc7662339e5..5bb5b48cf949 100644 --- a/rust/kernel/ioctl.rs +++ b/rust/kernel/ioctl.rs @@ -6,7 +6,7 @@ #![expect(non_snake_case)] -use crate::build_assert; +use crate::build_assert::build_assert; /// Build an ioctl number, analogous to the C macro of the same name. #[inline(always)] diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index a1edf7491579..cdee5f27bd7f 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -329,6 +329,7 @@ pub fn in_kunit_test() -> bool { !unsafe { bindings::kunit_get_current_test() }.is_null() } +#[cfg(CONFIG_RUST_KUNIT_SELFTEST)] #[kunit_tests(rust_kernel_kunit)] mod tests { use super::*; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b72b2fbe046d..9512af7156df 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -44,6 +44,7 @@ pub mod acpi; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; +pub mod bitfield; pub mod bitmap; pub mod bits; #[cfg(CONFIG_BLOCK)] diff --git a/rust/kernel/net/phy/reg.rs b/rust/kernel/net/phy/reg.rs index a7db0064cb7d..80e22c264ea8 100644 --- a/rust/kernel/net/phy/reg.rs +++ b/rust/kernel/net/phy/reg.rs @@ -9,9 +9,11 @@ //! defined in IEEE 802.3. use super::Device; -use crate::build_assert; -use crate::error::*; -use crate::uapi; +use crate::{ + build_assert::build_assert, + error::*, + uapi, // +}; mod private { /// Marker that a trait cannot be implemented outside of this crate diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index f9f90d6ec482..dafe77782d79 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -364,7 +364,7 @@ where // Always inline to optimize out error path of `build_assert`. #[inline(always)] pub fn from_expr(expr: T) -> Self { - crate::build_assert!( + crate::build_assert::build_assert!( fits_within(expr, N), "Requested value larger than maximal representable value." ); diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index adecb200c654..8affd8262891 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -3,17 +3,25 @@ //! Kernel page allocation and management. use crate::{ - alloc::{AllocError, Flags}, + alloc::{ + AllocError, + Flags, // + }, bindings, - error::code::*, - error::Result, - uaccess::UserSliceReader, + error::{ + code::*, + Result, // + }, + uaccess::UserSliceReader, // }; use core::{ marker::PhantomData, mem::ManuallyDrop, ops::Deref, - ptr::{self, NonNull}, + ptr::{ + self, + NonNull, // + }, // }; /// A bitwise shift for the page size. diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 44edf72a4a24..8a6da92e8da6 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -22,6 +22,7 @@ pub use core::{ pin::Pin, // }; +#[doc(no_inline)] pub use ::ffi::{ c_char, c_int, @@ -47,6 +48,7 @@ pub use macros::{ vtable, // }; +#[doc(no_inline)] pub use pin_init::{ init, pin_data, @@ -58,6 +60,13 @@ pub use pin_init::{ Zeroable, // }; +#[doc(no_inline)] +pub use zerocopy::FromBytes; + +#[doc(no_inline)] +pub use zerocopy_derive::FromBytes; + +#[doc(no_inline)] pub use super::{ alloc::{ flags::*, @@ -70,9 +79,12 @@ pub use super::{ VVec, Vec, // }, - build_assert, - build_error, - const_assert, + build_assert::{ + build_assert, + build_error, + const_assert, + static_assert, // + }, current, dev_alert, dev_crit, @@ -96,7 +108,6 @@ pub use super::{ pr_info, pr_notice, pr_warn, - static_assert, str::CStrExt as _, try_init, try_pin_init, diff --git a/rust/kernel/ptr/projection.rs b/rust/kernel/ptr/projection.rs index 140ea8e21617..af72d3b0e2a3 100644 --- a/rust/kernel/ptr/projection.rs +++ b/rust/kernel/ptr/projection.rs @@ -26,14 +26,14 @@ impl From<OutOfBound> for Error { /// /// # Safety /// -/// The implementation of `index` and `get` (if [`Some`] is returned) must ensure that, if provided -/// input pointer `slice` and returned pointer `output`, then: +/// For a given input pointer `slice` and return value `output`, the implementation of `index`, +/// `build_index` and `get` (if [`Some`] is returned) must ensure that: /// - `output` has the same provenance as `slice`; /// - `output.byte_offset_from(slice)` is between 0 to /// `KnownSize::size(slice) - KnownSize::size(output)`. /// -/// This means that if the input pointer is valid, then pointer returned by `get` or `index` is -/// also valid. +/// This means that if the input pointer is valid, then the pointer returned by `get`, `index` +/// or `build_index` is also valid. #[diagnostic::on_unimplemented(message = "`{Self}` cannot be used to index `{T}`")] #[doc(hidden)] pub unsafe trait ProjectIndex<T: ?Sized>: Sized { @@ -42,10 +42,16 @@ pub unsafe trait ProjectIndex<T: ?Sized>: Sized { /// Returns an index-projected pointer, if in bounds. fn get(self, slice: *mut T) -> Option<*mut Self::Output>; + /// Returns an index-projected pointer; panic if out of bounds. + fn index(self, slice: *mut T) -> *mut Self::Output; + /// Returns an index-projected pointer; fail the build if it cannot be proved to be in bounds. #[inline(always)] - fn index(self, slice: *mut T) -> *mut Self::Output { - Self::get(self, slice).unwrap_or_else(|| build_error!()) + fn build_index(self, slice: *mut T) -> *mut Self::Output { + match Self::get(self, slice) { + Some(v) => v, + None => build_error!(), + } } } @@ -67,6 +73,11 @@ where fn index(self, slice: *mut [T; N]) -> *mut Self::Output { <I as ProjectIndex<[T]>>::index(self, slice) } + + #[inline(always)] + fn build_index(self, slice: *mut [T; N]) -> *mut Self::Output { + <I as ProjectIndex<[T]>>::build_index(self, slice) + } } // SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to @@ -82,6 +93,16 @@ unsafe impl<T> ProjectIndex<[T]> for usize { Some(slice.cast::<T>().wrapping_add(self)) } } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut T { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + let () = zst_slice[self]; + slice.cast::<T>().wrapping_add(self) + } } // SAFETY: `get`-returned pointer has the same provenance as `slice` and the offset is checked to @@ -100,6 +121,18 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::Range<usize> { new_len, )) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + // Leverage Rust built-in operators for bounds checking. + // SAFETY: All non-null and aligned pointers are valid for ZST read. + let zst_slice = + unsafe { core::slice::from_raw_parts::<()>(core::ptr::dangling(), slice.len()) }; + _ = zst_slice[self.clone()]; + + // SAFETY: Bounds checked. + unsafe { self.get(slice).unwrap_unchecked() } + } } // SAFETY: Safety requirement guaranteed by the forwarded impl. @@ -110,6 +143,11 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeTo<usize> { fn get(self, slice: *mut [T]) -> Option<*mut [T]> { (0..self.end).get(slice) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (0..self.end).index(slice) + } } // SAFETY: Safety requirement guaranteed by the forwarded impl. @@ -120,6 +158,11 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFrom<usize> { fn get(self, slice: *mut [T]) -> Option<*mut [T]> { (self.start..slice.len()).get(slice) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + (self.start..slice.len()).index(slice) + } } // SAFETY: `get` returned the pointer as is, so it always has the same provenance and offset of 0. @@ -130,6 +173,11 @@ unsafe impl<T> ProjectIndex<[T]> for core::ops::RangeFull { fn get(self, slice: *mut [T]) -> Option<*mut [T]> { Some(slice) } + + #[inline(always)] + fn index(self, slice: *mut [T]) -> *mut [T] { + slice + } } /// A helper trait to perform field projection. @@ -207,10 +255,13 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// If a mutable pointer is needed, the macro input can be prefixed with the `mut` keyword, i.e. /// `kernel::ptr::project!(mut ptr, projection)`. By default, a const pointer is created. /// -/// `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. -/// `[index]` form performs build-time bounds checking; if compiler fails to prove `[index]` is in -/// bounds, compilation will fail. `[index]?` can be used to perform runtime bounds checking; -/// `OutOfBound` error is raised via `?` if the index is out of bounds. +/// The `ptr::project!` macro can perform both fallible indexing and build-time checked indexing. +/// The syntax is of the form `[<flavor>: index]` where `flavor` indicates the way of handling +/// index out-of-bounds errors. +/// - `try` will raise an [`OutOfBound`] error (which is convertible to [`ERANGE`]). +/// - `build` will use the [`build_assert!`] mechanism to have the compiler validate the index is +/// in bounds. +/// - `panic` will cause a Rust [`panic!`] if the index goes out of bounds. /// /// # Examples /// @@ -228,17 +279,21 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// } /// ``` /// -/// Index projections are performed with `[index]`: +/// Index projections are performed with `[<flavor>: index]`, where `flavor` is `try`, `build` or +/// `panic`: /// /// ``` /// fn proj(ptr: *const [u8; 32]) -> Result { -/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [1]); +/// let field_ptr: *const u8 = kernel::ptr::project!(ptr, [build: 1]); /// // The following invocation, if uncommented, would fail the build. /// // -/// // kernel::ptr::project!(ptr, [128]); +/// // kernel::ptr::project!(ptr, [build: 128]); /// /// // This will raise an `OutOfBound` error (which is convertible to `ERANGE`). -/// kernel::ptr::project!(ptr, [128]?); +/// kernel::ptr::project!(ptr, [try: 128]); +/// +/// // This will panic at runtime if executed. +/// kernel::ptr::project!(ptr, [panic: 128]); /// Ok(()) /// } /// ``` @@ -248,7 +303,7 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// ``` /// let ptr: *const [u8; 32] = core::ptr::dangling(); /// let field_ptr: Result<*const u8> = (|| -> Result<_> { -/// Ok(kernel::ptr::project!(ptr, [128]?)) +/// Ok(kernel::ptr::project!(ptr, [try: 128])) /// })(); /// assert!(field_ptr.is_err()); /// ``` @@ -257,7 +312,7 @@ unsafe impl<T: Deref> ProjectField<true> for T { /// /// ``` /// let ptr: *mut [(u8, u16); 32] = core::ptr::dangling_mut(); -/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [1].1); +/// let field_ptr: *mut u16 = kernel::ptr::project!(mut ptr, [build: 1].1); /// ``` #[macro_export] macro_rules! project_pointer { @@ -280,16 +335,22 @@ macro_rules! project_pointer { $crate::ptr::project!(@gen $ptr, $($rest)*) }; // Fallible index projection. - (@gen $ptr:ident, [$index:expr]? $($rest:tt)*) => { + (@gen $ptr:ident, [try: $index:expr] $($rest:tt)*) => { let $ptr = $crate::ptr::projection::ProjectIndex::get($index, $ptr) .ok_or($crate::ptr::projection::OutOfBound)?; $crate::ptr::project!(@gen $ptr, $($rest)*) }; - // Build-time checked index projection. - (@gen $ptr:ident, [$index:expr] $($rest:tt)*) => { + // Panicking index projection. + (@gen $ptr:ident, [panic: $index:expr] $($rest:tt)*) => { let $ptr = $crate::ptr::projection::ProjectIndex::index($index, $ptr); $crate::ptr::project!(@gen $ptr, $($rest)*) }; + // Build-time checked index projection. + (@gen $ptr:ident, [build: $index:expr] $($rest:tt)*) => { + let $ptr = $crate::ptr::projection::ProjectIndex::build_index($index, $ptr); + $crate::ptr::project!(@gen $ptr, $($rest)*) + }; + (mut $ptr:expr, $($proj:tt)*) => {{ let ptr: *mut _ = $ptr; $crate::ptr::project!(@gen ptr, $($proj)*); diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index 8311d91549e1..b3caa9a1c898 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -3,14 +3,27 @@ //! String representations. use crate::{ - alloc::{flags::*, AllocError, KVec}, - error::{to_result, Result}, - fmt::{self, Write}, - prelude::*, + alloc::{ + AllocError, + KVec, // + }, + error::{ + to_result, + Result, // + }, + fmt::{ + self, + Write, // + }, + prelude::*, // }; use core::{ marker::PhantomData, - ops::{Deref, DerefMut, Index}, + ops::{ + Deref, + DerefMut, + Index, // + }, // }; pub use crate::prelude::CStr; @@ -415,6 +428,7 @@ macro_rules! c_str { }}; } +#[cfg(CONFIG_RUST_STR_KUNIT_TEST)] #[kunit_tests(rust_kernel_str)] mod tests { use super::*; diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 18d6c0d62ce0..5ac4961b7cd2 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -712,6 +712,7 @@ impl<T> InPlaceInit<T> for UniqueArc<T> { impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { type Initialized = UniqueArc<T>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -721,6 +722,7 @@ impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { Ok(unsafe { self.assume_init() }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -758,6 +760,14 @@ impl<T> UniqueArc<T> { } } +impl<T: ?Sized> UniqueArc<T> { + /// Return a raw pointer to the data in this [`UniqueArc`]. + #[inline] + pub fn as_ptr(this: &Self) -> *const T { + Arc::as_ptr(&this.inner) + } +} + impl<T> UniqueArc<MaybeUninit<T>> { /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it. pub fn write(mut self, value: T) -> UniqueArc<T> { @@ -782,6 +792,7 @@ impl<T> UniqueArc<MaybeUninit<T>> { } /// Initialize `self` using the given initializer. + #[inline] pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { // SAFETY: The supplied pointer is valid for initialization. match unsafe { init.__init(self.as_mut_ptr()) } { @@ -792,6 +803,7 @@ impl<T> UniqueArc<MaybeUninit<T>> { } /// Pin-initialize `self` using the given pin-initializer. + #[inline] pub fn pin_init_with<E>( mut self, init: impl PinInit<T, E>, diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs index 9989f56d0605..b721b2e00b98 100644 --- a/rust/kernel/sync/aref.rs +++ b/rust/kernel/sync/aref.rs @@ -17,7 +17,12 @@ //! [`Arc`]: crate::sync::Arc //! [`Arc<T>`]: crate::sync::Arc -use core::{marker::PhantomData, mem::ManuallyDrop, ops::Deref, ptr::NonNull}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr::NonNull, // +}; /// Types that are _always_ reference counted. /// diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs index ad810c2172ec..9c8a7a203abd 100644 --- a/rust/kernel/sync/atomic/internal.rs +++ b/rust/kernel/sync/atomic/internal.rs @@ -4,8 +4,11 @@ //! //! Provides 1:1 mapping to the C atomic operations. -use crate::bindings; -use crate::macros::paste; +use crate::{ + bindings, + build_assert::static_assert, + macros::paste, // +}; use core::cell::UnsafeCell; use ffi::c_void; @@ -46,7 +49,7 @@ pub trait AtomicImpl: Sized + Copy + private::Sealed { // In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the // load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to // be added. -crate::static_assert!( +static_assert!( cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW), "The current implementation of atomic i8/i16/ptr relies on the architecure being \ ARCH_SUPPORTS_ATOMIC_RMW" diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 1d53834fcb12..3d63f40791fa 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -2,9 +2,7 @@ //! Pre-defined atomic types -use crate::static_assert; -use core::mem::{align_of, size_of}; -use ffi::c_void; +use crate::prelude::*; // Ensure size and alignment requirements are checked. static_assert!(size_of::<bool>() == size_of::<i8>()); @@ -154,9 +152,8 @@ unsafe impl super::AtomicAdd<usize> for usize { } } -use crate::macros::kunit_tests; - -#[kunit_tests(rust_atomics)] +#[cfg(CONFIG_RUST_ATOMICS_KUNIT_TEST)] +#[macros::kunit_tests(rust_atomics)] mod tests { use super::super::*; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index aecbdc34738f..ec2dd84316fc 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -85,6 +85,7 @@ impl<B: GlobalLockBackend> GlobalLock<B> { } /// Try to lock this global lock. + #[must_use = "if unused, the lock will be immediately unlocked"] #[inline] pub fn try_lock(&'static self) -> Option<GlobalGuard<B>> { Some(GlobalGuard { @@ -96,6 +97,7 @@ impl<B: GlobalLockBackend> GlobalLock<B> { /// A guard for a [`GlobalLock`]. /// /// See [`global_lock!`] for examples. +#[must_use = "the lock unlocks immediately when the guard is unused"] pub struct GlobalGuard<B: GlobalLockBackend> { inner: Guard<'static, B::Item, B::Backend>, } diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index 61f100a45b35..fb4a1430b3b4 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -3,7 +3,7 @@ //! A wrapper for data protected by a lock that does not wrap it. use super::{lock::Backend, lock::Lock}; -use crate::build_assert; +use crate::build_assert::build_assert; use core::{cell::UnsafeCell, mem::size_of, ptr}; /// Allows access to some data to be serialised by a lock that does not wrap it. diff --git a/rust/kernel/sync/refcount.rs b/rust/kernel/sync/refcount.rs index 6c7ae8b05a0b..23a5d201f343 100644 --- a/rust/kernel/sync/refcount.rs +++ b/rust/kernel/sync/refcount.rs @@ -4,9 +4,11 @@ //! //! C header: [`include/linux/refcount.h`](srctree/include/linux/refcount.h) -use crate::build_assert; -use crate::sync::atomic::Atomic; -use crate::types::Opaque; +use crate::{ + build_assert::build_assert, + sync::atomic::Atomic, + types::Opaque, // +}; /// Atomic reference counter. /// diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index 46e5f43223fe..987c9c0c2198 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -5,10 +5,16 @@ //! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) use crate::{ - alloc, bindings, build_assert, + alloc, + bindings, + build_assert::build_assert, error::{Error, Result}, ffi::c_void, - types::{ForeignOwnable, NotThreadSafe, Opaque}, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, // }; use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; |