diff options
Diffstat (limited to 'rust')
139 files changed, 7955 insertions, 2374 deletions
diff --git a/rust/Makefile b/rust/Makefile index a870d1616c71..da1a7409d984 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -3,6 +3,9 @@ # Where to place rustdoc generated documentation rustdoc_output := $(objtree)/Documentation/output/rust/rustdoc +# Clean generated host directory +clean-files := host/ + obj-$(CONFIG_RUST) += core.o compiler_builtins.o ffi.o always-$(CONFIG_RUST) += exports_core_generated.h @@ -33,7 +36,7 @@ endif obj-$(CONFIG_RUST) += exports.o -always-$(CONFIG_RUST) += libproc_macro2.rlib libquote.rlib libsyn.rlib +always-$(CONFIG_RUST) += host/libproc_macro2.rlib host/libquote.rlib host/libsyn.rlib always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated.rs always-$(CONFIG_RUST_KERNEL_DOCTESTS) += doctests_kernel_generated_kunit.c @@ -84,11 +87,15 @@ core-flags := \ --edition=$(core-edition) \ $(call cfgs-to-flags,$(core-cfgs)) +zerocopy-cfgs := \ + no_fp_fmt_parse + zerocopy-flags := \ - --cap-lints=allow + --cap-lints=allow \ + $(call cfgs-to-flags,$(zerocopy-cfgs)) zerocopy-envs := \ - CARGO_PKG_VERSION=0.8.50 + CARGO_PKG_VERSION=0.8.54 proc_macro2-cfgs := \ feature="proc-macro" \ @@ -128,11 +135,15 @@ syn-flags := \ --extern quote \ $(call cfgs-to-flags,$(syn-cfgs)) +zerocopy_derive-cfgs := \ + zerocopy_unstable_linux + zerocopy_derive-flags := \ --cap-lints=allow \ --extern proc_macro2 \ --extern quote \ - --extern syn + --extern syn \ + $(call cfgs-to-flags,$(zerocopy_derive-cfgs)) pin_init_internal-cfgs := \ kernel USE_RUSTC_FEATURES @@ -163,7 +174,7 @@ quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< $(rustc_target_envs) \ OBJTREE=$(abspath $(objtree)) \ $(RUSTDOC) $(filter-out $(skip_flags) --remap-path-scope=%,$(if $(rustdoc_host),$(rust_common_flags),$(rust_flags))) \ - $(rustc_target_flags) -L$(objtree)/$(obj) \ + $(rustc_target_flags) -L$(objtree)/$(obj)$(if $(rustdoc_host),/host) \ -Zunstable-options --generate-link-to-definition \ --output $(rustdoc_output) \ --crate-name $(subst rustdoc-,,$@) \ @@ -261,6 +272,7 @@ rustdoc-pin_init_internal: $(src)/pin-init/internal/src/lib.rs \ rustdoc-pin_init: private rustdoc_host = yes rustdoc-pin_init: private rustc_target_flags = $(pin_init-flags) \ + --extern pin_init_internal=$(objtree)/$(obj)/$(libpin_init_internal_name) \ --extern alloc --cfg feature=\"alloc\" rustdoc-pin_init: $(src)/pin-init/src/lib.rs rustdoc-pin_init_internal \ rustdoc-macros FORCE @@ -342,7 +354,8 @@ rusttestlib-pin_init: $(src)/pin-init/src/lib.rs rusttestlib-macros \ rusttestlib-kernel: private rustc_target_flags = --extern ffi \ --extern build_error --extern macros --extern pin_init \ --extern bindings --extern uapi \ - --extern zerocopy --extern zerocopy_derive + --extern zerocopy=$(objtree)/$(obj)/test/libzerocopy.rlib \ + --extern zerocopy_derive=$(objtree)/$(obj)/test/$(libzerocopy_derive_name) rusttestlib-kernel: $(src)/kernel/lib.rs rusttestlib-bindings rusttestlib-uapi \ rusttestlib-build_error rusttestlib-pin_init $(obj)/$(libmacros_name) \ $(obj)/bindings.o rusttestlib-zerocopy rusttestlib-zerocopy_derive FORCE @@ -444,7 +457,8 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ -fstrict-flex-arrays=% -fmin-function-alignment=% \ -fzero-init-padding-bits=% -mno-fdpic \ -fdiagnostics-show-context -fdiagnostics-show-context=% \ - --param=% --param asan-% -fno-isolate-erroneous-paths-dereference + --param=% --param asan-% -fno-isolate-erroneous-paths-dereference \ + -ffixed-r2 -mmultiple -mno-readonly-in-sdata # Derived from `scripts/Makefile.clang`. BINDGEN_TARGET_x86 := x86_64-linux-gnu @@ -454,6 +468,13 @@ BINDGEN_TARGET_loongarch := loongarch64-linux-gnusf BINDGEN_TARGET_s390 := s390x-linux-gnu # This is only for i386 UM builds, which need the 32-bit target not -m32 BINDGEN_TARGET_i386 := i386-linux-gnu + +ifdef CONFIG_PPC64 +BINDGEN_TARGET_powerpc := powerpc64le-linux-gnu +else +BINDGEN_TARGET_powerpc := powerpc-linux-gnu +endif + BINDGEN_TARGET_um := $(BINDGEN_TARGET_$(SUBARCH)) BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) @@ -572,23 +593,23 @@ quiet_cmd_rustc_procmacrolibrary = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_Q $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ $(filter-out $(skip_flags),$(rust_common_flags) $(rustc_target_flags)) \ --emit=dep-info=$(depfile) --emit=link=$@ --crate-type rlib -O \ - --out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \ + --out-dir $(objtree)/$(obj)/host -L$(objtree)/$(obj)/host \ --crate-name $(patsubst lib%.rlib,%,$(notdir $@)) $< -$(obj)/libproc_macro2.rlib: private skip_clippy = 1 -$(obj)/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) -$(obj)/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE +$(obj)/host/libproc_macro2.rlib: private skip_clippy = 1 +$(obj)/host/libproc_macro2.rlib: private rustc_target_flags = $(proc_macro2-flags) +$(obj)/host/libproc_macro2.rlib: $(src)/proc-macro2/lib.rs FORCE +$(call if_changed_dep,rustc_procmacrolibrary) -$(obj)/libquote.rlib: private skip_clippy = 1 -$(obj)/libquote.rlib: private skip_flags = $(quote-skip_flags) -$(obj)/libquote.rlib: private rustc_target_flags = $(quote-flags) -$(obj)/libquote.rlib: $(src)/quote/lib.rs $(obj)/libproc_macro2.rlib FORCE +$(obj)/host/libquote.rlib: private skip_clippy = 1 +$(obj)/host/libquote.rlib: private skip_flags = $(quote-skip_flags) +$(obj)/host/libquote.rlib: private rustc_target_flags = $(quote-flags) +$(obj)/host/libquote.rlib: $(src)/quote/lib.rs $(obj)/host/libproc_macro2.rlib FORCE +$(call if_changed_dep,rustc_procmacrolibrary) -$(obj)/libsyn.rlib: private skip_clippy = 1 -$(obj)/libsyn.rlib: private rustc_target_flags = $(syn-flags) -$(obj)/libsyn.rlib: $(src)/syn/lib.rs $(obj)/libquote.rlib FORCE +$(obj)/host/libsyn.rlib: private skip_clippy = 1 +$(obj)/host/libsyn.rlib: private rustc_target_flags = $(syn-flags) +$(obj)/host/libsyn.rlib: $(src)/syn/lib.rs $(obj)/host/libquote.rlib FORCE +$(call if_changed_dep,rustc_procmacrolibrary) quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) P $@ @@ -598,26 +619,26 @@ quiet_cmd_rustc_procmacro = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) -Clinker-flavor=gcc -Clinker=$(HOSTCC) \ -Clink-args='$(call escsq,$(KBUILD_PROCMACROLDFLAGS))' \ --emit=dep-info=$(depfile) --emit=link=$@ --extern proc_macro \ - --crate-type proc-macro -L$(objtree)/$(obj) \ + --crate-type proc-macro -L$(objtree)/$(obj)/host \ --crate-name $(patsubst lib%.$(procmacro-extension),%,$(notdir $@)) \ @$(objtree)/include/generated/rustc_cfg $< # Procedural macros can only be used with the `rustc` that compiled it. $(obj)/$(libzerocopy_derive_name): private skip_clippy = 1 $(obj)/$(libzerocopy_derive_name): private rustc_target_flags = $(zerocopy_derive-flags) -$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/libproc_macro2.rlib \ - $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE +$(obj)/$(libzerocopy_derive_name): $(src)/zerocopy-derive/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE +$(call if_changed_dep,rustc_procmacro) $(obj)/$(libmacros_name): private rustc_target_flags = \ --extern proc_macro2 --extern quote --extern syn -$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/libproc_macro2.rlib \ - $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE +$(obj)/$(libmacros_name): $(src)/macros/lib.rs $(obj)/host/libproc_macro2.rlib \ + $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE +$(call if_changed_dep,rustc_procmacro) $(obj)/$(libpin_init_internal_name): private rustc_target_flags = $(pin_init_internal-flags) $(obj)/$(libpin_init_internal_name): $(src)/pin-init/internal/src/lib.rs \ - $(obj)/libproc_macro2.rlib $(obj)/libquote.rlib $(obj)/libsyn.rlib FORCE + $(obj)/host/libproc_macro2.rlib $(obj)/host/libquote.rlib $(obj)/host/libsyn.rlib FORCE +$(call if_changed_dep,rustc_procmacro) # `rustc` requires `-Zunstable-options` to use custom target specifications @@ -636,7 +657,8 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L -Zunstable-options \ $(if $(link_helper),;$(LLVM_LINK) --internalize --suppress-warnings $(patsubst %.o,%.bc,$@) \ $(obj)/helpers/helpers$(if $(part-of-module),_module).bc -o $(patsubst %.o,%.m.bc,$@); \ - $(CC) $(CLANG_FLAGS) $(KBUILD_CFLAGS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ + $(CC) $(CLANG_FLAGS) $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS)) \ + $(CC_FLAGS_RUST_INLINE_HELPERS) -Wno-override-module -c $(patsubst %.o,%.m.bc,$@) -o $@ \ $(cmd_ld_single)) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) \ $(cmd_objtool) @@ -644,9 +666,11 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L rust-analyzer: $(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \ --cfgs='core=$(core-cfgs)' $(core-edition) \ + --cfgs='zerocopy=$(zerocopy-cfgs)' \ --cfgs='proc_macro2=$(proc_macro2-cfgs)' \ --cfgs='quote=$(quote-cfgs)' \ --cfgs='syn=$(syn-cfgs)' \ + --cfgs='zerocopy_derive=$(zerocopy_derive-cfgs)' \ --cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \ --cfgs='pin_init=$(pin_init-cfgs)' \ --envs='zerocopy=$(zerocopy-envs)' \ @@ -673,6 +697,10 @@ ifneq ($(or $(CONFIG_ARM64),$(and $(CONFIG_RISCV),$(CONFIG_64BIT))),) __ashrti3 \ __ashlti3 __lshrti3 endif +ifdef CONFIG_PPC32 + redirect-intrinsics += \ + __udivdi3 __umoddi3 +endif ifdef CONFIG_MODVERSIONS cmd_gendwarfksyms = $(if $(skip_gendwarfksyms),, \ diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 1124785e210b..4b31aa7f432f 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -60,6 +60,7 @@ #include <linux/fdtable.h> #include <linux/file.h> #include <linux/firmware.h> +#include <linux/fwctl.h> #include <linux/fs.h> #include <linux/i2c.h> #include <linux/interrupt.h> @@ -85,6 +86,7 @@ #include <linux/regulator/consumer.h> #include <linux/sched.h> #include <linux/security.h> +#include <linux/serdev.h> #include <linux/slab.h> #include <linux/sys_soc.h> #include <linux/task_work.h> @@ -93,6 +95,8 @@ #include <linux/wait.h> #include <linux/workqueue.h> #include <linux/xarray.h> +#include <net/genetlink.h> +#include <net/netlink.h> #include <trace/events/rust_sample.h> /* @@ -110,6 +114,7 @@ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; +const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 854e7c471434..812f8e5a08d5 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -27,6 +27,10 @@ #[allow(clippy::ref_as_ptr)] #[allow(clippy::undocumented_unsafe_blocks)] #[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#[cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] mod bindings_raw { use pin_init::{MaybeZeroable, Zeroable}; diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs index dd16c1dc899c..fc6b54636dd5 100644 --- a/rust/compiler_builtins.rs +++ b/rust/compiler_builtins.rs @@ -97,5 +97,11 @@ define_panicking_intrinsics!("`u64` division/modulo should not be used", { __aeabi_uldivmod, }); +#[cfg(target_arch = "powerpc")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __udivdi3, + __umoddi3, +}); + // NOTE: if you are adding a new intrinsic here, you should also add it to // `redirect-intrinsics` in `rust/Makefile`. diff --git a/rust/helpers/acpi.c b/rust/helpers/acpi.c deleted file mode 100644 index e75c9807bbad..000000000000 --- a/rust/helpers/acpi.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <linux/acpi.h> -#include <acpi/acpi_bus.h> - -__rust_helper bool rust_helper_acpi_of_match_device(const struct acpi_device *adev, - const struct of_device_id *of_match_table, - const struct of_device_id **of_id) -{ - return acpi_of_match_device(adev, of_match_table, of_id); -} - -__rust_helper struct acpi_device *rust_helper_to_acpi_device_node(struct fwnode_handle *fwnode) -{ - return to_acpi_device_node(fwnode); -} diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index fed8853745c8..dbc7a3017c78 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,6 +2,36 @@ #include <asm/barrier.h> +__rust_helper void rust_helper_mb(void) +{ + mb(); +} + +__rust_helper void rust_helper_rmb(void) +{ + rmb(); +} + +__rust_helper void rust_helper_wmb(void) +{ + wmb(); +} + +__rust_helper void rust_helper_dma_mb(void) +{ + dma_mb(); +} + +__rust_helper void rust_helper_dma_rmb(void) +{ + dma_rmb(); +} + +__rust_helper void rust_helper_dma_wmb(void) +{ + dma_wmb(); +} + __rust_helper void rust_helper_smp_mb(void) { smp_mb(); diff --git a/rust/helpers/fwctl.c b/rust/helpers/fwctl.c new file mode 100644 index 000000000000..c7eecd4336a7 --- /dev/null +++ b/rust/helpers/fwctl.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/fwctl.h> + +#if IS_ENABLED(CONFIG_RUST_FWCTL_ABSTRACTIONS) + +__rust_helper struct fwctl_device *rust_helper_fwctl_get(struct fwctl_device *fwctl) +{ + return fwctl_get(fwctl); +} + +__rust_helper void rust_helper_fwctl_put(struct fwctl_device *fwctl) +{ + fwctl_put(fwctl); +} + +#endif diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..440fb7638e3c 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -38,7 +38,6 @@ #define __rust_helper __always_inline #endif -#include "acpi.c" #include "atomic.c" #include "atomic_ext.c" #include "auxiliary.c" @@ -58,20 +57,25 @@ #include "cred.c" #include "device.c" #include "dma.c" +#ifdef CONFIG_DMA_SHARED_BUFFER #include "dma-resv.c" +#endif #include "drm.c" #include "drm_gpuvm.c" #include "err.c" -#include "irq.c" #include "fs.c" +#include "fwctl.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" +#include "irq.c" #include "jump_label.c" #include "kunit.c" #include "list.c" #include "maple_tree.c" #include "mm.c" #include "mutex.c" +#include "net/genetlink.c" #include "of.c" #include "page.c" #include "pci.c" @@ -87,10 +91,12 @@ #include "regulator.c" #include "scatterlist.c" #include "security.c" +#include "serdev.c" #include "signal.c" #include "slab.c" #include "spinlock.c" #include "string.c" +#include "srcu.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c new file mode 100644 index 000000000000..69595498620f --- /dev/null +++ b/rust/helpers/interrupt.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/spinlock.h> + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ + local_interrupt_disable(); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ + local_interrupt_enable(); +} diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 397810864a24..308950aae19c 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -3,6 +3,7 @@ #include <linux/io.h> #include <linux/ioport.h> +#ifdef CONFIG_HAS_IOMEM __rust_helper void __iomem *rust_helper_ioremap(phys_addr_t offset, size_t size) { return ioremap(offset, size); @@ -18,6 +19,20 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr) { iounmap(addr); } +#endif /* CONFIG_HAS_IOMEM */ + +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} __rust_helper u8 rust_helper_readb(const void __iomem *addr) { diff --git a/rust/helpers/net/genetlink.c b/rust/helpers/net/genetlink.c new file mode 100644 index 000000000000..3530b69f6cf7 --- /dev/null +++ b/rust/helpers/net/genetlink.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2026 Google LLC. + */ + +#include <net/genetlink.h> + +#ifdef CONFIG_NET + +__rust_helper struct sk_buff *rust_helper_genlmsg_new(size_t payload, gfp_t flags) +{ + return genlmsg_new(payload, flags); +} + +__rust_helper +int rust_helper_genlmsg_multicast(const struct genl_family *family, + struct sk_buff *skb, u32 portid, + unsigned int group, gfp_t flags) +{ + return genlmsg_multicast(family, skb, portid, group, flags); +} + +__rust_helper void rust_helper_genlmsg_cancel(struct sk_buff *skb, void *hdr) +{ + genlmsg_cancel(skb, hdr); +} + +__rust_helper void rust_helper_genlmsg_end(struct sk_buff *skb, void *hdr) +{ + genlmsg_end(skb, hdr); +} + +__rust_helper void rust_helper_nlmsg_free(struct sk_buff *skb) +{ + nlmsg_free(skb); +} + +__rust_helper +int rust_helper_genl_has_listeners(const struct genl_family *family, + struct net *net, unsigned int group) +{ + return genl_has_listeners(family, net, group); +} + +#endif diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index e44905317d75..a714cc2bfb7a 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -24,6 +24,19 @@ __rust_helper bool rust_helper_dev_is_pci(const struct device *dev) return dev_is_pci(dev); } +__rust_helper unsigned int rust_helper_pci_irq_type(struct pci_dev *pdev) +{ + return pci_irq_type(pdev); +} + +#ifndef CONFIG_PCI_IOV +__rust_helper unsigned int +rust_helper_pci_sriov_get_totalvfs(struct pci_dev *pdev) +{ + return pci_sriov_get_totalvfs(pdev); +} +#endif + #ifndef CONFIG_PCI_MSI __rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, diff --git a/rust/helpers/serdev.c b/rust/helpers/serdev.c new file mode 100644 index 000000000000..c52b78ca3fc7 --- /dev/null +++ b/rust/helpers/serdev.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/serdev.h> + +__rust_helper +void rust_helper_serdev_device_driver_unregister(struct serdev_device_driver *sdrv) +{ + serdev_device_driver_unregister(sdrv); +} + +__rust_helper +void rust_helper_serdev_device_put(struct serdev_device *serdev) +{ + serdev_device_put(serdev); +} + +__rust_helper +void rust_helper_serdev_device_set_client_ops(struct serdev_device *serdev, + const struct serdev_device_ops *ops) +{ + serdev_device_set_client_ops(serdev, ops); +} diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 4d13062cf253..d53400c15022 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) { lockdep_assert_held(lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ + spin_lock_irq_disable(lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ + spin_unlock_irq_enable(lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ + return spin_trylock_irq_disable(lock); +} diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000000..1a2f563640e0 --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/srcu.h> + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index 82d6aff73b04..4f474fe847c4 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) { lockdep_unregister_key(k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ + lockdep_assert_irqs_disabled(); +} diff --git a/rust/helpers/task.c b/rust/helpers/task.c index c0e1a06ede78..b46b1433a67e 100644 --- a/rust/helpers/task.c +++ b/rust/helpers/task.c @@ -28,11 +28,6 @@ __rust_helper kuid_t rust_helper_task_uid(struct task_struct *task) return task_uid(task); } -__rust_helper kuid_t rust_helper_task_euid(struct task_struct *task) -{ - return task_euid(task); -} - #ifndef CONFIG_USER_NS __rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid) { diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs index 9b8efa623130..ea2ce61ee393 100644 --- a/rust/kernel/acpi.rs +++ b/rust/kernel/acpi.rs @@ -25,10 +25,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } impl DeviceId { @@ -53,13 +49,7 @@ impl DeviceId { /// Create an ACPI `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! acpi_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::acpi::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("acpi", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("acpi", $crate::acpi::DeviceId, $($tt)*); }; } diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 35d1e015848d..c63d6acdbb6f 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -372,13 +372,13 @@ where // - `ptr` is a valid pointer to uninitialized memory. // - `ptr` is not used if an error is returned. // - `ptr` won't be moved until it is dropped, i.e. it is pinned. - unsafe { init(i).__pinned_init(ptr)? }; + unsafe { pin_init::raw_try_init(ptr, init(i))? }; // SAFETY: // - `i + 1 <= len`, hence we don't exceed the capacity, due to the call to // `with_capacity()` above. // - The new value at index buffer.len() + 1 is the only element being added here, and - // it has been initialized above by `init(i).__pinned_init(ptr)`. + // it has been initialized above by `raw_try_init(ptr, i)`. unsafe { buffer.inc_len(1) }; } @@ -463,7 +463,7 @@ where let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. - unsafe { init.__init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { Box::assume_init(self) }) } @@ -473,7 +473,7 @@ where let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { Box::assume_init(self) }.into()) } diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index f7af62835aa8..c7546b9da4fa 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -9,6 +9,7 @@ use super::{ Vmalloc, VmallocPageIter, // }, + flags::__GFP_ZERO, layout::ArrayLayout, AllocError, Allocator, @@ -51,6 +52,8 @@ use core::{ }, // }; +use pin_init::Zeroable; + mod errors; pub use self::errors::{InsertError, PushError, RemoveError}; @@ -532,6 +535,30 @@ where Ok(v) } + /// Creates a new [`Vec`] with `n` zero-initialized elements. + /// + /// # Examples + /// + /// ``` + /// let v = KVec::<u32>::zeroed(20, GFP_KERNEL)?; + /// + /// assert!(v.iter().all(|&x| x == 0)); + /// # Ok::<(), Error>(()) + /// ``` + pub fn zeroed(n: usize, flags: Flags) -> Result<Self, AllocError> + where + T: Zeroable, + { + let mut v = Self::with_capacity(n, flags | __GFP_ZERO)?; + + // SAFETY: + // - `n <= capacity - len`: `with_capacity(n)` guarantees capacity >= n, len is 0. + // - All elements in `[0, n)` are initialized: `__GFP_ZERO` zeroes the allocation, + // and `T: Zeroable` guarantees all-zeroes is a valid bit pattern. + unsafe { v.inc_len(n) }; + Ok(v) + } + /// Creates a `Vec<T, A>` from a pointer, a length and a capacity using the allocator `A`. /// /// # Examples diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index c42928d5a239..60dfbec8f330 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -20,6 +20,7 @@ use crate::{ }, prelude::*, types::{ + CovariantForLt, ForLt, ForeignOwnable, Opaque, // @@ -69,7 +70,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `adrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__auxiliary_driver_register(adrv.get(), module.0, name.as_char_ptr()) + bindings::__auxiliary_driver_register(adrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -93,7 +94,9 @@ impl<T: Driver> Adapter<T> { // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` // and does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; - let info = T::ID_TABLE.info(id.index()); + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. + let info = unsafe { id.info_unchecked::<T::IdInfo>() }; from_result(|| { let data = T::probe(adev, info); @@ -169,10 +172,6 @@ unsafe impl RawDeviceId for DeviceId { unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// IdTable type for auxiliary drivers. @@ -181,14 +180,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a auxiliary `IdTable` with its alias for modpost. #[macro_export] macro_rules! auxiliary_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::auxiliary::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("auxiliary", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("auxiliary", $crate::auxiliary::DeviceId, $($tt)*); }; } @@ -270,18 +263,15 @@ impl Device<device::Bound> { unsafe { parent.as_bound() } } - /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// Returns the stored registration data as a pinned reference. /// - /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The returned - /// reference has its lifetime shortened from `'static` to `&self`'s borrow lifetime via - /// [`ForLt::cast_ref`]. + /// Performs null and [`TypeId`] checks, then borrows the stored [`KBox`]. /// - /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling - /// [`Registration::new()`]. + /// # Safety /// - /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was - /// registered by a C driver. - pub fn registration_data<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> { + /// Callers must ensure that the lifetime shortening from the original `'static` storage to + /// `'_` is sound, e.g. via an HRTB closure or [`CovariantForLt`] guarantee. + unsafe fn registration_data_pinned<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> { // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. let ptr = unsafe { (*self.as_raw()).registration_data_rust }; if ptr.is_null() { @@ -300,17 +290,59 @@ impl Device<device::Bound> { return Err(EINVAL); } - // SAFETY: The `TypeId` check above confirms that the stored type matches - // `F::Of<'static>`; `ptr` remains valid until `Registration::drop()` calls - // `from_foreign()`. - let wrapper = unsafe { Pin::<KBox<RegistrationData<F::Of<'static>>>>::borrow(ptr) }; + // SAFETY: The `TypeId` check above confirms that the stored type matches `F`'s + // encoding; lifetimes are erased at runtime, so borrowing as `F::Of<'_>` is + // layout-compatible with the stored `F::Of<'static>`. `ptr` remains valid until + // `Registration::drop()` calls `from_foreign()`. + let wrapper = unsafe { Pin::<KBox<RegistrationData<F::Of<'_>>>>::borrow(ptr) }; // SAFETY: `data` is a structurally pinned field of `RegistrationData`. - let pinned: Pin<&F::Of<'_>> = unsafe { wrapper.map_unchecked(|w| &w.data) }; + Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + } + + /// Access the registration data set by the registering (parent) driver through a closure. + /// + /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned + /// reference to the registration data. + /// + /// For covariant types that implement [`trait@CovariantForLt`], prefer + /// [`registration_data`](Self::registration_data) which returns a direct reference. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data_with<F: ForLt + 'static, R>( + &self, + f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + ) -> Result<R> { + // SAFETY: The HRTB closure prevents the caller from smuggling in references with a + // concrete short lifetime, making the round-trip from `'static` sound regardless of + // variance. + let pinned = unsafe { self.registration_data_pinned::<F>()? }; + + Ok(f(pinned)) + } - // SAFETY: The data was pinned when stored; `cast_ref` only shortens - // the lifetime, so the pinning guarantee is preserved. - Ok(unsafe { Pin::new_unchecked(F::cast_ref(pinned.get_ref())) }) + /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// + /// This method is only available when `F` implements [`trait@CovariantForLt`], which guarantees + /// that the lifetime shortening is sound. + /// + /// For non-covariant types, use the closure-based [`Self::registration_data_with`]. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data<F: CovariantForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> { + // SAFETY: `CovariantForLt` guarantees covariance, which makes the lifetime shortening + // from `'static` to `'_` performed by `registration_data_pinned` sound. + unsafe { self.registration_data_pinned::<F>() } } } @@ -401,7 +433,9 @@ struct RegistrationData<T> { /// /// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration /// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt). -/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`]. +/// +/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`] and +/// [`Device::registration_data_with()`]. /// /// # Invariants /// diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index 35ede53f2b8e..a0d089423f21 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -581,6 +581,7 @@ mod tests { } impl From<MemoryType> for Bounded<u64, 4> { + #[inline(always)] fn from(mt: MemoryType) -> Bounded<u64, 4> { Bounded::from_expr(mt as u64) } @@ -606,6 +607,7 @@ mod tests { } impl From<Priority> for Bounded<u16, 2> { + #[inline(always)] fn from(p: Priority) -> Bounded<u16, 2> { Bounded::from_expr(p as u16) } diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index ed943960f851..3566f0234ca4 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -8,6 +8,7 @@ #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(CONFIG_DEBUG_BUGVERBOSE)] macro_rules! warn_flags { @@ -47,12 +48,17 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { const FLAGS: u32 = $crate::bindings::BUGFLAG_WARNING | $flags; + if false { + _ = $file; + } + // SAFETY: // - `flags` and `size` are all compile-time constants, preventing // any invalid memory access. @@ -73,14 +79,19 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, CONFIG_UML))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + } + // SAFETY: It is always safe to call `warn_slowpath_fmt()` // with a valid null-terminated string. unsafe { $crate::bindings::warn_slowpath_fmt( - $crate::c_str!(::core::file!()).as_char_ptr(), + $crate::str::CStrExt::as_char_ptr($crate::c_str!(::core::file!())), line!() as $crate::ffi::c_int, $flags as $crate::ffi::c_uint, ::core::ptr::null(), @@ -91,9 +102,15 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] macro_rules! warn_flags { ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + // SAFETY: It is always safe to call `WARN_ON()`. unsafe { $crate::bindings::WARN_ON(true) } }; @@ -101,9 +118,14 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] -#[cfg(not(CONFIG_BUG))] +#[cfg(any(testlib, not(CONFIG_BUG)))] macro_rules! warn_flags { - ($file:expr, $flags:expr) => {}; + ($file:expr, $flags:expr) => { + if false { + _ = $file; + _ = $flags; + } + }; } #[doc(hidden)] @@ -118,14 +140,14 @@ macro_rules! warn_on { let cond = $cond; #[cfg(CONFIG_DEBUG_BUGVERBOSE_DETAILED)] - const _COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); + const COND_STR: &str = concat!("[", stringify!($cond), "] ", file!()); #[cfg(not(CONFIG_DEBUG_BUGVERBOSE_DETAILED))] - const _COND_STR: &str = file!(); + const COND_STR: &str = file!(); if cond { const WARN_ON_FLAGS: u32 = $crate::bug::bugflag_taint($crate::bindings::TAINT_WARN); - $crate::warn_flags!(_COND_STR, WARN_ON_FLAGS); + $crate::warn_flags!(COND_STR, WARN_ON_FLAGS); } cond }}; diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs index 2339c6467325..cd082b83e9e7 100644 --- a/rust/kernel/configfs.rs +++ b/rust/kernel/configfs.rs @@ -875,13 +875,14 @@ impl<Container, Data> ItemType<Container, Data> { /// configfs::Subsystem<Configuration>, /// Configuration /// >::new_with_child_ctor::<N,Child>( -/// &THIS_MODULE, +/// ::kernel::module::this_module::<crate::LocalModule>(), /// &CONFIGURATION_ATTRS /// ); /// /// &CONFIGURATION_TPE /// } /// ``` +#[allow(clippy::crate_in_macro_def)] #[macro_export] macro_rules! configfs_attrs { ( @@ -1021,7 +1022,8 @@ macro_rules! configfs_attrs { static [< $data:upper _TPE >] : $crate::configfs::ItemType<$container, $data> = $crate::configfs::ItemType::<$container, $data>::new::<N>( - &THIS_MODULE, &[<$ data:upper _ATTRS >] + $crate::module::this_module::<crate::LocalModule>(), + &[<$ data:upper _ATTRS >] ); )? @@ -1030,7 +1032,8 @@ macro_rules! configfs_attrs { $crate::configfs::ItemType<$container, $data> = $crate::configfs::ItemType::<$container, $data>:: new_with_child_ctor::<N, $child>( - &THIS_MODULE, &[<$ data:upper _ATTRS >] + $crate::module::this_module::<crate::LocalModule>(), + &[<$ data:upper _ATTRS >] ); )? diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index 58ac04c650a1..affa2b9490ef 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -361,23 +361,28 @@ impl TableBuilder { } } - /// Adds a new entry to the table. - pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { // Adds the new entry at the end of the vector. Ok(self.entries.push( bindings::cpufreq_frequency_table { flags, driver_data, - frequency: freq.as_khz() as u32, + frequency, }, GFP_KERNEL, )?) } + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. pub fn to_table(mut self) -> Result<TableBox> { // Add last entry to the table. - self.add(Hertz(c_ulong::MAX), 0, 0)?; + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; TableBox::new(self.entries) } @@ -792,7 +797,13 @@ pub trait Driver { } /// Driver's `adjust_perf` callback. - fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) { + fn adjust_perf( + _policy: &mut Policy, + _min_perf: usize, + _target_perf: usize, + _max_perf: usize, + _capacity: usize, + ) { build_error!(VTABLE_DEFAULT_ERROR) } @@ -817,7 +828,9 @@ pub trait Driver { } /// Driver's `bios_limit` callback. - fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result { + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result<u32> { build_error!(VTABLE_DEFAULT_ERROR) } @@ -1263,12 +1276,13 @@ impl<T: Driver> Registration<T> { ptr: *mut bindings::cpufreq_policy, min_perf: c_ulong, target_perf: c_ulong, + max_perf: c_ulong, capacity: c_ulong, ) { // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the // lifetime of `policy`. let policy = unsafe { Policy::from_raw_mut(ptr) }; - T::adjust_perf(policy, min_perf, target_perf, capacity); + T::adjust_perf(policy, min_perf, target_perf, max_perf, capacity); } /// Driver's `get_intermediate` callback. @@ -1352,9 +1366,12 @@ impl<T: Driver> Registration<T> { from_result(|| { let mut policy = PolicyCpu::from_cpu(cpu_id)?; - + let val = T::bios_limit(&mut policy)?; // SAFETY: `limit` is guaranteed by the C code to be valid. - T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) + unsafe { + *limit = val; + } + Ok(0) }) } diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs index 8c39524b6a99..b295f8420abd 100644 --- a/rust/kernel/debugfs/traits.rs +++ b/rust/kernel/debugfs/traits.rs @@ -18,10 +18,6 @@ use crate::{ Arc, Mutex, // }, - transmute::{ - AsBytes, - FromBytes, // - }, uaccess::{ UserSliceReader, UserSliceWriter, // @@ -36,6 +32,8 @@ use core::{ str::FromStr, }; +use zerocopy::Immutable; + /// A trait for types that can be written into a string. /// /// This works very similarly to `Debug`, and is automatically implemented if `Debug` is @@ -76,8 +74,8 @@ pub trait BinaryWriter { ) -> Result<usize>; } -// Base implementation for any `T: AsBytes`. -impl<T: AsBytes> BinaryWriter for T { +// Base implementation for any `T: Immutable + IntoBytes`. +impl<T: Immutable + IntoBytes> BinaryWriter for T { fn write_to_slice( &self, writer: &mut UserSliceWriter, @@ -147,7 +145,7 @@ where // Delegate for `Vec<T, A>`. impl<T, A> BinaryWriter for Vec<T, A> where - T: AsBytes, + T: Immutable + IntoBytes, A: Allocator, { fn write_to_slice( @@ -155,14 +153,7 @@ where writer: &mut UserSliceWriter, offset: &mut file::Offset, ) -> Result<usize> { - let slice = self.as_slice(); - - // SAFETY: `T: AsBytes` allows us to treat `&[T]` as `&[u8]`. - let buffer = unsafe { - core::slice::from_raw_parts(slice.as_ptr().cast(), core::mem::size_of_val(slice)) - }; - - writer.write_slice_file(buffer, offset) + writer.write_slice_file(self.as_bytes(), offset) } } @@ -230,14 +221,14 @@ pub trait BinaryReaderMut { ) -> Result<usize>; } -// Base implementation for any `T: AsBytes + FromBytes`. -impl<T: AsBytes + FromBytes> BinaryReaderMut for T { +// Base implementation for any `T: FromBytes + IntoBytes`. +impl<T: FromBytes + IntoBytes> BinaryReaderMut for T { fn read_from_slice_mut( &mut self, reader: &mut UserSliceReader, offset: &mut file::Offset, ) -> Result<usize> { - reader.read_slice_file(self.as_bytes_mut(), offset) + reader.read_slice_file(self.as_mut_bytes(), offset) } } @@ -255,7 +246,7 @@ impl<T: ?Sized + BinaryReaderMut, A: Allocator> BinaryReaderMut for Box<T, A> { // Delegate for `Vec<T, A>`: Support a `Vec<T, A>` with an outer lock. impl<T, A> BinaryReaderMut for Vec<T, A> where - T: AsBytes + FromBytes, + T: FromBytes + IntoBytes, A: Allocator, { fn read_from_slice_mut( @@ -263,17 +254,7 @@ where reader: &mut UserSliceReader, offset: &mut file::Offset, ) -> Result<usize> { - let slice = self.as_mut_slice(); - - // SAFETY: `T: AsBytes + FromBytes` allows us to treat `&mut [T]` as `&mut [u8]`. - let buffer = unsafe { - core::slice::from_raw_parts_mut( - slice.as_mut_ptr().cast(), - core::mem::size_of_val(slice), - ) - }; - - reader.read_slice_file(buffer, offset) + reader.read_slice_file(self.as_mut_bytes(), offset) } } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 645afc49a27d..2291d85b6849 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -54,7 +54,8 @@ pub mod property; /// reference is valid in. For instance, the [`Bound`] context guarantees that the [`Device`] is /// bound to a driver for the entire duration of the existence of a [`Device<Bound>`] reference. /// -/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`] and [`CoreInternal`]. +/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`], [`CoreInternal`] and +/// [`BoundInternal`]. /// /// Unless selected otherwise [`Device`] defaults to the [`Normal`] [`DeviceContext`], which by /// itself has no additional requirements. @@ -235,7 +236,9 @@ impl<'a> Device<CoreInternal<'a>> { // in `into_foreign()`. Some(unsafe { Pin::<KBox<T>>::from_foreign(ptr.cast()) }) } +} +impl<Ctx: InternalBoundContext> Device<Ctx> { /// Borrow the driver's private data bound to this [`Device`]. /// /// # Safety @@ -245,22 +248,6 @@ impl<'a> Device<CoreInternal<'a>> { /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. pub unsafe fn drvdata_borrow<T>(&self) -> Pin<&T> { - // SAFETY: `drvdata_unchecked()` has the exact same safety requirements as the ones - // required by this method. - unsafe { self.drvdata_unchecked() } - } -} - -impl Device<Bound> { - /// Borrow the driver's private data bound to this [`Device`]. - /// - /// # Safety - /// - /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before - /// the device is fully unbound. - /// - The type `T` must match the type of the `ForeignOwnable` previously stored by - /// [`Device::set_drvdata`]. - unsafe fn drvdata_unchecked<T>(&self) -> Pin<&T> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; @@ -476,7 +463,8 @@ unsafe impl Sync for Device<Bound> {} /// [`DeviceContext`] is a marker trait for types representing the context of a bus specific /// [`Device`]. /// -/// The specific device context types are: [`CoreInternal`], [`Core`], [`Bound`] and [`Normal`]. +/// The specific device context types are: [`CoreInternal`], [`Core`], [`BoundInternal`], [`Bound`] +/// and [`Normal`]. /// /// [`DeviceContext`] types are hierarchical, which means that there is a strict hierarchy that /// defines which [`DeviceContext`] type can be derived from another. For instance, any @@ -485,6 +473,11 @@ unsafe impl Sync for Device<Bound> {} /// The following enumeration illustrates the dereference hierarchy of [`DeviceContext`] types. /// /// - [`CoreInternal`] => [`Core`] => [`Bound`] => [`Normal`] +/// - [`BoundInternal`] => [`Bound`] => [`Normal`] +/// +/// Both [`CoreInternal`] and [`BoundInternal`] implement the [`InternalBoundContext`] trait, +/// which provides access to internal bus abstraction methods on [`Device`] that are not available +/// to drivers. /// /// Bus devices can automatically implement the dereference hierarchy by using /// [`impl_device_context_deref`]. @@ -511,7 +504,11 @@ pub struct Normal; /// callback it appears in. It is intended to be used for synchronization purposes. Bus device /// implementations can implement methods for [`Device<Core>`], such that they can only be called /// from bus callbacks. -pub struct Core<'a>(PhantomData<&'a ()>); +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device<Core<'_>>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData<fn(&'a ()) -> &'a ()>); /// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus /// abstraction. @@ -522,7 +519,21 @@ pub struct Core<'a>(PhantomData<&'a ()>); /// /// This context mainly exists to share generic [`Device`] infrastructure that should only be called /// from bus callbacks with bus abstractions, but without making them accessible for drivers. -pub struct CoreInternal<'a>(PhantomData<&'a ()>); +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData<fn(&'a ()) -> &'a ()>); + +/// Semantically the same as [`Bound`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal bound context is intended to be used in exactly the same way as the [`Bound`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context exists for cases where the bus abstraction needs access to internal device +/// infrastructure (such as [`Device::drvdata_borrow`]), where [`CoreInternal`] would not be +/// justified. +pub struct BoundInternal; /// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to /// be bound to a driver. @@ -546,16 +557,28 @@ mod private { pub trait Sealed {} impl Sealed for super::Bound {} + impl Sealed for super::BoundInternal {} impl<'a> Sealed for super::Core<'a> {} impl<'a> Sealed for super::CoreInternal<'a> {} impl Sealed for super::Normal {} } impl DeviceContext for Bound {} +impl DeviceContext for BoundInternal {} impl<'a> DeviceContext for Core<'a> {} impl<'a> DeviceContext for CoreInternal<'a> {} impl DeviceContext for Normal {} +/// Marker trait for [`DeviceContext`] types that have internal bound-level access. +/// +/// This trait is implemented by [`CoreInternal`] and [`BoundInternal`], allowing methods that +/// require internal bus abstraction access to a bound device to be generic over both contexts. +/// +/// Methods bounded by this trait are available to bus abstractions but not to drivers. +pub trait InternalBoundContext: DeviceContext {} +impl<'a> InternalBoundContext for CoreInternal<'a> {} +impl InternalBoundContext for BoundInternal {} + impl<Ctx: DeviceContext> AsRef<Device<Ctx>> for Device<Ctx> { #[inline] fn as_ref(&self) -> &Device<Ctx> { @@ -665,6 +688,13 @@ macro_rules! impl_device_context_deref { // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { $device, + $crate::device::BoundInternal => $crate::device::Bound + }); + + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, $crate::device::Bound => $crate::device::Normal }); }; @@ -700,6 +730,7 @@ macro_rules! impl_device_context_into_aref { ::kernel::__impl_device_context_into_aref!( <'a> $crate::device::Core<'a>, $device ); + ::kernel::__impl_device_context_into_aref!($crate::device::BoundInternal, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; } @@ -708,9 +739,7 @@ macro_rules! impl_device_context_into_aref { #[macro_export] macro_rules! dev_printk { ($method:ident, $dev:expr, $($f:tt)*) => { - { - $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) - } + $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*)) } } @@ -737,7 +766,7 @@ macro_rules! dev_printk { /// ``` #[macro_export] macro_rules! dev_emerg { - ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*) } } /// Prints an alert-level message (level 1) prefixed with device information. @@ -763,7 +792,7 @@ macro_rules! dev_emerg { /// ``` #[macro_export] macro_rules! dev_alert { - ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*) } } /// Prints a critical-level message (level 2) prefixed with device information. @@ -789,7 +818,7 @@ macro_rules! dev_alert { /// ``` #[macro_export] macro_rules! dev_crit { - ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*) } } /// Prints an error-level message (level 3) prefixed with device information. @@ -815,7 +844,7 @@ macro_rules! dev_crit { /// ``` #[macro_export] macro_rules! dev_err { - ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*) } } /// Prints a warning-level message (level 4) prefixed with device information. @@ -841,7 +870,7 @@ macro_rules! dev_err { /// ``` #[macro_export] macro_rules! dev_warn { - ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*) } } /// Prints a notice-level message (level 5) prefixed with device information. @@ -867,7 +896,7 @@ macro_rules! dev_warn { /// ``` #[macro_export] macro_rules! dev_notice { - ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*) } } /// Prints an info-level message (level 6) prefixed with device information. @@ -893,7 +922,7 @@ macro_rules! dev_notice { /// ``` #[macro_export] macro_rules! dev_info { - ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*) } } /// Prints a debug-level message (level 7) prefixed with device information. @@ -919,5 +948,5 @@ macro_rules! dev_info { /// ``` #[macro_export] macro_rules! dev_dbg { - ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*) } } diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index 8e9721446014..c81fca5b4986 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -5,7 +5,10 @@ //! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is //! expected to implement [`RawDeviceId`]. -use core::mem::MaybeUninit; +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; /// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. /// @@ -47,112 +50,110 @@ pub unsafe trait RawDeviceIdIndex: RawDeviceId { /// The offset (in bytes) to the context/data field in the raw device ID. const DRIVER_DATA_OFFSET: usize; - /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceIdIndex`] - /// trait. - fn index(&self) -> usize; -} - -/// A zero-terminated device id array. -#[repr(C)] -pub struct RawIdArray<T: RawDeviceId, const N: usize> { - ids: [T::RawType; N], - sentinel: MaybeUninit<T::RawType>, -} + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray<Self, U>`. + unsafe fn info_unchecked<U>(&self) -> &'static U { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, this is stored in `IdArray<Self, U>` so is + // convertible to `&'static U`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<&U>() + .read() + } + } -impl<T: RawDeviceId, const N: usize> RawIdArray<T, N> { - #[doc(hidden)] - pub const fn size(&self) -> usize { - core::mem::size_of::<Self>() + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray<Self, U>`, or has NULL (or 0) as driver data. + unsafe fn info_unchecked_opt<U>(&self) -> Option<&'static U> { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, if this is stored in `IdArray<Self, U>`, this is + // convertible to `Option<&'static U>`. Otherwise it is NULL which is `None` as + // `Option<&U>`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<Option<&U>>() + .read() + } } } /// A zero-terminated device id array, followed by context data. #[repr(C)] -pub struct IdArray<T: RawDeviceId, U, const N: usize> { - raw_ids: RawIdArray<T, N>, - id_infos: [U; N], +pub struct IdArray<T: RawDeviceId, U: 'static, const N: usize> { + // This is `MaybeUninit<T::RawType>` so any bytes inside it can carry provenance in CTFE. + // If this were `T::RawType`, integer fields would not be able to contain pointers. + ids: [MaybeUninit<T::RawType>; N], + sentinel: MaybeUninit<T::RawType>, + phantom: PhantomData<&'static U>, } -impl<T: RawDeviceId, U, const N: usize> IdArray<T, U, N> { +// SAFETY: device ID is plain data plus a `&'static U` and can thus be sent between threads safely +// if `&U` can. +unsafe impl<T: RawDeviceId, U: Sync + 'static, const N: usize> Send for IdArray<T, U, N> {} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be shared between threads safely +// if `&U` can. +unsafe impl<T: RawDeviceId, U: Sync + 'static, const N: usize> Sync for IdArray<T, U, N> {} + +impl<T: RawDeviceId + RawDeviceIdIndex, U: 'static, const N: usize> IdArray<T, U, N> { /// Creates a new instance of the array. /// /// The contents are derived from the given identifiers and context information. - /// - /// # Safety - /// - /// `data_offset` as `None` is always safe. - /// If `data_offset` is `Some(data_offset)`, then: - /// - `data_offset` must be the correct offset (in bytes) to the context/data field - /// (e.g., the `driver_data` field) within the raw device ID structure. - /// - The field at `data_offset` must be correctly sized to hold a `usize`. - const unsafe fn build(ids: [(T, U); N], data_offset: Option<usize>) -> Self { + pub const fn new(ids: [(T, &'static U); N]) -> Self { let mut raw_ids = [const { MaybeUninit::<T::RawType>::uninit() }; N]; - let mut infos = [const { MaybeUninit::uninit() }; N]; let mut i = 0usize; while i < N { // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is // layout-wise compatible with `RawType`. raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; - if let Some(data_offset) = data_offset { - // SAFETY: by the safety requirement of this function, this would be effectively - // `raw_ids[i].driver_data = i;`. - unsafe { - raw_ids[i] - .as_mut_ptr() - .byte_add(data_offset) - .cast::<usize>() - .write(i); - } + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, this would be effectively + // `raw_ids[i].driver_data = ids[i].1;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_add(T::DRIVER_DATA_OFFSET) + .cast::<&U>() + .write(ids[i].1); } - // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but - // later forget `ids`. - infos[i] = MaybeUninit::new(unsafe { core::ptr::read(&ids[i].1) }); i += 1; } core::mem::forget(ids); Self { - raw_ids: RawIdArray { - // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use - // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this - // `array_assume_init` is safe. - ids: unsafe { core::mem::transmute_copy(&raw_ids) }, - sentinel: MaybeUninit::zeroed(), - }, - // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is - // safe. - id_infos: unsafe { core::mem::transmute_copy(&infos) }, + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, } } +} +impl<T: RawDeviceId, const N: usize> IdArray<T, (), N> { /// Creates a new instance of the array without writing index values. /// /// The contents are derived from the given identifiers and context information. /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. - pub const fn new_without_index(ids: [(T, U); N]) -> Self { - // SAFETY: Calling `Self::build` with `offset = None` is always safe, - // because no raw memory writes are performed in this case. - unsafe { Self::build(ids, None) } - } - - /// Reference to the contained [`RawIdArray`]. - pub const fn raw_ids(&self) -> &RawIdArray<T, N> { - &self.raw_ids - } -} + pub const fn new_without_index(ids: [T; N]) -> Self { + // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. + let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { core::mem::transmute_copy(&ids) }; + core::mem::forget(ids); -impl<T: RawDeviceId + RawDeviceIdIndex, U, const N: usize> IdArray<T, U, N> { - /// Creates a new instance of the array. - /// - /// The contents are derived from the given identifiers and context information. - pub const fn new(ids: [(T, U); N]) -> Self { - // SAFETY: by the safety requirement of `RawDeviceIdIndex`, - // `T::DRIVER_DATA_OFFSET` is guaranteed to be the correct offset (in bytes) to - // a field within `T::RawType`. - unsafe { Self::build(ids, Some(T::DRIVER_DATA_OFFSET)) } + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } } } @@ -165,12 +166,6 @@ impl<T: RawDeviceId + RawDeviceIdIndex, U, const N: usize> IdArray<T, U, N> { pub trait IdTable<T: RawDeviceId, U> { /// Obtain the pointer to the ID table. fn as_ptr(&self) -> *const T::RawType; - - /// Obtain the pointer to the bus specific device ID from an index. - fn id(&self, index: usize) -> &T::RawType; - - /// Obtain the pointer to the driver-specific information from an index. - fn info(&self, index: usize) -> &U; } impl<T: RawDeviceId, U, const N: usize> IdTable<T, U> for IdArray<T, U, N> { @@ -179,28 +174,45 @@ impl<T: RawDeviceId, U, const N: usize> IdTable<T, U> for IdArray<T, U, N> { // to access the sentinel. core::ptr::from_ref(self).cast() } - - fn id(&self, index: usize) -> &T::RawType { - &self.raw_ids.ids[index] - } - - fn info(&self, index: usize) -> &U { - &self.id_infos[index] - } } /// Create device table alias for modpost. #[macro_export] macro_rules! module_device_table { - ($table_type: literal, $module_table_name:ident, $table_name:ident) => { - #[rustfmt::skip] + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, $id_info_type: ty, + [$(($id: expr, $info:expr $(,)?)),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + $id_info_type, + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new([$(($id, &$info),)*]); + }; + + // Case for no ID info. + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, @none, + [$($id: expr),* $(,)?] + ) => { #[export_name = - concat!("__mod_device_table__", line!(), + concat!("__mod_device_table__", ::core::line!(), "__kmod_", module_path!(), "__", $table_type, "__", stringify!($table_name)) ] - static $module_table_name: [::core::mem::MaybeUninit<u8>; $table_name.raw_ids().size()] = - unsafe { ::core::mem::transmute_copy($table_name.raw_ids()) }; + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + (), + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new_without_index([$($id),*]); }; } diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 11ce500e9b76..d2924aaae008 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -21,9 +21,12 @@ use crate::{ sync::{ aref::ARef, rcu, - Arc, // + Arc, + Completion, // }, types::{ + CovariantForLt, + ForLt, ForeignOwnable, Opaque, // }, @@ -37,6 +40,8 @@ struct Inner<T> { node: Opaque<bindings::devres_node>, #[pin] data: Revocable<T>, + #[pin] + revocation: Completion, } /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to @@ -53,12 +58,17 @@ struct Inner<T> { /// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource /// anymore. /// +/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time +/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in +/// progress. +/// /// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s /// [`Drop`] implementation. /// /// # Examples /// /// ```no_run +/// # #![cfg(CONFIG_HAS_IOMEM)] /// use kernel::{ /// bindings, /// device::{ @@ -68,17 +78,19 @@ struct Inner<T> { /// devres::Devres, /// io::{ /// Io, -/// IoKnownSize, +/// IoBase, /// Mmio, /// MmioRaw, -/// PhysAddr, // +/// MmioBackend, +/// PhysAddr, +/// Region, // /// }, /// prelude::*, /// }; /// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. -/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>); +/// struct IoMem<const SIZE: usize>(MmioRaw<Region<SIZE>>); /// /// impl<const SIZE: usize> IoMem<SIZE> { /// /// # Safety @@ -93,7 +105,7 @@ struct Inner<T> { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) /// } /// } /// @@ -104,12 +116,13 @@ struct Inner<T> { /// } /// } /// -/// impl<const SIZE: usize> Deref for IoMem<SIZE> { -/// type Target = Mmio<SIZE>; +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<SIZE> { +/// type Backend = MmioBackend; +/// type Target = Region<SIZE>; /// -/// fn deref(&self) -> &Self::Target { +/// fn as_view(self) -> Mmio<'a, Region<SIZE>> { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } +/// unsafe { Mmio::from_raw(self.0) } /// } /// } /// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> { @@ -217,6 +230,7 @@ impl<T: Send + 'static> Devres<T> { }; }), data <- Revocable::new(data), + revocation <- Completion::new(), }), GFP_KERNEL, )?; @@ -254,7 +268,14 @@ impl<T: Send + 'static> Devres<T> { // SAFETY: `inner` is a valid `Inner<T>` pointer. let inner = unsafe { &*inner }; - inner.data.revoke(); + if inner.data.revoke() { + inner.revocation.complete_all(); + } else { + // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()` + // before returning to `devres_release_all()`, ensuring `T` is fully torn down before + // the device finishes unbinding. + inner.revocation.wait_for_completion(); + } } #[allow(clippy::missing_safety_doc)] @@ -297,10 +318,7 @@ impl<T: Send + 'static> Devres<T> { /// use kernel::{ /// device::Core, /// devres::Devres, - /// io::{ - /// Io, - /// IoKnownSize, // - /// }, + /// io::Io, /// pci, // /// }; /// @@ -354,6 +372,8 @@ impl<T: Send + 'static> Drop for Devres<T> { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. if unsafe { self.data().revoke_nosync() } { + self.inner.revocation.complete_all(); + // We revoked `self.data` before devres did, hence try to remove it. if self.remove_node() { // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` @@ -361,10 +381,114 @@ impl<T: Send + 'static> Drop for Devres<T> { // this additional reference count. drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) }); } + } else { + // The release callback is concurrently revoking; wait for it to finish + // `drop_in_place()` of the wrapped object before returning. + self.inner.revocation.wait_for_completion(); } } } +/// Guard returned by [`DevresLt::try_access`]. +/// +/// Dereferences to `F::Of<'a>`, shortening the lifetime of the stored data to the guard's borrow +/// lifetime. +pub struct DevresGuard<'a, F: CovariantForLt>(RevocableGuard<'a, F::Of<'static>>); + +impl<'a, F: CovariantForLt> core::ops::Deref for DevresGuard<'a, F> { + type Target = F::Of<'a>; + + #[inline] + fn deref(&self) -> &Self::Target { + F::cast_ref(&*self.0) + } +} + +/// Device-managed resource with [`ForLt`](trait@ForLt)-aware access. +/// +/// `DevresLt` wraps [`Devres`] and shortens the stored `'static` lifetime to the caller's borrow +/// lifetime in all access methods. +/// +/// Types that implement [`trait@CovariantForLt`] get direct-reference accessors ([`Self::access`], +/// [`Self::try_access`]). Plain [`ForLt`](trait@ForLt) types use closure-based accessors +/// ([`Self::access_with`], [`Self::try_access_with`]). +pub struct DevresLt<F: ForLt>(Devres<F::Of<'static>>) +where + for<'a> F::Of<'a>: Send; + +impl<F: ForLt> DevresLt<F> +where + for<'a> F::Of<'a>: Send, +{ + /// Creates a new [`DevresLt`] instance of the given `data`. + /// + /// # Safety + /// + /// The data must remain valid for the device's full bound scope. [`DevresLt`] allows + /// access until the device is unbound, which may outlast `'a`. + pub unsafe fn new<'a, E>( + dev: &'a Device<Bound>, + data: impl PinInit<F::Of<'a>, E>, + ) -> Result<Self> + where + Error: From<E>, + { + // SAFETY: The caller guarantees the data is valid for the device's full bound scope. + // Lifetimes do not affect layout, so F::Of<'a> and F::Of<'static> have identical + // representation; casting the slot pointer is sound. + let data = unsafe { pin_init::cast_pin_init(data) }; + + Ok(Self(Devres::new(dev, data)?)) + } + + /// Return a reference of the [`Device`] this [`DevresLt`] instance has been created with. + #[inline] + pub fn device(&self) -> &Device { + self.0.device() + } + + /// Obtain `&F::Of<'_>`, bypassing the [`Revocable`], through a closure. + /// + /// This method works like [`DevresLt::access`](DevresLt::access) but accepts any + /// [`trait@ForLt`] type, not just [`trait@CovariantForLt`]. + #[inline] + pub fn access_with<R, G>(&self, dev: &Device<Bound>, f: G) -> Result<R> + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.access(dev).map(f) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access_with`]. + #[inline] + pub fn try_access_with<R, G>(&self, f: G) -> Option<R> + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.data().try_access_with(f) + } +} + +impl<F: CovariantForLt> DevresLt<F> +where + for<'a> F::Of<'a>: Send, +{ + /// Obtain `&'a F::Of<'a>`, bypassing the [`Revocable`]. + /// + /// This method works like [`Devres::access`], but shortens the returned reference's lifetime + /// from `'static` to `'a` via [`CovariantForLt::cast_ref`]. + #[inline] + pub fn access<'a>(&'a self, dev: &'a Device<Bound>) -> Result<&'a F::Of<'a>> { + self.0.access(dev).map(F::cast_ref) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access`]. + #[inline] + pub fn try_access(&self) -> Option<DevresGuard<'_, F>> { + self.0.data().try_access().map(DevresGuard) + } +} + /// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. fn register_foreign<P>(dev: &Device<Bound>, data: P) -> Result where diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 200def84fb69..2ce09f8e90c6 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -14,14 +14,22 @@ use crate::{ }, error::to_result, fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + IoCopyable, + SysMem, + SysMemBackend, // + }, prelude::*, ptr::KnownSize, sync::aref::ARef, transmute::{ AsBytes, FromBytes, // - }, // - uaccess::UserSliceWriter, + }, + uaccess::UserSliceWriter, // }; use core::{ ops::{ @@ -449,7 +457,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> { // - `T: AsBytes + FromBytes` guarantees all bit patterns are valid, so partial writes on // error cannot leave the element in an invalid state. // - The DMA address has not been exposed yet, so there is no concurrent device access. - unsafe { init.__init(ptr)? }; + unsafe { pin_init::raw_try_init(ptr, init)? }; Ok(()) } @@ -577,7 +585,7 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coher /// # Invariants /// /// - For the lifetime of an instance of [`Coherent`], the `cpu_addr` is a valid pointer -/// to an allocated region of coherent memory and `dma_handle` is the DMA address base of the +/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the /// region. /// - The size in bytes of the allocation is equal to size information via pointer. // TODO @@ -594,7 +602,7 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coher // entire `Coherent` including the allocated memory itself. pub struct Coherent<T: KnownSize + ?Sized> { dev: ARef<device::Device>, - dma_handle: DmaAddress, + dma_addr: DmaAddress, cpu_addr: NonNull<T>, dma_attrs: Attrs, } @@ -619,11 +627,10 @@ impl<T: KnownSize + ?Sized> Coherent<T> { self.cpu_addr.as_ptr() } - /// Returns a DMA handle which may be given to the device as the DMA address base of - /// the region. + /// Returns a DMA address which may be given to the device as the base of the region. #[inline] - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr } /// Returns a reference to the data in the region. @@ -654,52 +661,6 @@ impl<T: KnownSize + ?Sized> Coherent<T> { // SAFETY: per safety requirement. unsafe { &mut *self.as_mut_ptr() } } - - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } - } - - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } - } } impl<T: AsBytes + FromBytes> Coherent<T> { @@ -716,13 +677,13 @@ impl<T: AsBytes + FromBytes> Coherent<T> { ); } - let mut dma_handle = 0; + let mut dma_addr = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), core::mem::size_of::<T>(), - &mut dma_handle, + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) @@ -734,7 +695,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> { // - We also hold a refcounted reference to the device. Ok(Self { dev: dev.into(), - dma_handle, + dma_addr, cpu_addr, dma_attrs, }) @@ -791,10 +752,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> { // SAFETY: // - `ptr` is valid, properly aligned, and points to exclusively owned memory. - // - If `__init` fails, `self` is dropped, which safely frees the underlying `Coherent`'s - // DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` requirements - // we are bypassing. - unsafe { init.__init(ptr)? }; + // - If `raw_try_init` fails, `self` is dropped, which safely frees the underlying + // `Coherent`'s DMA memory. `T: AsBytes + FromBytes` ensures there are no complex `Drop` + // requirements we are bypassing. + unsafe { pin_init::raw_try_init(ptr, init)? }; Ok(dmem) } @@ -833,13 +794,13 @@ impl<T: AsBytes + FromBytes> Coherent<T> { } let size = core::mem::size_of::<T>().checked_mul(len).ok_or(ENOMEM)?; - let mut dma_handle = 0; + let mut dma_addr = 0; // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. let addr = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), size, - &mut dma_handle, + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) @@ -851,7 +812,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> { // - We also hold a refcounted reference to the device. Ok(Coherent { dev: dev.into(), - dma_handle, + dma_addr, cpu_addr, dma_attrs, }) @@ -965,14 +926,14 @@ impl<T: KnownSize + ?Sized> Drop for Coherent<T> { fn drop(&mut self) { let size = T::size(self.cpu_addr.as_ptr()); // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. - // The cpu address, and the dma handle are valid due to the type invariants on + // The cpu address, and the dma address are valid due to the type invariants on // `Coherent`. unsafe { bindings::dma_free_attrs( self.dev.as_raw(), size, self.cpu_addr.as_ptr().cast(), - self.dma_handle, + self.dma_addr, self.dma_attrs.as_raw(), ) } @@ -1005,7 +966,11 @@ impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> { return Ok(0); }; - let count = self.size().saturating_sub(offset_val).min(writer.len()); + if offset_val >= self.size() { + return Ok(0); + } + + let count = (self.size() - offset_val).min(writer.len()); writer.write_dma(self, offset_val, count)?; @@ -1027,13 +992,13 @@ impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> { /// /// - `cpu_handle` holds the opaque handle returned by `dma_alloc_attrs` with /// `DMA_ATTR_NO_KERNEL_MAPPING` set, and is only valid for passing back to `dma_free_attrs`. -/// - `dma_handle` is the corresponding bus address for device DMA. +/// - `dma_addr` is the corresponding bus address for device DMA. /// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. /// - `dma_attrs` contains the attributes used for the allocation, always including /// `DMA_ATTR_NO_KERNEL_MAPPING`. pub struct CoherentHandle { dev: ARef<device::Device>, - dma_handle: DmaAddress, + dma_addr: DmaAddress, cpu_handle: NonNull<c_void>, size: usize, dma_attrs: Attrs, @@ -1057,13 +1022,13 @@ impl CoherentHandle { } let dma_attrs = dma_attrs | Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); - let mut dma_handle = 0; + let mut dma_addr = 0; // SAFETY: `dev.as_raw()` is valid by the type invariant on `device::Device`. let cpu_handle = unsafe { bindings::dma_alloc_attrs( dev.as_raw(), size, - &mut dma_handle, + &mut dma_addr, gfp_flags.as_raw(), dma_attrs.as_raw(), ) @@ -1072,11 +1037,11 @@ impl CoherentHandle { let cpu_handle = NonNull::new(cpu_handle).ok_or(ENOMEM)?; // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call - // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_handle` is the corresponding DMA address, + // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, // and we hold a refcounted reference to the device. Ok(Self { dev: dev.into(), - dma_handle, + dma_addr, cpu_handle, size, dma_attrs, @@ -1093,12 +1058,12 @@ impl CoherentHandle { Self::alloc_with_attrs(dev, size, gfp_flags, Attrs(0)) } - /// Returns the DMA handle for this allocation. + /// Returns the DMA address for this allocation. /// /// This address can be programmed into device hardware for DMA access. #[inline] - pub fn dma_handle(&self) -> DmaAddress { - self.dma_handle + pub fn dma_address(&self) -> DmaAddress { + self.dma_addr } /// Returns the size in bytes of this allocation. @@ -1117,100 +1082,170 @@ impl Drop for CoherentHandle { self.dev.as_raw(), self.size, self.cpu_handle.as_ptr(), - self.dma_handle, + self.dma_addr, self.dma_attrs.as_raw(), ) } } } -// SAFETY: `CoherentHandle` only holds a device reference, a DMA handle, an opaque CPU handle, +// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, // and a size. None of these are tied to a specific thread. unsafe impl Send for CoherentHandle {} // SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only -// operations on `&CoherentHandle` are reading the DMA handle and size, both of which are +// operations on `&CoherentHandle` are reading the DMA address and size, both of which are // plain `Copy` values. unsafe impl Sync for CoherentHandle {} -/// Reads a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!). -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { field: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; +/// View type for `Coherent`. /// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// let whole = kernel::dma_read!(alloc, [try: 2]); -/// let field = kernel::dma_read!(alloc, [panic: 1].field); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_read { - ($dma:expr, $($proj:tt)*) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - $crate::dma::Coherent::as_ptr(dma), $($proj)* - ); - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_read(dma, ptr) } - }}; +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA +/// address. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_addr: DmaAddress, } -/// Writes to a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`Coherent`], `proj` is a -/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the -/// projected location. -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { member: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// kernel::dma_write!(alloc, [try: 2].member, 0xf); -/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf }); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_write { - (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)* - ); - let val = $val; - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) } - }}; - (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*]) - }; - ($dma:expr, $($rest:tt)*) => { - $crate::dma_write!(@parse [$dma] [] [$($rest)*]) - }; +impl<T: ?Sized> Copy for CoherentView<'_, T> {} +impl<T: ?Sized> Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA address information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns the DMA address which may be given to the device as base of the region. + #[inline] + pub fn dma_address(self) -> DmaAddress { + self.dma_addr + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_addr = view.dma_addr + offset as DmaAddress; + CoherentView { + dma_addr, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl<T> IoCapable<T> for CoherentIoBackend +where + SysMemBackend: IoCapable<T>, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_addr: self.dma_addr, + } + } } diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index bf5ba0d27553..c9c74c4dde8f 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -291,26 +291,6 @@ macro_rules! module_driver { } } -// Calling the FFI function directly from the `Adapter` impl may result in it being called -// directly from driver modules. This happens since the Rust compiler will use monomorphisation, so -// it might happen that functions are instantiated within the calling driver module. For now, work -// around this with `#[inline(never)]` helpers. -// -// TODO: Remove once a more generic solution has been implemented. For instance, we may be able to -// leverage `bindgen` to take care of this depending on whether a symbol is (already) exported. -#[inline(never)] -#[allow(clippy::missing_safety_doc)] -#[allow(dead_code)] -#[must_use] -unsafe fn acpi_of_match_device( - adev: *const bindings::acpi_device, - of_match_table: *const bindings::of_device_id, - of_id: *mut *const bindings::of_device_id, -) -> bool { - // SAFETY: Safety requirements are the same as `bindings::acpi_of_match_device`. - unsafe { bindings::acpi_of_match_device(adev, of_match_table, of_id) } -} - /// The bus independent adapter to match a drivers and a devices. /// /// This trait should be implemented by the bus specific adapter, which represents the connection @@ -324,118 +304,23 @@ pub trait Adapter { /// The [`acpi::IdTable`] of the corresponding driver fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>>; - /// Returns the driver's private data from the matching entry in the [`acpi::IdTable`], if any. - /// - /// If this returns `None`, it means there is no match with an entry in the [`acpi::IdTable`]. - fn acpi_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - #[cfg(not(CONFIG_ACPI))] - { - let _ = dev; - None - } - - #[cfg(CONFIG_ACPI)] - { - let table = Self::acpi_id_table()?; - - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. - let raw_id = unsafe { bindings::acpi_match_device(table.as_ptr(), dev.as_raw()) }; - - if raw_id.is_null() { - None - } else { - // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct acpi_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<acpi::DeviceId>() }; - - Some(table.info(<acpi::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id))) - } - } - } - /// The [`of::IdTable`] of the corresponding driver. fn of_id_table() -> Option<of::IdTable<Self::IdInfo>>; - /// Returns the driver's private data from the matching entry in the [`of::IdTable`], if any. - /// - /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`]. - fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let table = Self::of_id_table()?; - - #[cfg(not(any(CONFIG_OF, CONFIG_ACPI)))] - { - let _ = (dev, table); - } - - #[cfg(CONFIG_OF)] - { - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. - let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; - - if !raw_id.is_null() { - // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; - - return Some(table.info( - <of::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id), - )); - } - } - - #[cfg(CONFIG_ACPI)] - { - use core::ptr; - use device::property::FwNode; - - let mut raw_id = ptr::null(); - - let fwnode = dev.fwnode().map_or(ptr::null_mut(), FwNode::as_raw); - - // SAFETY: `fwnode` is a pointer to a valid `fwnode_handle`. A null pointer will be - // passed through the function. - let adev = unsafe { bindings::to_acpi_device_node(fwnode) }; - - // SAFETY: - // - `adev` is a valid pointer to `acpi_device` or is null. It is guaranteed to be - // valid as long as `dev` is alive. - // - `table` has static lifetime, hence it's valid for read. - if unsafe { acpi_of_match_device(adev, table.as_ptr(), &raw mut raw_id) } { - // SAFETY: - // - the function returns true, therefore `raw_id` has been set to a pointer to a - // valid `of_device_id`. - // - `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::<of::DeviceId>() }; - - return Some(table.info( - <of::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id), - )); - } - } - - None - } - /// Returns the driver's private data from the matching entry of any of the ID tables, if any. /// /// If this returns `None`, it means that there is no match in any of the ID tables directly /// associated with a [`device::Device`]. - fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let id = Self::acpi_id_info(dev); - if id.is_some() { - return id; - } - - let id = Self::of_id_info(dev); - if id.is_some() { - return id; - } - - None + /// + /// # Safety + /// + /// The caller must ensure that the `dev` matched data is of type `Self::IdInfo`. + #[inline] + unsafe fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + // SAFETY: `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let data = unsafe { bindings::device_get_match_data(dev.as_raw()) }; + + // SAFETY: Per safety requirement, `data` is of type `Self::IdInfo`. + unsafe { data.cast::<Self::IdInfo>().as_ref() } } } diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 477cf771fb10..6b88ade28e24 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -32,6 +32,7 @@ use crate::{ }; use core::{ alloc::Layout, + cell::UnsafeCell, marker::PhantomData, mem, ops::Deref, @@ -74,66 +75,59 @@ macro_rules! drm_legacy_fields { /// A trait implemented by all possible contexts a [`Device`] can be used in. /// -/// Setting up a new [`Device`] is a multi-stage process. Each step of the process that a user -/// interacts with in Rust has a respective [`DeviceContext`] typestate. For example, -/// `Device<T, Registered>` would be a [`Device`] that reached the [`Registered`] [`DeviceContext`]. +/// A [`Device`] can be in one of the following contexts: /// -/// Each stage of this process is described below: +/// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may +/// or may not be registered with userspace. +/// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl +/// dispatch context. +/// - [`Registered`]: The device is currently registered with userspace and the parent bus device +/// is bound. /// -/// ```text -/// 1 2 3 -/// +--------------+ +------------------+ +-----------------------+ -/// |Device created| → |Device initialized| → |Registered w/ userspace| -/// +--------------+ +------------------+ +-----------------------+ -/// (Uninit) (Registered) -/// ``` -/// -/// 1. The [`Device`] is in the [`Uninit`] context and is not guaranteed to be initialized or -/// registered with userspace. Only a limited subset of DRM core functionality is available. -/// 2. The [`Device`] is guaranteed to be fully initialized, but is not guaranteed to be registered -/// with userspace. All DRM core functionality which doesn't interact with userspace is -/// available. We currently don't have a context for representing this. -/// 3. The [`Device`] is guaranteed to be fully initialized, and is guaranteed to have been -/// registered with userspace at some point - thus putting it in the [`Registered`] context. +/// Both `Device<T, Ioctl>` and `Device<T, Registered>` dereference to `Device<T>` ([`Normal`]), +/// so any method available on a [`Normal`] device is also available in the other contexts. +pub trait DeviceContext: Sealed + Send + Sync + 'static {} + +/// The general-purpose, reference-counted [`DeviceContext`]. /// -/// An important caveat of [`DeviceContext`] which must be kept in mind: when used as a typestate -/// for a reference type, it can only guarantee that a [`Device`] reached a particular stage in the -/// initialization process _at the time the reference was taken_. No guarantee is made in regards to -/// what stage of the process the [`Device`] is currently in. This means for instance that a -/// `&Device<T, Uninit>` may actually be registered with userspace, it just wasn't known to be -/// registered at the time the reference was taken. -pub trait DeviceContext: Sealed + Send + Sync {} - -/// The [`DeviceContext`] of a [`Device`] that was registered with userspace at some point. +/// A [`Device`] in this context may or may not be registered with userspace. This context is used +/// for reference-counted device handles and during device setup via [`UnregisteredDevice`]. /// -/// This represents a [`Device`] which is guaranteed to have been registered with userspace at -/// some point in time. Such a DRM device is guaranteed to have been fully-initialized. +/// [`AlwaysRefCounted`] is only implemented for `Device<T, Normal>`, making this the required +/// context for [`ARef`]-based device handles. +pub struct Normal; + +impl Sealed for Normal {} +impl DeviceContext for Normal {} + +/// The [`DeviceContext`] of a [`Device`] that is currently registered with userspace. /// -/// Note: A device in this context is not guaranteed to remain registered with userspace for its -/// entire lifetime, as this is impossible to guarantee at compile-time. +/// A [`Device`] in this context is guaranteed to be registered and its parent bus device is +/// guaranteed to be bound. This is enforced at runtime by [`RegistrationGuard`], which holds a +/// `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. /// /// # Invariants /// -/// A [`Device`] in this [`DeviceContext`] is guaranteed to have been registered with userspace -/// at some point in time. +/// The parent bus device is bound for the duration of any reference to a `Device<T, Registered>`. pub struct Registered; impl Sealed for Registered {} impl DeviceContext for Registered {} -/// The [`DeviceContext`] of a [`Device`] that may be unregistered and partly uninitialized. +/// The [`DeviceContext`] of a [`Device`] that has been registered with userspace previously. /// -/// A [`Device`] in this context is only guaranteed to be partly initialized, and may or may not -/// be registered with userspace. Thus operations which depend on the [`Device`] being fully -/// initialized, or which depend on the [`Device`] being registered with userspace are not -/// available through this [`DeviceContext`]. +/// A [`Device`] in this context has been registered at some point, but may be concurrently +/// unregistering or already unregistered. `drm_dev_enter()` can guard against this, ensuring the +/// device remains registered for the duration of the critical section. /// -/// A [`Device`] in this context can be used to create a -/// [`Registration`](drm::driver::Registration). -pub struct Uninit; +/// # Invariants +/// +/// A [`Device`] in this context has been registered with userspace via `drm_dev_register()` at +/// some point. +pub struct Ioctl; -impl Sealed for Uninit {} -impl DeviceContext for Uninit {} +impl Sealed for Ioctl {} +impl DeviceContext for Ioctl {} /// A [`Device`] which is known at compile-time to be unregistered with userspace. /// @@ -147,10 +141,10 @@ impl DeviceContext for Uninit {} /// /// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been /// registered with userspace until this type is dropped. -pub struct UnregisteredDevice<T: drm::Driver>(ARef<Device<T, Uninit>>, NotThreadSafe); +pub struct UnregisteredDevice<T: drm::Driver>(ARef<Device<T, Normal>>, NotThreadSafe); impl<T: drm::Driver> Deref for UnregisteredDevice<T> { - type Target = Device<T, Uninit>; + type Target = Device<T, Normal>; fn deref(&self) -> &Self::Target { &self.0 @@ -178,15 +172,13 @@ impl<T: drm::Driver> UnregisteredDevice<T> { master_drop: None, debugfs_init: None, - // Ignore the Uninit DeviceContext below. It is only provided because it is required by the - // compiler, and it is not actually used by these functions. - gem_create_object: T::Object::<Uninit>::ALLOC_OPS.gem_create_object, - prime_handle_to_fd: T::Object::<Uninit>::ALLOC_OPS.prime_handle_to_fd, - prime_fd_to_handle: T::Object::<Uninit>::ALLOC_OPS.prime_fd_to_handle, - gem_prime_import: T::Object::<Uninit>::ALLOC_OPS.gem_prime_import, - gem_prime_import_sg_table: T::Object::<Uninit>::ALLOC_OPS.gem_prime_import_sg_table, - dumb_create: T::Object::<Uninit>::ALLOC_OPS.dumb_create, - dumb_map_offset: T::Object::<Uninit>::ALLOC_OPS.dumb_map_offset, + gem_create_object: T::Object::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, + dumb_create: T::Object::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, show_fdinfo: None, fbdev_probe: None, @@ -203,15 +195,19 @@ impl<T: drm::Driver> UnregisteredDevice<T> { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::<T::OwnerModule>().as_ptr()); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// /// This can be used to create a [`Registration`](kernel::drm::Registration). - pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<Self> { + pub fn new( + dev: &T::ParentDevice<device::Bound>, + data: impl PinInit<T::Data, Error>, + ) -> Result<Self> { // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()` // compatible `Layout`. - let layout = Kmalloc::aligned_layout(Layout::new::<Device<T, Uninit>>()); + let layout = Kmalloc::aligned_layout(Layout::new::<Device<T, Normal>>()); // Use a temporary vtable without a `release` callback until `data` is initialized, so // init failure can release the DRM device without dropping uninitialized fields. @@ -223,12 +219,12 @@ impl<T: drm::Driver> UnregisteredDevice<T> { // SAFETY: // - `alloc_vtable` reference remains valid until no longer used, // - `dev` is valid by its type invarants, - let raw_drm: *mut Device<T, Uninit> = unsafe { + let raw_drm: *mut Device<T, Normal> = unsafe { bindings::__drm_dev_alloc( - dev.as_raw(), + dev.as_ref().as_raw(), &alloc_vtable, layout.size(), - mem::offset_of!(Device<T, Uninit>, dev), + mem::offset_of!(Device<T, Normal>, dev), ) } .cast(); @@ -244,7 +240,7 @@ impl<T: drm::Driver> UnregisteredDevice<T> { // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; @@ -253,6 +249,9 @@ impl<T: drm::Driver> UnregisteredDevice<T> { // SAFETY: `drm_dev` is still private to this function. unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; + // SAFETY: `raw_drm` is valid; no concurrent access before registration. + unsafe { (*raw_drm.as_ptr()).registration_data = UnsafeCell::new(NonNull::dangling()) }; + // SAFETY: The reference count is one, and now we take ownership of that reference as a // `drm::Device`. // INVARIANT: We just created the device above, but have yet to call `drm_dev_register`. @@ -264,16 +263,8 @@ impl<T: drm::Driver> UnregisteredDevice<T> { /// A typed DRM device with a specific [`drm::Driver`] implementation and [`DeviceContext`]. /// -/// Since DRM devices can be used before being fully initialized and registered with userspace, `C` -/// represents the furthest [`DeviceContext`] we can guarantee that this [`Device`] has reached. -/// -/// Keep in mind: this means that an unregistered device can still have the registration state -/// [`Registered`] as long as it was registered with userspace once in the past, and that the -/// behavior of such a device is still well-defined. Additionally, a device with the registration -/// state [`Uninit`] simply does not have a guaranteed registration state at compile time, and could -/// be either registered or unregistered. Since there is no way to guarantee a long-lived reference -/// to an unregistered device would remain unregistered, we do not provide a [`DeviceContext`] for -/// this. +/// A device in the [`Registered`] context is currently registered with userspace and its parent +/// bus device is bound. The [`Normal`] context is the general-purpose, reference-counted context. /// /// # Invariants /// @@ -281,9 +272,10 @@ impl<T: drm::Driver> UnregisteredDevice<T> { /// * The data layout of `Self` remains the same across all implementations of `C`. /// * Any invariants for `C` also apply. #[repr(C)] -pub struct Device<T: drm::Driver, C: DeviceContext = Registered> { +pub struct Device<T: drm::Driver, C: DeviceContext = Normal> { dev: Opaque<bindings::drm_device>, data: T::Data, + pub(super) registration_data: UnsafeCell<NonNull<T::RegistrationData<'static>>>, _ctx: PhantomData<C>, } @@ -352,7 +344,111 @@ impl<T: drm::Driver, C: DeviceContext> Device<T, C> { } } -impl<T: drm::Driver, C: DeviceContext> Deref for Device<T, C> { +impl<T: drm::Driver> Device<T, Ioctl> { + /// Guard against the parent bus device being unbound. + /// + /// Returns a [`RegistrationGuard`] if the device has not been unplugged, [`None`] otherwise. + /// + /// While [`RegistrationGuard`] is held the parent device is guaranteed to be bound. + #[must_use] + pub fn registration_guard(&self) -> Option<RegistrationGuard<'_, T>> { + let mut idx: i32 = 0; + // SAFETY: `self.as_raw()` is a valid pointer to a `struct drm_device`. + if unsafe { bindings::drm_dev_enter(self.as_raw(), &mut idx) } { + // INVARIANT: + // - `idx` is the SRCU index from the successful `drm_dev_enter()` above. + // - The parent bus device is bound: `drm_dev_enter()` succeeded, meaning + // `drm_dev_unplug()` has not completed; since it is only called from + // `Registration::drop()` during parent unbind, the parent is still bound. + Some(RegistrationGuard { + // SAFETY: See INVARIANT above; the `Registered` context invariant holds. + dev: unsafe { self.assume_ctx() }, + idx, + _not_send: NotThreadSafe, + }) + } else { + None + } + } +} + +/// A guard proving the DRM device is registered and the parent bus device is bound. +/// +/// The guard dereferences to [`Device<T, Registered>`], providing access to the DRM device with +/// the guarantee that the parent bus device is bound for the entire duration of the critical +/// section. +/// +/// Internally this is backed by a `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section. +/// +/// # Invariants +/// +/// - `idx` is the SRCU read lock index returned by a successful `drm_dev_enter()` call. +/// - The parent bus device of `dev` is bound for the lifetime of this guard. +#[must_use] +pub struct RegistrationGuard<'a, T: drm::Driver> { + dev: &'a Device<T, Registered>, + idx: i32, + _not_send: NotThreadSafe, +} + +impl<T: drm::Driver> Device<T, Registered> { + /// Returns a reference to the registration data with lifetime shortened from `'static`. + /// + /// # Safety + /// + /// The returned reference must not be exposed to code that can choose a concrete lifetime for + /// it, as that would be unsound for types that are invariant over their lifetime parameter + /// (e.g. it must be passed through an HRTB-bounded closure). + #[inline] + unsafe fn registration_data_unchecked(&self) -> &T::RegistrationData<'_> { + // SAFETY: + // - `Registered` guarantees the parent bus device is bound, hence the pointer is valid. + // - The pointer cast from `Of<'static>` to `Of<'_>` is layout-compatible since lifetimes + // are erased at runtime. + // - Caller guarantees the reference is only used behind an HRTB, making the lifetime + // shortening sound regardless of variance. + unsafe { (*self.registration_data.get()).cast::<_>().as_ref() } + } + + /// Access the registration data through a closure, with the lifetime tied to the closure + /// scope. + /// + /// The data is owned by [`Registration`](drm::Registration) and is guaranteed to remain valid + /// as long as the device is registered, since [`Registration`](drm::Registration)'s `drop` + /// calls `drm_dev_unplug()` which waits for all `drm_dev_enter()` critical sections to + /// complete. + #[inline] + pub fn registration_data_with<R, F>(&self, f: F) -> R + where + F: for<'a> FnOnce(&'a T::RegistrationData<'a>) -> R, + { + // SAFETY: `Registered` guarantees the device is registered and the parent bus device is + // bound. The closure's HRTB `for<'a>` prevents the caller from smuggling in references + // with a concrete short lifetime, satisfying the lifetime requirement of + // `registration_data_unchecked`. + f(unsafe { self.registration_data_unchecked() }) + } +} + +impl<T: drm::Driver> Deref for RegistrationGuard<'_, T> { + type Target = Device<T, Registered>; + + #[inline] + fn deref(&self) -> &Self::Target { + self.dev + } +} + +impl<T: drm::Driver> Drop for RegistrationGuard<'_, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: `self.idx` was returned by a successful `drm_dev_enter()` call, as guaranteed + // by the type invariants of `RegistrationGuard`. + unsafe { bindings::drm_dev_exit(self.idx) }; + } +} + +impl<T: drm::Driver> Deref for Device<T> { type Target = T::Data; fn deref(&self) -> &Self::Target { @@ -360,9 +456,31 @@ impl<T: drm::Driver, C: DeviceContext> Deref for Device<T, C> { } } +impl<T: drm::Driver> Deref for Device<T, Registered> { + type Target = Device<T>; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device<T, Registered>`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + +impl<T: drm::Driver> Deref for Device<T, Ioctl> { + type Target = Device<T>; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: The caller holds a `Device<T, Ioctl>`, which guarantees all invariants + // of the weaker `Normal` context. + unsafe { self.assume_ctx() } + } +} + // SAFETY: DRM device objects are always reference counted and the get/put functions // satisfy the requirements. -unsafe impl<T: drm::Driver, C: DeviceContext> AlwaysRefCounted for Device<T, C> { +unsafe impl<T: drm::Driver> AlwaysRefCounted for Device<T> { fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::drm_dev_get(self.as_raw()) }; @@ -377,11 +495,29 @@ unsafe impl<T: drm::Driver, C: DeviceContext> AlwaysRefCounted for Device<T, C> } } -impl<T: drm::Driver, C: DeviceContext> AsRef<device::Device> for Device<T, C> { - fn as_ref(&self) -> &device::Device { +impl<T: drm::Driver> AsRef<T::ParentDevice<device::Normal>> for Device<T> { + fn as_ref(&self) -> &T::ParentDevice<device::Normal> { // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, // which is guaranteed by the type invariant. - unsafe { device::Device::from_raw((*self.as_raw()).dev) } + let dev = unsafe { device::Device::from_raw((*self.as_raw()).dev) }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } + } +} + +impl<T: drm::Driver> AsRef<T::ParentDevice<device::Bound>> for Device<T, Registered> { + #[inline] + fn as_ref(&self) -> &T::ParentDevice<device::Bound> { + let dev = (**self).as_ref().as_ref(); + + // SAFETY: A `Device<T, Registered>` guarantees that the parent device is bound. + let dev = unsafe { dev.as_bound() }; + + // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent + // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`. + unsafe { device::AsBusDevice::from_device(dev) } } } @@ -392,12 +528,10 @@ unsafe impl<T: drm::Driver, C: DeviceContext> Send for Device<T, C> {} // by the synchronization in `struct drm_device`. unsafe impl<T: drm::Driver, C: DeviceContext> Sync for Device<T, C> {} -impl<T, C, const ID: u64> WorkItem<ID> for Device<T, C> +impl<T: drm::Driver, const ID: u64> WorkItem<ID> for Device<T> where - T: drm::Driver, T::Data: WorkItem<ID, Pointer = ARef<Self>>, T::Data: HasWork<Self, ID>, - C: DeviceContext, { type Pointer = ARef<Self>; diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 25f7e233884d..74f6ed690d8b 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -7,16 +7,12 @@ use crate::{ bindings, device, - devres, drm, error::to_result, prelude::*, sync::aref::ARef, // }; -use core::{ - mem, - ptr::NonNull, // -}; +use core::ptr::NonNull; /// Driver use the GEM memory manager. This should be set for all modern drivers. pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; @@ -110,12 +106,23 @@ pub trait Driver { /// Context data associated with the DRM driver type Data: Sync + Send; + /// Data owned by the [`Registration`] and accessible within a + /// [`RegistrationGuard`](drm::RegistrationGuard) critical section via + /// [`Device::registration_data_with()`](drm::Device::registration_data_with). + /// + /// The lifetime parameter is tied to the [`Registration`] scope, which is enclosed in the + /// parent bus device binding scope but may be shorter. + type RegistrationData<'a>: Send + Sync + 'a; + /// The type used to manage memory for this driver. - type Object<Ctx: drm::DeviceContext>: AllocImpl; + type Object: AllocImpl; /// The type used to represent a DRM File (client) type File: drm::file::DriverFile; + /// The bus device type of the parent device that the DRM device is associated with. + type ParentDevice<Ctx: device::DeviceContext>: device::AsBusDevice<Ctx>; + /// Driver metadata const INFO: DriverInfo; @@ -125,7 +132,7 @@ pub trait Driver { /// Sets the `DRIVER_RENDER` feature for this driver. /// /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to - /// userspace. The render node is an alternate low-priviledge way to access + /// userspace. The render node is an alternate low-privilege way to access /// the driver, which is enforced on a per-ioctl level. Userspace processes /// that open the render node can only invoke ioctls explicitly listed as /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas @@ -136,68 +143,84 @@ pub trait Driver { /// The registration type of a `drm::Device`. /// /// Once the `Registration` structure is dropped, the device is unregistered. -pub struct Registration<T: Driver>(ARef<drm::Device<T>>); - -impl<T: Driver> Registration<T> { - fn new(drm: drm::UnregisteredDevice<T>, flags: usize) -> Result<Self> { - // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`. - to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?; - - // SAFETY: We just called `drm_dev_register` above - let new = NonNull::from(unsafe { drm.assume_ctx() }); - - // Leak the ARef from UnregisteredDevice in preparation for transferring its ownership. - mem::forget(drm); - - // SAFETY: `drm`'s `Drop` constructor was never called, ensuring that there remains at least - // one reference to the device - which we take ownership over here. - let new = unsafe { ARef::from_raw(new) }; - - Ok(Self(new)) - } +pub struct Registration<'a, T: Driver> { + drm: ARef<drm::Device<T>>, + _reg_data: Pin<KBox<T::RegistrationData<'a>>>, +} - /// Registers a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. +impl<'a, T: Driver> Registration<'a, T> { + /// Register a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace. /// - /// Ownership of the [`Registration`] object is passed to [`devres::register`]. - pub fn new_foreign_owned<'a>( - drm: drm::UnregisteredDevice<T>, + /// # Safety + /// + /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since the registration data may contain borrowed + /// references that become invalid after `'a` ends. + pub unsafe fn new<E>( dev: &'a device::Device<device::Bound>, + drm: drm::UnregisteredDevice<T>, + reg_data: impl PinInit<T::RegistrationData<'a>, E>, flags: usize, - ) -> Result<&'a drm::Device<T>> + ) -> Result<Self> where - T: 'static, + Error: From<E>, { - if drm.as_ref().as_raw() != dev.as_raw() { + let parent = drm.as_ref(); + if parent.as_ref().as_raw() != dev.as_raw() { return Err(EINVAL); } - let reg = Registration::<T>::new(drm, flags)?; - let drm = NonNull::from(reg.device()); + let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once ioctls can be called. + let ptr: NonNull<T::RegistrationData<'static>> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); - devres::register(dev, reg, GFP_KERNEL)?; + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *drm.registration_data.get() = ptr }; + + // SAFETY: `drm` is a valid, initialized but not yet registered DRM device. + let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) }; + if let Err(e) = to_result(ret) { + // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent + // access to `registration_data` is possible at this point. + unsafe { *drm.registration_data.get() = NonNull::dangling() }; + return Err(e); + } - // SAFETY: Since `reg` was passed to devres::register(), the device now owns the lifetime - // of the DRM registration - ensuring that this references lives for at least as long as 'a. - Ok(unsafe { drm.as_ref() }) + Ok(Self { + drm: (&*drm).into(), + _reg_data: reg_data, + }) } /// Returns a reference to the `Device` instance for this registration. pub fn device(&self) -> &drm::Device<T> { - &self.0 + &self.drm } } // SAFETY: `Registration` doesn't offer any methods or access to fields when shared between // threads, hence it's safe to share it. -unsafe impl<T: Driver> Sync for Registration<T> {} +unsafe impl<T: Driver> Sync for Registration<'_, T> {} // SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. -unsafe impl<T: Driver> Send for Registration<T> {} +unsafe impl<T: Driver> Send for Registration<'_, T> {} -impl<T: Driver> Drop for Registration<T> { +impl<T: Driver> Drop for Registration<'_, T> { fn drop(&mut self) { + // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing + // `drm_dev_enter()` critical sections complete before unregistration proceeds. This + // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in + // `drm_dev_unplug()` to guarantee that the parent device is still bound within the + // critical section. + // // SAFETY: Safe by the invariant of `ARef<drm::Device<T>>`. The existence of this - // `Registration` also guarantees the this `drm::Device` is actually registered. - unsafe { bindings::drm_dev_unregister(self.0.as_raw()) }; + // `Registration` also guarantees that this `drm::Device` is actually registered. + unsafe { bindings::drm_dev_unplug(self.drm.as_raw()) }; + // After drm_dev_unplug(), the SRCU barrier guarantees that all RegistrationGuard critical + // sections have completed, so no one holds a reference to reg_data anymore. + // reg_data is dropped here automatically. } } diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index c8b66d816871..e1ebad77ebe2 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -10,7 +10,7 @@ use crate::{ self, device::{ DeviceContext, - Registered, // + Normal, // }, driver::{ AllocImpl, @@ -81,11 +81,10 @@ pub type DriverFile<T> = drm::File<<<T as DriverObject>::Driver as drm::Driver>: /// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`]. /// /// [`Driver`]: drm::Driver -pub type DriverAllocImpl<T, Ctx = Registered> = - <<T as DriverObject>::Driver as drm::Driver>::Object<Ctx>; +pub type DriverAllocImpl<T> = <<T as DriverObject>::Driver as drm::Driver>::Object; /// GEM object functions, which must be implemented by drivers. -pub trait DriverObject: Sync + Send + Sized { +pub trait DriverObject: Sync + Send + Sized + 'static { /// Parent `Driver` for this object. type Driver: drm::Driver; @@ -93,8 +92,8 @@ pub trait DriverObject: Sync + Send + Sized { type Args; /// Create a new driver data object for a GEM object of a given size. - fn new<Ctx: DeviceContext>( - dev: &drm::Device<Self::Driver, Ctx>, + fn new( + dev: &drm::Device<Self::Driver>, size: usize, args: Self::Args, ) -> impl PinInit<Self, Error>; @@ -109,7 +108,7 @@ pub trait DriverObject: Sync + Send + Sized { } /// Trait that represents a GEM object subtype -pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { +pub trait IntoGEMObject: Sized + super::private::Sealed { /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as /// this owning object is valid. fn as_raw(&self) -> *mut bindings::drm_gem_object; @@ -118,7 +117,8 @@ pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { /// /// # Safety /// - /// - `self_ptr` must be a valid pointer to `Self`. + /// - `self_ptr` must be a valid pointer to the `struct drm_gem_object` embedded in a + /// valid instance of `Self`. /// - The caller promises that holding the immutable reference returned by this function does /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; @@ -183,7 +183,7 @@ pub trait BaseObject: IntoGEMObject { fn create_handle<D, F>(&self, file: &drm::File<F>) -> Result<u32> where Self: AllocImpl<Driver = D>, - D: drm::Driver<Object<Registered> = Self, File = F>, + D: drm::Driver<Object = Self, File = F>, F: drm::file::DriverFile<Driver = D>, { let mut handle: u32 = 0; @@ -197,8 +197,8 @@ pub trait BaseObject: IntoGEMObject { /// Looks up an object by its handle for a given `File`. fn lookup_handle<D, F>(file: &drm::File<F>, handle: u32) -> Result<ARef<Self>> where - Self: AllocImpl<Driver = D>, - D: drm::Driver<Object<Registered> = Self, File = F>, + Self: AllocImpl<Driver = D> + AlwaysRefCounted, + D: drm::Driver<Object = Self, File = F>, F: drm::file::DriverFile<Driver = D>, { // SAFETY: The arguments are all valid per the type invariants. @@ -254,7 +254,7 @@ impl<T: IntoGEMObject> BaseObjectPrivate for T {} /// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object<T: DriverObject + Send + Sync, Ctx: DeviceContext = Registered> { +pub struct Object<T: DriverObject + Send + Sync, Ctx: DeviceContext = Normal> { obj: Opaque<bindings::drm_gem_object>, #[pin] data: T, @@ -280,12 +280,43 @@ impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> { rss: None, }; + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &drm::Device<T::Driver, Ctx> { + // SAFETY: + // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM + // object lives. + // - The device we used for creating the gem object is passed as &drm::Device<T::Driver> to + // Object::<T>::new(), so we know that `T::Driver` is the right generic parameter to use + // here. + // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we + // return. + unsafe { drm::Device::from_raw((*self.as_raw()).dev) } + } + + fn as_raw(&self) -> *mut bindings::drm_gem_object { + self.obj.get() + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + let ptr: *mut Opaque<bindings::drm_gem_object> = obj.cast(); + + // SAFETY: All of our objects are of type `Object<T>`. + let this = unsafe { crate::container_of!(ptr, Self, obj) }; + + // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct + // drm_gem_object`. + unsafe { bindings::drm_gem_object_release(obj) }; + + // SAFETY: All of our objects are allocated via `KBox`, and we're in the + // free callback which guarantees this object has zero remaining references, + // so we can drop it. + let _ = unsafe { KBox::from_raw(this) }; + } +} + +impl<T: DriverObject> Object<T> { /// Create a new GEM object. - pub fn new( - dev: &drm::Device<T::Driver, Ctx>, - size: usize, - args: T::Args, - ) -> Result<ARef<Self>> { + pub fn new(dev: &drm::Device<T::Driver>, size: usize, args: T::Args) -> Result<ARef<Self>> { let obj: Pin<KBox<Self>> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), @@ -321,46 +352,12 @@ impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> { // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. Ok(unsafe { ARef::from_raw(ptr) }) } - - /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &drm::Device<T::Driver, Ctx> { - // SAFETY: - // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM - // object lives. - // - The device we used for creating the gem object is passed as &drm::Device<T::Driver> to - // Object::<T>::new(), so we know that `T::Driver` is the right generic parameter to use - // here. - // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we - // return. - unsafe { drm::Device::from_raw((*self.as_raw()).dev) } - } - - fn as_raw(&self) -> *mut bindings::drm_gem_object { - self.obj.get() - } - - extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { - let ptr: *mut Opaque<bindings::drm_gem_object> = obj.cast(); - - // SAFETY: All of our objects are of type `Object<T>`. - let this = unsafe { crate::container_of!(ptr, Self, obj) }; - - // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct - // drm_gem_object`. - unsafe { bindings::drm_gem_object_release(obj) }; - - // SAFETY: All of our objects are allocated via `KBox`, and we're in the - // free callback which guarantees this object has zero remaining references, - // so we can drop it. - let _ = unsafe { KBox::from_raw(this) }; - } } impl_aref_for_gem_obj! { - impl<T, C> for Object<T, C> + impl<T> for Object<T> where - T: DriverObject, - C: DeviceContext + T: DriverObject } impl<T: DriverObject, Ctx: DeviceContext> super::private::Sealed for Object<T, Ctx> {} @@ -387,10 +384,10 @@ impl<T: DriverObject, Ctx: DeviceContext> AllocImpl for Object<T, Ctx> { }; } -pub(super) const fn create_fops() -> bindings::file_operations { +pub(super) const fn create_fops(owner: *mut bindings::module) -> bindings::file_operations { let mut fops: bindings::file_operations = pin_init::zeroed(); - fops.owner = core::ptr::null_mut(); + fops.owner = owner; fops.open = Some(bindings::drm_open); fops.release = Some(bindings::drm_release); fops.unlocked_ioctl = Some(bindings::drm_ioctl); diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 34af402899a0..a687d46d170d 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -11,28 +11,57 @@ use crate::{ container_of, + device::{ + self, + Bound, // + }, + devres::*, drm::{ driver, gem, private::Sealed, - Device, - DeviceContext, - Registered, // + Device, // + }, + error::{ + from_err_ptr, + to_result, // + }, + io::{ + IoBase, + Region, + SysMem, + SysMemBackend, // }, - error::to_result, prelude::*, - sync::aref::ARef, - types::Opaque, // + scatterlist, + sync::{ + aref::ARef, + new_mutex, + Mutex, + SetOnce, // + }, + types::{ + NotThreadSafe, + Opaque, // + }, }; use core::{ - marker::PhantomData, + ffi::c_void, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, ops::{ Deref, DerefMut, // }, - ptr::NonNull, // + ptr::{ + self, + NonNull, // + }, }; use gem::{ + BaseObject, BaseObjectPrivate, DriverObject, IntoGEMObject, // @@ -42,15 +71,24 @@ use gem::{ /// /// This is used with [`Object::new()`] to control various properties that can only be set when /// initially creating a shmem-backed GEM object. -#[derive(Default)] -pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { +pub struct ObjectConfig<'a, T: DriverObject> { /// Whether to set the write-combine map flag. pub map_wc: bool, /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. - pub parent_resv_obj: Option<&'a Object<T, C>>, + pub parent_resv_obj: Option<&'a Object<T>>, +} + +impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { + #[inline(always)] + fn default() -> Self { + Self { + map_wc: false, + parent_resv_obj: None, + } + } } /// A shmem-backed GEM object. @@ -59,33 +97,35 @@ pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> { /// /// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this /// object. -/// - Any type invariants of `C` apply to the parent DRM device for this GEM object. #[repr(C)] #[pin_data] -pub struct Object<T: DriverObject, C: DeviceContext = Registered> { +pub struct Object<T: DriverObject> { #[pin] obj: Opaque<bindings::drm_gem_shmem_object>, /// Parent object that owns this object's DMA reservation object. - parent_resv_obj: Option<ARef<Object<T, C>>>, + parent_resv_obj: Option<ARef<Object<T>>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop<SetOnce<Devres<SGTableMap<T>>>>, + #[pin] + /// Lock for protecting initialization of `sgt_res`. + sgt_lock: Mutex<()>, #[pin] inner: T, - _ctx: PhantomData<C>, } super::impl_aref_for_gem_obj! { - impl<T, C> for Object<T, C> + impl<T> for Object<T> where - T: DriverObject, - C: DeviceContext + T: DriverObject } // SAFETY: All GEM objects are thread-safe. -unsafe impl<T: DriverObject, C: DeviceContext> Send for Object<T, C> {} +unsafe impl<T: DriverObject> Send for Object<T> {} // SAFETY: All GEM objects are thread-safe. -unsafe impl<T: DriverObject, C: DeviceContext> Sync for Object<T, C> {} +unsafe impl<T: DriverObject> Sync for Object<T> {} -impl<T: DriverObject, C: DeviceContext> Object<T, C> { +impl<T: DriverObject> Object<T> { /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { free: Some(Self::free_callback), @@ -112,21 +152,166 @@ impl<T: DriverObject, C: DeviceContext> Object<T, C> { self.obj.get() } + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &Device<T::Driver> { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within an + // `Object<T>` + let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); + + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // alive. + // SAFETY: + // - We verified above that `this` is valid. + // - We are in free_callback, guaranteeing we have exclusive access to `this` and that + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(base) }; + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Attempt to create a vmap from the gem object, and confirm the size of said vmap. + fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result<VMap<T, R, SIZE>> + where + R: Deref<Target = Self> + From<&'a Self>, + { + // INVARIANT: We check here that the gem object is at least as large as `SIZE`. + if self.size() < SIZE { + return Err(ENOSPC); + } + + let mut map: MaybeUninit<bindings::iosys_map> = MaybeUninit::uninit(); + let guard = DmaResvGuard::new(self); + + // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held. + to_result(unsafe { + bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()) + })?; + + // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which + // re-acquires the lock). + drop(guard); + + // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed + // that map is properly initialized. + let map = unsafe { map.assume_init() }; + + // XXX: We don't currently support iomem allocations + if map.is_iomem { + // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid + // memory mapping. + unsafe { self.raw_vunmap(map) }; + + Err(ENOTSUPP) + } else { + Ok(VMap { + // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the + // lifetime of `VMap` itself. + // SAFETY: We checked that this is not an iomem allocation, making it safe to read + // vaddr. + addr: unsafe { map.__bindgen_anon_1.vaddr }, + owner: self.into(), + }) + } + } + + /// Unmap a vmap from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` is a valid vmap on this gem object. + /// - The caller promises that the memory pointed to by map will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) { + let _guard = DmaResvGuard::new(self); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held. + // - The caller promises that `map` is a valid vmap on this gem object. + unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) }; + } + + /// Creates and returns a virtual kernel memory mapping for this object. + #[inline] + pub fn vmap<const SIZE: usize>(&self) -> Result<VMapRef<'_, T, SIZE>> { + self.make_vmap() + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device<Bound>, + ) -> Result<&'a scatterlist::SGTable> { + let parent = self.dev().as_ref(); + if dev.as_raw() != parent.as_ref().as_raw() { + return Err(EINVAL); + } + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. + let _guard = self.sgt_lock.lock(); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + }; + + Ok(sgt_res.access(dev)?) + } + /// Create a new shmem-backed DRM object of the given size. /// /// Additional config options can be specified using `config`. pub fn new( - dev: &Device<T::Driver, C>, + dev: &Device<T::Driver>, size: usize, - config: ObjectConfig<'_, T, C>, + config: ObjectConfig<'_, T>, args: T::Args, ) -> Result<ARef<Self>> { let new: Pin<KBox<Self>> = KBox::try_pin_init( try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_lock <- new_mutex!(()), inner <- T::new(dev, size, args), - _ctx: PhantomData::<C>, }), GFP_KERNEL, )?; @@ -158,36 +343,14 @@ impl<T: DriverObject, C: DeviceContext> Object<T, C> { Ok(obj) } - /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &Device<T::Driver, C> { - // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. - unsafe { Device::from_raw((*self.as_raw()).dev) } - } - - extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { - // SAFETY: - // - DRM always passes a valid gem object here - // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that - // `obj` is contained within a drm_gem_shmem_object - let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; - - // SAFETY: - // - We're in free_callback - so this function is safe to call. - // - We won't be using the gem resources on `this` after this call. - unsafe { bindings::drm_gem_shmem_release(this) }; - - // SAFETY: - // - We verified above that `obj` is valid, which makes `this` valid - // - This function is set in AllocOps, so we know that `this` is contained within a - // `Object<T, C>` - let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); - - // SAFETY: We're recovering the Kbox<> we created in gem_create_object() - let _ = unsafe { KBox::from_raw(this) }; + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + #[inline] + pub fn owned_vmap<const SIZE: usize>(&self) -> Result<VMapOwned<T, SIZE>> { + self.make_vmap() } } -impl<T: DriverObject, C: DeviceContext> Deref for Object<T, C> { +impl<T: DriverObject> Deref for Object<T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -195,15 +358,15 @@ impl<T: DriverObject, C: DeviceContext> Deref for Object<T, C> { } } -impl<T: DriverObject, C: DeviceContext> DerefMut for Object<T, C> { +impl<T: DriverObject> DerefMut for Object<T> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } } -impl<T: DriverObject, C: DeviceContext> Sealed for Object<T, C> {} +impl<T: DriverObject> Sealed for Object<T> {} -impl<T: DriverObject, C: DeviceContext> gem::IntoGEMObject for Object<T, C> { +impl<T: DriverObject> gem::IntoGEMObject for Object<T> { fn as_raw(&self) -> *mut bindings::drm_gem_object { // SAFETY: // - Our immutable reference is proof that this is safe to dereference. @@ -222,7 +385,7 @@ impl<T: DriverObject, C: DeviceContext> gem::IntoGEMObject for Object<T, C> { } } -impl<T: DriverObject, C: DeviceContext> driver::AllocImpl for Object<T, C> { +impl<T: DriverObject> driver::AllocImpl for Object<T> { type Driver = T::Driver; const ALLOC_OPS: driver::AllocOps = driver::AllocOps { @@ -235,3 +398,324 @@ impl<T: DriverObject, C: DeviceContext> driver::AllocImpl for Object<T, C> { dumb_map_offset: None, }; } + +/// Private helper-type for holding the `dma_resv` object for a GEM shmem object. +/// +/// When this is dropped, the `dma_resv` lock is dropped as well. +/// +// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel. +struct DmaResvGuard<'a, T: DriverObject>(&'a Object<T>, NotThreadSafe); + +impl<'a, T: DriverObject> DmaResvGuard<'a, T> { + #[inline] + fn new(obj: &'a Object<T>) -> Self { + // SAFETY: This lock is initialized throughout the lifetime of `object`. + unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) }; + + Self(obj, NotThreadSafe) + } +} + +impl<'a, T: DriverObject> Drop for DmaResvGuard<'a, T> { + #[inline] + fn drop(&mut self) { + // SAFETY: We are releasing the lock grabbed during the creation of this object. + unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) }; + } +} + +/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The size of `owner` is >= SIZE. +/// - The memory pointed to by `addr` remains valid at least until this object is dropped. +pub struct VMap<D, R, const SIZE: usize = 0> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + addr: *mut c_void, + owner: R, +} + +/// An alias type for a reference to a shmem-based GEM object's VMap. +pub type VMapRef<'a, D, const SIZE: usize = 0> = VMap<D, &'a Object<D>, SIZE>; + +/// An alias type for an owned reference to a shmem-based GEM object's VMap. +pub type VMapOwned<D, const SIZE: usize = 0> = VMap<D, ARef<Object<D>>, SIZE>; + +impl<D, R, const SIZE: usize> VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + /// Borrows a reference to the object that owns this virtual mapping. + #[inline] + pub fn owner(&self) -> &Object<D> { + &self.owner + } +} + +impl<'a, D, R, const SIZE: usize> IoBase<'a> for &'a VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + type Backend = SysMemBackend; + type Target = Region<SIZE>; + + #[inline] + fn as_view(self) -> SysMem<'a, Region<SIZE>> { + let ptr = Region::ptr_from_raw_parts_mut(self.addr.cast(), self.owner.size()); + + // SAFETY: Per type invariants of `VMap`: + // - `addr .. addr + owner.size()` is a valid kernel accessible memory region. + // - `addr` is page-aligned, which satisfies `Region`'s 4-byte alignment requirement. + // - The memory remains valid until this `VMap` is dropped; since `self` is `&'a VMap`, + // the borrow prevents the `VMap` from being dropped for the lifetime `'a`. + unsafe { SysMem::new(ptr) } + } +} + +impl<D, R, const SIZE: usize> Drop for VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>>, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: + // - Our existence is proof that this map was previously created using self.owner. + // - Since we are in Drop, we are guaranteed that no one will access the memory + // through this mapping after calling this. + unsafe { + self.owner.raw_vunmap(bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr }, + }) + }; + } +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Send` so is `VMap`. +unsafe impl<D, R, const SIZE: usize> Send for VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>> + Send, +{ +} + +// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so +// long as `owner` is `Sync` so is `VMap`. +unsafe impl<D, R, const SIZE: usize> Sync for VMap<D, R, SIZE> +where + D: DriverObject, + R: Deref<Target = Object<D>> + Sync, +{ +} + +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. +/// +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap<T: DriverObject> { + obj: NonNull<Object<T>>, +} + +impl<T: DriverObject> Deref for SGTableMap<T> { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + } +} + +impl<T: DriverObject> Drop for SGTableMap<T> { + fn drop(&mut self) { + // SAFETY: `obj` is always valid via our type invariants + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; + } +} + +impl<T: DriverObject> SGTableMap<T> { + fn new(obj: &Object<T>) -> impl Init<Self, Error> { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl<T: DriverObject> Send for SGTableMap<T> {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl<T: DriverObject> Sync for SGTableMap<T> {} + +#[kunit_tests(rust_drm_gem_shmem)] +mod tests { + use super::*; + use crate::{ + drm::{ + self, + UnregisteredDevice, // + }, + faux, + io::Io, + page::PAGE_SIZE, // + }; + + // The bare minimum needed to create a fake drm driver for kunit + + #[pin_data] + struct KunitData {} + struct KunitDriver; + struct KunitFile; + #[pin_data] + struct KunitObject {} + + const INFO: drm::DriverInfo = drm::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c"kunit", + desc: c"Kunit", + }; + + impl drm::file::DriverFile for KunitFile { + type Driver = KunitDriver; + + fn open(_dev: &drm::Device<KunitDriver>) -> Result<Pin<KBox<Self>>> { + Ok(KBox::new(Self, GFP_KERNEL)?.into()) + } + } + + impl gem::DriverObject for KunitObject { + type Driver = KunitDriver; + type Args = (); + + fn new( + _dev: &drm::Device<KunitDriver>, + _size: usize, + _args: Self::Args, + ) -> impl PinInit<Self, Error> { + try_pin_init!(KunitObject {}) + } + } + + #[vtable] + impl drm::Driver for KunitDriver { + type Data = KunitData; + type RegistrationData<'a> = (); + type File = KunitFile; + type Object = Object<KunitObject>; + type ParentDevice<Ctx: device::DeviceContext> = faux::Device<Ctx>; + + const INFO: drm::DriverInfo = INFO; + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[]; + } + + fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice<KunitDriver>)> { + // Create a faux DRM device so we can test gem object creation. + let data = try_pin_init!(KunitData {}); + let reg = faux::Registration::new(c"Kunit", None)?; + let fdev = reg.as_ref(); + let drm = UnregisteredDevice::new(fdev, data)?; + + Ok((reg, drm)) + } + + #[test] + fn compile_time_vmap_sizes() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + // Try creating a normal vmap + obj.vmap::<PAGE_SIZE>()?; + + // Try creating a vmap that's smaller then the size we specified + let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?; + + // Verify the owner matches + assert!(ptr::eq(vmap.owner(), obj.deref())); + + // Verify the size matches the actual object size + assert_eq!(vmap.size(), PAGE_SIZE); + + // Make sure creating a vmap that's too large fails + assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err()); + + Ok(()) + } + + #[test] + fn vmap_io() -> Result { + let (_dev, drm) = create_drm_dev()?; + + let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + let vmap = obj.vmap::<PAGE_SIZE>()?; + + vmap.write8(0xDE, 0x0); + assert_eq!(vmap.read8(0x0), 0xDE); + vmap.write32(0xFEDCBA98, 0x20); + + assert_eq!(vmap.read32(0x20), 0xFEDCBA98); + + // Ensure the ordering in memory is correct + let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); + for (offset, expected) in (0x20..=0x23).zip(expected) { + assert_eq!(vmap.try_read8(offset).unwrap(), expected); + } + + Ok(()) + } + + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. + + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. + #[test] + fn fail_sg_table_on_wrong_dev() -> Result { + let (_dev, drm) = create_drm_dev()?; + let reg = faux::Registration::new(c"EvilKunit", None)?; + let wrong_dev = reg.as_ref(); + + let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); + + // TODO: Someday, we should test that creating an sg_table here still succeeds. + + Ok(()) + } +} diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs index ae58f6f667c1..d9d43d719761 100644 --- a/rust/kernel/drm/gpuvm/mod.rs +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -72,10 +72,12 @@ pub struct GpuVm<T: DriverGpuVm> { data: UnsafeCell<T>, } -// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will -// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. +// SAFETY: It is safe to send a `GpuVm<T>` to another thread: all data reachable through it +// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds. unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. +// SAFETY: It is safe to share a `&GpuVm<T>` between threads: `&self` methods only alias data +// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the +// reference and ultimately drop `T`, which the same bounds make `Send`. unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. @@ -116,9 +118,9 @@ impl<T: DriverGpuVm> GpuVm<T> { /// Creates a GPUVM instance. #[expect(clippy::new_ret_no_self)] - pub fn new<E>( + pub fn new<E, Ctx: drm::DeviceContext>( name: &'static CStr, - dev: &drm::Device<T::Driver>, + dev: &drm::Device<T::Driver, Ctx>, r_obj: &T::Object, range: Range<u64>, reserve_range: Range<u64>, @@ -250,21 +252,27 @@ impl<T: DriverGpuVm> GpuVm<T> { } /// The manager for a GPUVM. -pub trait DriverGpuVm: Sized + Send { +pub trait DriverGpuVm: Sized + Send + Sync { /// Parent `Driver` for this object. - type Driver: drm::Driver<Object = Self::Object>; + type Driver: drm::Driver; /// The kind of GEM object stored in this GPUVM. - type Object: IntoGEMObject; + type Object: drm::driver::AllocImpl<Driver = Self::Driver> + Send + Sync; /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). - type VaData; + /// + /// Only `Send` is required: the data has a single owner at all times, moving + /// between threads by value (handed back as a [`GpuVaRemoved`]) but never + /// accessed by two threads concurrently. + type VaData: Send; /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). - type VmBoData; + type VmBoData: Send + Sync; /// The private data passed to callbacks. - type SmContext<'ctx>; + type SmContext<'ctx> + where + Self: 'ctx; /// Indicates that a new mapping should be created. fn sm_step_map<'op, 'ctx>( @@ -296,12 +304,10 @@ pub trait DriverGpuVm: Sized + Send { /// # Invariants /// /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +// `Send`/`Sync` derive from `ARef<GpuVm<T>>`; the trait bounds make them correct for the unique +// handle's `&mut T` access. pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); -// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and -// concurrent access to `data` is safe due to the `T: Sync` requirement. -unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {} - impl<T: DriverGpuVm> UniqueRefGpuVm<T> { /// Access the data owned by this `UniqueRefGpuVm` immutably. #[inline] diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs index 69a8e5ab2821..742c151b2540 100644 --- a/rust/kernel/drm/gpuvm/sm_ops.rs +++ b/rust/kernel/drm/gpuvm/sm_ops.rs @@ -3,7 +3,7 @@ use super::*; /// The actual data that gets threaded through the callbacks. -struct SmData<'a, 'ctx, T: DriverGpuVm> { +struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> { gpuvm: &'a mut UniqueRefGpuVm<T>, user_context: &'a mut T::SmContext<'ctx>, } @@ -20,7 +20,7 @@ struct SmMapData<'a, 'ctx, T: DriverGpuVm> { } /// The argument for [`UniqueRefGpuVm::sm_map`]. -pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> { +pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> { /// Address in GPU virtual address space. pub addr: u64, /// Length of mapping to create. diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs index 0b09fe44ab39..46493f0ec5a6 100644 --- a/rust/kernel/drm/gpuvm/va.rs +++ b/rust/kernel/drm/gpuvm/va.rs @@ -104,6 +104,14 @@ impl<T: DriverGpuVm> GpuVa<T> { /// The memory is zeroed. pub struct GpuVaAlloc<T: DriverGpuVm>(KBox<MaybeUninit<GpuVa<T>>>); +// SAFETY: A `GpuVaAlloc` is an owned, uninitialised allocation with no live `T::VaData` and no +// thread-bound state. +unsafe impl<T: DriverGpuVm> Send for GpuVaAlloc<T> {} + +// SAFETY: A `GpuVaAlloc` has no `&self` method that reaches its contents, so a shared +// `&GpuVaAlloc` cannot access the allocation. +unsafe impl<T: DriverGpuVm> Sync for GpuVaAlloc<T> {} + impl<T: DriverGpuVm> GpuVaAlloc<T> { /// Pre-allocate a [`GpuVa`] object. pub fn new(flags: AllocFlags) -> Result<GpuVaAlloc<T>, AllocError> { @@ -116,7 +124,7 @@ impl<T: DriverGpuVm> GpuVaAlloc<T> { pub(super) fn prepare(mut self, va_data: impl PinInit<T::VaData>) -> *mut bindings::drm_gpuva { let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0); // SAFETY: The `data` field is pinned. - let Ok(()) = unsafe { va_data.__pinned_init(&raw mut (*va_ptr).data) }; + unsafe { pin_init::raw_init(&raw mut (*va_ptr).data, va_data) }; KBox::into_raw(self.0).cast() } } diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs index c064ac63897b..5989972da829 100644 --- a/rust/kernel/drm/gpuvm/vm_bo.rs +++ b/rust/kernel/drm/gpuvm/vm_bo.rs @@ -19,6 +19,15 @@ pub struct GpuVmBo<T: DriverGpuVm> { data: T::VmBoData, } +// SAFETY: It is safe to send a `GpuVmBo<T>` to another thread: dropping it there drops +// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds. +unsafe impl<T: DriverGpuVm> Send for GpuVmBo<T> {} + +// SAFETY: It is safe to share a `&GpuVmBo<T>` between threads: it effectively shares +// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an +// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds. +unsafe impl<T: DriverGpuVm> Sync for GpuVmBo<T> {} + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`. unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVmBo<T> { fn inc_ref(&self) { @@ -181,7 +190,7 @@ impl<T: DriverGpuVm> GpuVmBoAlloc<T> { }; let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; // SAFETY: `ptr->data` is a valid pinned location. - let Ok(()) = unsafe { value.__pinned_init(&raw mut (*raw_ptr).data) }; + unsafe { pin_init::raw_init(&raw mut (*raw_ptr).data, value) }; // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid // as we just initialized it. Ok(GpuVmBoAlloc(ptr)) diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index cf328101dde4..64af9eacc306 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -70,6 +70,18 @@ pub mod internal { pub use bindings::drm_device; pub use bindings::drm_file; pub use bindings::drm_ioctl_desc; + + /// Cast an [`Ioctl`] DRM device pointer to [`Registered`], preserving the driver type + /// parameter `T`. + /// + /// Used by [`declare_drm_ioctls!`] to anchor type inference. + #[doc(hidden)] + #[inline] + pub const fn __dev_ctx_cast<T: crate::drm::Driver>( + ptr: *const crate::drm::Device<T, crate::drm::Ioctl>, + ) -> *const crate::drm::Device<T, crate::drm::Registered> { + ptr.cast() + } } /// Declare the DRM ioctls for a driver. @@ -82,7 +94,8 @@ pub mod internal { /// `user_callback` should have the following prototype: /// /// ```ignore -/// fn foo(device: &kernel::drm::Device<Self>, +/// fn foo(device: &kernel::drm::Device<Self, kernel::drm::Registered>, +/// reg_data: &Self::RegistrationData<'_>, /// data: &mut uapi::argument_type, /// file: &kernel::drm::File<Self::File>, /// ) -> Result<u32> @@ -131,10 +144,45 @@ macro_rules! declare_drm_ioctls { // - The DRM device must have been registered when we're called through // an IOCTL. // + // INVARIANT: The `Ioctl` context requires that the device has been + // registered via `drm_dev_register()` at some point; the DRM core + // guarantees this for ioctl dispatch callbacks. + // // FIXME: Currently there is nothing enforcing that the types of the // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. - let dev = $crate::drm::device::Device::from_raw(raw_dev); + let dev: &$crate::drm::device::Device<_, $crate::drm::Ioctl> = + $crate::drm::device::Device::from_raw(raw_dev); + + // Type-inference anchor: the closure is never called but ties `dev`'s + // type to `$func`'s first parameter, which the compiler cannot infer + // through method resolution and associated-type projections alone. + #[allow(unreachable_code)] + let _ = || { + let __ptr = $crate::drm::ioctl::internal::__dev_ctx_cast( + ::core::ptr::from_ref(dev), + ); + + $func( + // SAFETY: This closure is never executed; the dereference + // exists purely to unify the type parameter with `$func`. + // The pointer is valid regardless. + unsafe { &*__ptr }, + unreachable!(), + unreachable!(), + unreachable!(), + ) + }; + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a _, &'a mut _, &'a _) -> _ = $func; + + let Some(guard) = dev.registration_guard() else { + return $crate::error::code::ENODEV.to_errno(); + }; + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. @@ -147,7 +195,9 @@ macro_rules! declare_drm_ioctls { // SAFETY: This is just the DRM file structure let file = unsafe { $crate::drm::File::from_raw(raw_file) }; - match $func(dev, data, file) { + match guard.registration_data_with(|reg_data| { + $func(&*guard, reg_data, data, file) + }) { Err(e) => e.to_errno(), Ok(i) => i.try_into() .unwrap_or($crate::error::code::ERANGE.to_errno()), diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index a66e7166f66b..fd6ed35bc35a 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -11,8 +11,10 @@ pub mod ioctl; pub use self::device::Device; pub use self::device::DeviceContext; +pub use self::device::Ioctl; +pub use self::device::Normal; pub use self::device::Registered; -pub use self::device::Uninit; +pub use self::device::RegistrationGuard; pub use self::device::UnregisteredDevice; pub use self::driver::Driver; pub use self::driver::DriverInfo; diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index a56ba6309594..e52793f77196 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -30,6 +30,7 @@ pub mod code { }; } + // From `include/uapi/asm-generic/errno-base.h`. declare_err!(EPERM, "Operation not permitted."); declare_err!(ENOENT, "No such file or directory."); declare_err!(ESRCH, "No such process."); @@ -64,9 +65,110 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + + // From `include/uapi/asm-generic/errno.h`. + declare_err!(EDEADLK, "Resource deadlock would occur."); + declare_err!(ENAMETOOLONG, "File name too long."); + declare_err!(ENOLCK, "No record locks available."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENOTEMPTY, "Directory not empty."); + declare_err!(ELOOP, "Too many symbolic links encountered."); + declare_err!(ENOMSG, "No message of desired type."); + declare_err!(EIDRM, "Identifier removed."); + declare_err!(ECHRNG, "Channel number out of range."); + declare_err!(EL2NSYNC, "Level 2 not synchronized."); + declare_err!(EL3HLT, "Level 3 halted."); + declare_err!(EL3RST, "Level 3 reset."); + declare_err!(ELNRNG, "Link number out of range."); + declare_err!(EUNATCH, "Protocol driver not attached."); + declare_err!(ENOCSI, "No CSI structure available."); + declare_err!(EL2HLT, "Level 2 halted."); + declare_err!(EBADE, "Invalid exchange."); + declare_err!(EBADR, "Invalid request descriptor."); + declare_err!(EXFULL, "Exchange full."); + declare_err!(ENOANO, "No anode."); + declare_err!(EBADRQC, "Invalid request code."); + declare_err!(EBADSLT, "Invalid slot."); + declare_err!(EBFONT, "Bad font file format."); + declare_err!(ENOSTR, "Device not a stream."); + declare_err!(ENODATA, "No data available."); + declare_err!(ETIME, "Timer expired."); + declare_err!(ENOSR, "Out of streams resources."); + declare_err!(ENONET, "Machine is not on the network."); + declare_err!(ENOPKG, "Package not installed."); + declare_err!(EREMOTE, "Object is remote."); + declare_err!(ENOLINK, "Link has been severed."); + declare_err!(EADV, "Advertise error."); + declare_err!(ESRMNT, "Srmount error."); + declare_err!(ECOMM, "Communication error on send."); + declare_err!(EPROTO, "Protocol error."); + declare_err!(EMULTIHOP, "Multihop attempted."); + declare_err!(EDOTDOT, "RFS specific error."); + declare_err!(EBADMSG, "Not a data message."); + declare_err!(EFSBADCRC, "Bad CRC detected."); declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ENOTUNIQ, "Name not unique on network."); + declare_err!(EBADFD, "File descriptor in bad state."); + declare_err!(EREMCHG, "Remote address changed."); + declare_err!(ELIBACC, "Can not access a needed shared library."); + declare_err!(ELIBBAD, "Accessing a corrupted shared library."); + declare_err!(ELIBSCN, ".lib section in a.out corrupted."); + declare_err!(ELIBMAX, "Attempting to link in too many shared libraries."); + declare_err!(ELIBEXEC, "Cannot exec a shared library directly."); + declare_err!(EILSEQ, "Illegal byte sequence."); + declare_err!(ERESTART, "Interrupted system call should be restarted."); + declare_err!(ESTRPIPE, "Streams pipe error."); + declare_err!(EUSERS, "Too many users."); + declare_err!(ENOTSOCK, "Socket operation on non-socket."); + declare_err!(EDESTADDRREQ, "Destination address required."); declare_err!(EMSGSIZE, "Message too long."); + declare_err!(EPROTOTYPE, "Protocol wrong type for socket."); + declare_err!(ENOPROTOOPT, "Protocol not available."); + declare_err!(EPROTONOSUPPORT, "Protocol not supported."); + declare_err!(ESOCKTNOSUPPORT, "Socket type not supported."); + declare_err!(EOPNOTSUPP, "Operation not supported on transport endpoint."); + declare_err!(EPFNOSUPPORT, "Protocol family not supported."); + declare_err!(EAFNOSUPPORT, "Address family not supported by protocol."); + declare_err!(EADDRINUSE, "Address already in use."); + declare_err!(EADDRNOTAVAIL, "Cannot assign requested address."); + declare_err!(ENETDOWN, "Network is down."); + declare_err!(ENETUNREACH, "Network is unreachable."); + declare_err!(ENETRESET, "Network dropped connection because of reset."); + declare_err!(ECONNABORTED, "Software caused connection abort."); + declare_err!(ECONNRESET, "Connection reset by peer."); + declare_err!(ENOBUFS, "No buffer space available."); + declare_err!(EISCONN, "Transport endpoint is already connected."); + declare_err!(ENOTCONN, "Transport endpoint is not connected."); + declare_err!(ESHUTDOWN, "Cannot send after transport endpoint shutdown."); + declare_err!(ETOOMANYREFS, "Too many references: cannot splice."); declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECONNREFUSED, "Connection refused."); + declare_err!(EHOSTDOWN, "Host is down."); + declare_err!(EHOSTUNREACH, "No route to host."); + declare_err!(EALREADY, "Operation already in progress."); + declare_err!(EINPROGRESS, "Operation now in progress."); + declare_err!(ESTALE, "Stale file handle."); + declare_err!(EUCLEAN, "Structure needs cleaning."); + declare_err!(EFSCORRUPTED, "Filesystem is corrupted."); + declare_err!(ENOTNAM, "Not a XENIX named type file."); + declare_err!(ENAVAIL, "No XENIX semaphores available."); + declare_err!(EISNAM, "Is a named type file."); + declare_err!(EREMOTEIO, "Remote I/O error."); + declare_err!(EDQUOT, "Quota exceeded."); + declare_err!(ENOMEDIUM, "No medium found."); + declare_err!(EMEDIUMTYPE, "Wrong medium type."); + declare_err!(ECANCELED, "Operation Canceled."); + declare_err!(ENOKEY, "Required key not available."); + declare_err!(EKEYEXPIRED, "Key has expired."); + declare_err!(EKEYREVOKED, "Key has been revoked."); + declare_err!(EKEYREJECTED, "Key was rejected by service."); + declare_err!(EOWNERDEAD, "Owner died."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); + declare_err!(ERFKILL, "Operation not possible due to RF-kill."); + declare_err!(EHWPOISON, "Memory page has hardware error."); + declare_err!(EFTYPE, "Wrong file type for the intended operation."); + + // From `include/linux/errno.h`. declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs index 43b4974f48cd..cd4198fbb232 100644 --- a/rust/kernel/faux.rs +++ b/rust/kernel/faux.rs @@ -9,15 +9,63 @@ use crate::{ bindings, device, - prelude::*, // + prelude::*, + types::Opaque, // }; -use core::ptr::{ - addr_of_mut, - null, - null_mut, - NonNull, // +use core::{ + marker::PhantomData, + ptr::{ + null, + null_mut, + NonNull, // + }, }; +/// A faux device. +/// +/// A faux device is a virtual device backed by the faux bus, primarily used for scenarios where a +/// real hardware device is not available or for testing. +/// +/// # Invariants +/// +/// The underlying `struct faux_device` is valid. +#[repr(transparent)] +pub struct Device<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::faux_device>, + PhantomData<Ctx>, +); + +impl<Ctx: device::DeviceContext> Device<Ctx> { + #[inline] + fn as_raw(&self) -> *mut bindings::faux_device { + self.0.get() + } + + /// # Safety + /// + /// `ptr` must be a valid pointer to a `struct faux_device`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::faux_device) -> &'a Self { + // SAFETY: `Device` is a transparent wrapper of `Opaque<bindings::faux_device>`. + unsafe { &*ptr.cast() } + } +} + +impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { + #[inline] + fn as_ref(&self) -> &device::Device<Ctx> { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct faux_device`. `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) } + } +} + +// SAFETY: `faux::Device` is a transparent wrapper of `struct faux_device`. +// The offset is guaranteed to point to a valid device field inside `faux::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = core::mem::offset_of!(bindings::faux_device, dev); +} + /// The registration of a faux device. /// /// This type represents the registration of a [`struct faux_device`]. When an instance of this type @@ -25,7 +73,8 @@ use core::ptr::{ /// /// # Invariants /// -/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`]. +/// - This object is proof that the object described by this `Registration` is bound to a device. /// /// [`struct faux_device`]: srctree/include/linux/device/faux.h pub struct Registration(NonNull<bindings::faux_device>); @@ -59,11 +108,19 @@ impl Registration { } } -impl AsRef<device::Device> for Registration { - fn as_ref(&self) -> &device::Device { - // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be - // a valid initialized `device`. - unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) } +impl AsRef<Device<device::Bound>> for Registration { + #[inline] + fn as_ref(&self) -> &Device<device::Bound> { + // SAFETY: + // - The underlying `struct faux_device` is guaranteed by the C API to be a valid + // initialized `device`. + // - `faux_match()` always returns 1, and probe runs synchronously + // (PROBE_FORCE_SYNCHRONOUS). + // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via + // sysfs. + // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux + // device stays bound forever. + unsafe { Device::from_raw(self.as_raw()) } } } diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 71168d8004e2..a18f8b84f3e3 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -7,9 +7,9 @@ use crate::{ bindings, device::Device, - error::Error, - error::Result, + error::to_result, ffi, + prelude::*, str::{CStr, CStrExt as _}, }; use core::ptr::NonNull; @@ -51,12 +51,8 @@ impl FwFunc { /// # Examples /// /// ```no_run -/// # use kernel::{device::Device, firmware::Firmware}; -/// -/// # fn no_run() -> Result<(), Error> { -/// # // SAFETY: *NOT* safe, just for the example to get an `ARef<Device>` instance -/// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) }; -/// +/// # use kernel::{device::Device, firmware::Firmware, sync::aref::ARef}; +/// # fn no_run(dev: ARef<Device>) -> Result<(), Error> { /// let fw = Firmware::request(c"path/to/firmware.bin", &dev)?; /// let blob = fw.data(); /// @@ -120,6 +116,48 @@ impl Drop for Firmware { } } +/// Load firmware directly into the caller-provided `buf`. +/// +/// On success the firmware image has been copied into `buf`; the caller accesses the data +/// through `buf` itself. +/// +/// This is intentionally a stand-alone function rather than a `Firmware` constructor. For +/// the `into_buf` path, the firmware data lives in the caller's `buf`, not in a +/// kernel-owned buffer, so returning a `Firmware` would expose `Firmware::data()` as a +/// second handle aliasing `buf` (and `release_firmware()` does not free `buf` anyway). +pub fn request_into_buf(name: &CStr, dev: &Device, buf: &mut [u8]) -> Result { + // `as_mut_ptr()` on an empty slice returns a non-NULL pointer to + // memory which the loader does not own. Passing that pointer with `size == 0` + // makes the loader believe that it is buffer it allocated itself, so when + // `release_firmware()` is called, it will vfree the pointer and trigger a + // bug. Reject empty slices to avoid this situation. + if buf.is_empty() { + return Err(EINVAL); + } + + let mut fw: *const bindings::firmware = core::ptr::null(); + + // SAFETY: `&raw mut fw` is a valid pointer to a NULL initialized `bindings::firmware` pointer. + // `name` and `dev` are valid as by their type invariants. `buf` is a valid writable + // buffer of `buf.len()` bytes. + to_result(unsafe { + bindings::request_firmware_into_buf( + &raw mut fw, + name.as_char_ptr(), + dev.as_raw(), + buf.as_mut_ptr().cast(), + buf.len(), + ) + })?; + + // The firmware bytes are now in `buf`, which the caller owns, so we don't need + // the kernel to hang on to it any more. + // SAFETY: `fw` is a valid pointer returned by `request_firmware_into_buf`. + unsafe { bindings::release_firmware(fw) }; + + Ok(()) +} + // SAFETY: `Firmware` only holds a pointer to a C `struct firmware`, which is safe to be used from // any thread. unsafe impl Send for Firmware {} diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs index 73afbc51ba33..29582b053ab1 100644 --- a/rust/kernel/fmt.rs +++ b/rust/kernel/fmt.rs @@ -4,6 +4,8 @@ //! //! This module is intended to be used in place of `core::fmt` in kernel code. +use kernel::prelude::*; + pub use core::fmt::{ Arguments, Debug, @@ -39,11 +41,115 @@ use core::fmt::{ LowerExp, LowerHex, Octal, - Pointer, UpperExp, UpperHex, // }; -impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp); +use core::ptr::NonNull; +impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, LowerExp, UpperExp); + +/// A copy of [`core::fmt::Pointer`] that allows implementing pointer formatting for foreign types. +/// +/// Together with the [`Adapter`] type and [`fmt!`] macro, it enables raw pointer formatting to be +/// intercepted and routed to [`HashedPtr`] (kernel's `%p` hashed format), preventing kernel address +/// leaks. +/// +/// [`fmt!`]: crate::prelude::fmt! +pub trait Pointer { + /// Same as [`core::fmt::Pointer::fmt`]. + fn fmt(&self, f: &mut Formatter<'_>) -> Result; +} + +/// A wrapper for pointers that formats them using kernel's `%p` format specifier. +/// +/// By default, `%p` prints a hashed representation of the pointer address to prevent kernel address +/// leaks. When the `no_hash_pointers` kernel command-line parameter is enabled, the real address is +/// printed instead (for debugging purposes). +pub struct HashedPtr<T: ?Sized>(pub *const T); + +impl<T: ?Sized> Pointer for HashedPtr<T> { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + use crate::str::CStrExt as _; + + let mut buf = [0u8; 32]; + + // Use `%#0*p` for the `0x` prefix and zero-padding; `+2` compensates for + // the prefix counting toward the field width. + let default_width = (2 * size_of::<usize>() + 2) as c_int; + let width = match (f.sign_aware_zero_pad(), f.width()) { + (true, Some(w)) if w > 0 => w.min(buf.len() - 1) as c_int, + _ => default_width, + }; + + // SAFETY: `buf` is a valid, writable 32-byte buffer, sufficient for + // all architectures (max 19 bytes for 64-bit under the default width). + // The format string is null-terminated; `width` (c_int) and pointer + // match the `%*` and `%p` specifiers. + let len = unsafe { + crate::bindings::scnprintf( + buf.as_mut_ptr().cast(), + buf.len(), + c"%#0*p".as_char_ptr(), + width, + self.0.cast::<c_void>(), + ) + }; + + // SAFETY: `%#0*p` produces only ASCII, which is valid UTF-8. + let s = unsafe { core::str::from_utf8_unchecked(&buf[..len as usize]) }; + + if f.sign_aware_zero_pad() { + // `scnprintf` already applied the width and zero-padding via `%#0*p`. + f.write_str(s) + } else { + f.pad(s) + } + } +} + +// Raw pointers are formatted via `HashedPtr` (kernel `%p`: hashed by default, plain with +// `no_hash_pointers`). +impl<T: ?Sized> Pointer for *const T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for *mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for &T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(*self), f) + } +} + +impl<T: ?Sized> Pointer for &mut T { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(core::ptr::from_ref(*self)), f) + } +} + +impl<T: ?Sized> Pointer for NonNull<T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(&HashedPtr(self.as_ptr()), f) + } +} + +// `Adapter<&T>` bridges our `Pointer` trait to `core::fmt::Pointer` +impl<T: Pointer> core::fmt::Pointer for Adapter<&T> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + Pointer::fmt(self.0, f) + } +} /// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types. /// @@ -105,3 +211,88 @@ impl_display_forward!( {<T: ?Sized>} crate::sync::Arc<T> {where crate::sync::Arc<T>: core::fmt::Display}, {<T: ?Sized>} crate::sync::UniqueArc<T> {where crate::sync::UniqueArc<T>: core::fmt::Display}, ); + +#[macros::kunit_tests(rust_kernel_fmt)] +mod tests { + use crate::{ + bindings, + prelude::fmt, + str::CString, // + }; + + #[cfg(CONFIG_64BIT)] + mod expected { + pub(super) const PTR_VALUE: usize = 0xffffffffdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(____ptrval____)"; + pub(super) const HASHED_PREFIX: &str = "0x00000000"; + pub(super) const RAW_POINTER: &str = "0xffffffffdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xffffffffdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x000000ffffffffdeadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000ffffffffdeadbeef"; + } + + #[cfg(not(CONFIG_64BIT))] + mod expected { + pub(super) const PTR_VALUE: usize = 0xdeadbeef; + pub(super) const PTR_VAL_NO_CRNG: &str = "(ptrval)"; + pub(super) const HASHED_PREFIX: &str = "0x"; + pub(super) const RAW_POINTER: &str = "0xdeadbeef"; + pub(super) const PADDED_RIGHT: &str = " 0xdeadbeef"; + pub(super) const ZERO_PADDED: &str = "0x00000000000000deadbeef"; + pub(super) const HASHED_PADDED_RIGHT_PREFIX: &str = " "; + pub(super) const HASHED_ZERO_PADDED_PREFIX: &str = "0x00000000000000"; + pub(super) const CLAMPED: &str = "0x0000000000000000000000deadbeef"; + } + + #[test] + fn test_ptr_formatting() -> core::result::Result<(), crate::error::Error> { + let ptr: *const u8 = core::ptr::without_provenance(expected::PTR_VALUE); + + // SAFETY: `no_hash_pointers` is a global variable that is never concurrently modified — + // KUnit tests may run at boot (before `mark_readonly()`) or manually afterwards (when the + // variable is read-only). Reading is always safe. + let no_hash = unsafe { bindings::no_hash_pointers }; + + if no_hash { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::PADDED_RIGHT); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::ZERO_PADDED); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + assert_eq!(cstr.to_str()?, expected::CLAMPED); + } else { + let cstr = CString::try_from_fmt(fmt!("{:p}", ptr))?; + let formatted = cstr.to_str()?; + // If the RNG is not yet ready, `%p` falls back to a placeholder. + if formatted == expected::PTR_VAL_NO_CRNG { + return Ok(()); + } + assert!(formatted.starts_with(expected::HASHED_PREFIX)); + assert_ne!(formatted, expected::RAW_POINTER); + + let cstr = CString::try_from_fmt(fmt!("{:>24p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_PADDED_RIGHT_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:024p}", ptr))?; + assert!(cstr + .to_str()? + .starts_with(expected::HASHED_ZERO_PADDED_PREFIX)); + + let cstr = CString::try_from_fmt(fmt!("{:0100p}", ptr))?; + let output = cstr.to_str()?; + assert!(output.starts_with("0x")); + assert!(!output[2..].chars().all(|c| c == '0')); + } + + Ok(()) + } +} diff --git a/rust/kernel/fwctl.rs b/rust/kernel/fwctl.rs new file mode 100644 index 000000000000..f29244fb0d1d --- /dev/null +++ b/rust/kernel/fwctl.rs @@ -0,0 +1,593 @@ +// SPDX-License-Identifier: GPL-2.0-only + +//! Abstractions for the fwctl subsystem. +//! +//! C header: `include/linux/fwctl.h` + +use crate::{ + bindings, + container_of, + device, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + cell::UnsafeCell, + marker::PhantomData, + ptr::NonNull, + slice, // +}; + +/// Returns a kmalloc-compatible allocation size for `T`. +const fn kmalloc_aligned_size<T>() -> usize { + Layout::new::<T>().pad_to_align().size() +} + +/// Represents a fwctl device type. +/// +/// Corresponds to the C `enum fwctl_device_type`. All non-error UAPI values are represented so +/// Rust drivers can select a device type without passing an untyped integer, while +/// `FWCTL_DEVICE_TYPE_ERROR` remains unrepresentable. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum DeviceType { + /// Mellanox ConnectX (mlx5) device. + Mlx5 = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_MLX5, + /// CXL (Compute Express Link) device. + Cxl = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_CXL, + /// AMD/Pensando PDS device. + Pds = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_PDS, + /// Broadcom NetXtreme (bnxt) device. + Bnxt = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_BNXT, +} + +/// Scope of access for an RPC request. +/// +/// Corresponds to the C `enum fwctl_rpc_scope`. +#[repr(u32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum RpcScope { + /// Read/write access to device configuration. + Configuration = bindings::fwctl_rpc_scope_FWCTL_RPC_CONFIGURATION, + /// Read-only access to debug information. + DebugReadOnly = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_READ_ONLY, + /// Write access to lockdown-compatible debug information. + DebugWrite = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE, + /// Full read/write access to all debug information (requires `CAP_SYS_RAWIO`). + DebugWriteFull = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE_FULL, +} + +impl TryFrom<u32> for RpcScope { + type Error = Error; + + #[inline] + fn try_from(value: u32) -> Result<Self, Error> { + match value { + v if v == Self::Configuration as u32 => Ok(Self::Configuration), + v if v == Self::DebugReadOnly as u32 => Ok(Self::DebugReadOnly), + v if v == Self::DebugWrite as u32 => Ok(Self::DebugWrite), + v if v == Self::DebugWriteFull as u32 => Ok(Self::DebugWriteFull), + _ => Err(EINVAL), + } + } +} + +/// Response from a [`Operations::fw_rpc`] call. +pub enum FwRpcResponse { + /// Reuse the input buffer as the output, with the given output length. + /// + /// The callback returns `EINVAL` if the output length exceeds the input buffer length. + InPlace(usize), + /// Return a newly allocated buffer as the output. + NewBuffer(KVVec<u8>), +} + +/// Trait implemented by each Rust driver that integrates with the fwctl subsystem. +/// +/// The implementing type **is** the per-FD user context: one instance is +/// created for each `open()` call and dropped when the FD is closed. +/// +/// Each implementation corresponds to a specific device type and provides the +/// vtable used by the core `fwctl` layer to manage per-FD user contexts and +/// handle RPC requests. +pub trait Operations: Sized + Send + Sync + 'static { + /// Data owned by the [`Registration`] and accessible during callbacks. + /// + /// The lifetime `'a` is tied to the [`Registration`] scope (which lives within the parent bus + /// device binding scope). Drivers use it to store references to resources bound to this scope, + /// such as PCI BARs or typed bus device references. + type RegistrationData<'a>: Send + Sync + 'a + where + Self: 'a; + + /// fwctl device type identifier. + const DEVICE_TYPE: DeviceType; + + /// Called when a new user context is opened. + /// + /// Returns a [`PinInit`] initializer for `Self`. The instance is dropped + /// automatically when the FD is closed (after [`close`](Self::close)). + fn open<'a>( + device: &Device<Self>, + reg_data: &Self::RegistrationData<'a>, + ) -> impl PinInit<Self, Error>; + + /// Called when the user context is closed. + /// + /// The driver may perform additional cleanup here that requires access + /// to the owning [`Device`]. `Self` is dropped automatically after this + /// returns. + fn close<'a>( + _this: Pin<&mut Self>, + _device: &Device<Self>, + _reg_data: &Self::RegistrationData<'a>, + ) { + } + + /// Return device information to userspace. + /// + /// The default implementation returns no device-specific data. + fn info<'a>( + _this: Pin<&Self>, + _device: &Device<Self>, + _reg_data: &Self::RegistrationData<'a>, + ) -> Result<KVec<u8>, Error> { + Ok(KVec::new()) + } + + /// Handle a userspace RPC request. + /// + /// `max_output_len` is the size of the userspace output buffer. A driver may return a larger + /// response to report the required size; the fwctl core copies only the bytes that fit and + /// reports the full response length to userspace. + fn fw_rpc<'a>( + this: Pin<&Self>, + device: &Device<Self>, + reg_data: &Self::RegistrationData<'a>, + scope: RpcScope, + rpc_buf: &mut [u8], + max_output_len: usize, + ) -> Result<FwRpcResponse, Error>; +} + +/// A fwctl device. +/// +/// `#[repr(C)]` with the `fwctl_device` at offset 0, matching the C `fwctl_alloc_device()` layout +/// convention. Contains a pointer to the [`Registration`]'s data, set at registration time and +/// cleared on unregistration. +/// +/// # Invariants +/// +/// - `dev` is embedded at offset 0 and is initialised by fwctl. +/// - The fwctl refcount owns the allocation lifetime. +/// - `registration_data` is either [`NonNull::dangling()`] (before registration / after +/// unregistration) or points to valid data owned by the [`Registration`]. +#[repr(C)] +pub struct Device<T: Operations> { + dev: Opaque<bindings::fwctl_device>, + registration_data: UnsafeCell<NonNull<T::RegistrationData<'static>>>, +} + +impl<T: Operations> Device<T> { + /// Allocate a new fwctl device. + /// + /// Returns an [`ARef`] that can be passed to [`Registration::new()`] + /// to make the device visible to userspace. + pub fn new(parent: &device::Device<device::Bound>) -> Result<ARef<Self>> { + const_assert!( + core::mem::offset_of!(Self, dev) == 0, + "struct fwctl_device must be at offset 0" + ); + + let size = kmalloc_aligned_size::<Self>(); + let ops = core::ptr::from_ref::<bindings::fwctl_ops>(&VTable::<T>::VTABLE).cast_mut(); + + // SAFETY: `ops` is static, `parent` is bound, and `size` is padded so the allocation made + // by `_fwctl_alloc_device` satisfies the size and alignment required by `Device<T>`. + let raw = unsafe { bindings::_fwctl_alloc_device(parent.as_raw(), ops, size) }; + let this = NonNull::new(raw.cast::<Self>()).ok_or(ENOMEM)?; + + // INVARIANT: Set `registration_data` to dangling (no registration yet). + // SAFETY: `this` points to the allocation just returned by fwctl. + unsafe { + (&raw mut (*this.as_ptr()).registration_data) + .write(UnsafeCell::new(NonNull::dangling())); + }; + + // SAFETY: `this` owns the initial reference. + Ok(unsafe { ARef::from_raw(this) }) + } + + /// Returns the underlying `fwctl_device` pointer. + #[inline] + fn as_raw(&self) -> *mut bindings::fwctl_device { + self.dev.get() + } + + /// Borrows a Rust fwctl device from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a valid `fwctl_device` embedded in a [`Device<T>`]. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_device) -> &'a Self { + // SAFETY: The caller upholds the offset-0 `Device<T>` invariant. + unsafe { &*ptr.cast() } + } + + /// Invokes `f` with the registration data. + /// + /// The higher-ranked callback prevents the erased registration lifetime from escaping and + /// permits registration data that is invariant over its lifetime parameter. + /// + /// # Safety + /// + /// The caller must ensure that the device is registered and that this is called from a fwctl + /// callback protected by `registration_lock`. + #[inline] + unsafe fn with_registration_data<R>( + &self, + f: impl for<'a> FnOnce(&Device<T>, &'a T::RegistrationData<'a>) -> R, + ) -> R { + // SAFETY: Caller guarantees the device is registered, so the pointer is valid. + // Lifetimes do not affect layout. The higher-ranked callback prevents the shortened + // lifetime from escaping or being selected by the caller. + let reg_data = unsafe { + (*self.registration_data.get()) + .cast::<T::RegistrationData<'_>>() + .as_ref() + }; + + f(self, reg_data) + } +} + +impl<T: Operations> AsRef<device::Device> for Device<T> { + #[inline] + fn as_ref(&self) -> &device::Device { + // SAFETY: `self` contains a live fwctl_device. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + // SAFETY: The embedded device is initialised by fwctl. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: `fwctl_get` increments the refcount of a valid fwctl_device. +// `fwctl_put` decrements it and frees the device when it reaches zero. +unsafe impl<T: Operations> AlwaysRefCounted for Device<T> { + #[inline] + fn inc_ref(&self) { + // SAFETY: `self` holds a live reference. + unsafe { bindings::fwctl_get(self.as_raw()) }; + } + + #[inline] + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The caller owns a live reference. + unsafe { bindings::fwctl_put(obj.cast().as_ptr()) }; + } +} + +// SAFETY: `Device<T>` is refcounted by the fwctl core and may be released from any thread. +unsafe impl<T: Operations> Send for Device<T> {} + +// SAFETY: Shared access to the embedded `fwctl_device` is protected by the fwctl core. The +// `registration_data` field is only mutated before registration and after unregistration (both +// single-threaded with respect to callbacks). +unsafe impl<T: Operations> Sync for Device<T> {} + +/// A registered fwctl device. +/// +/// Owns the [`RegistrationData`](Operations::RegistrationData) made available to driver callbacks. +/// The parent device lifetime ensures that [`fwctl_unregister`] runs before the parent driver +/// unbinds. +/// +/// On drop the device is unregistered (all user contexts are closed and `ops` is set to `NULL`) +/// and the registration data is dropped. +/// +/// [`fwctl_unregister`]: srctree/drivers/fwctl/main.c +pub struct Registration<'a, T: Operations> { + dev: ARef<Device<T>>, + _reg_data: Pin<KBox<T::RegistrationData<'a>>>, +} + +impl<'a, T: Operations> Registration<'a, T> { + /// Register a previously allocated fwctl device with the given registration data. + /// + /// The `reg_data` is owned by the registration and accessible during callbacks. + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running, since `fwctl_unregister` must be called before the + /// parent device is unbound. + /// + /// `dev` must be an unregistered [`Device`] that is not associated with any live + /// [`Registration`], and no other thread may attempt to register the same device concurrently. + pub unsafe fn new( + parent: &'a device::Device<device::Bound>, + dev: &Device<T>, + reg_data: impl PinInit<T::RegistrationData<'a>, Error>, + ) -> Result<Self> { + let actual_parent = dev.as_ref().parent().ok_or(EINVAL)?; + let parent_device: &device::Device = parent; + if !core::ptr::eq(actual_parent, parent_device) { + return Err(EINVAL); + } + + let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?; + + // Store the registration data pointer in the device before registration, so that it is + // visible once callbacks can be invoked. The `'static` type is only an erased storage + // handle; callbacks access the pointer through a higher-ranked closure. + let ptr: NonNull<T::RegistrationData<'static>> = + NonNull::from(Pin::get_ref(reg_data.as_ref())).cast(); + + // SAFETY: No concurrent access; the device is not yet registered. + unsafe { *dev.registration_data.get() = ptr }; + + // SAFETY: `dev` is a valid fwctl_device backed by an ARef. + let ret = unsafe { bindings::fwctl_register(dev.as_raw()) }; + if ret != 0 { + // SAFETY: No concurrent readers; registration failed. + unsafe { *dev.registration_data.get() = NonNull::dangling() }; + return Err(Error::from_errno(ret)); + } + + Ok(Self { + dev: dev.into(), + _reg_data: reg_data, + }) + } +} + +impl<T: Operations> Drop for Registration<'_, T> { + fn drop(&mut self) { + // SAFETY: The Registration lifetime guarantees that the parent device is still bound. + // `fwctl_unregister` takes the write lock, closes all user contexts, and sets ops=NULL. + // After it returns, no callbacks can be running or will run. + unsafe { bindings::fwctl_unregister(self.dev.as_raw()) }; + + // SAFETY: `fwctl_unregister` guarantees no concurrent readers. + unsafe { *self.dev.registration_data.get() = NonNull::dangling() }; + + // `self._reg_data` is dropped here, after callbacks have stopped. + } +} + +/// Internal per-FD user context wrapping `struct fwctl_uctx` and `T`. +/// +/// Not exposed to drivers; they work with `&T` / `Pin<&mut T>` directly. +#[repr(C)] +#[pin_data] +struct UserCtx<T: Operations> { + #[pin] + fwctl_uctx: Opaque<bindings::fwctl_uctx>, + #[pin] + uctx: T, +} + +impl<T: Operations> UserCtx<T> { + /// Borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx<T>` that remains + /// valid and does not move for the duration of `'a`. + #[inline] + unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a Self> { + // SAFETY: The caller upholds the `UserCtx<T>` embedding, lifetime, and pinning invariants. + unsafe { Pin::new_unchecked(&*container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx)) } + } + + /// Mutably borrows a pinned Rust user context from its raw C pointer. + /// + /// # Safety + /// + /// - `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx<T>` that remains + /// valid and does not move for the duration of `'a`. + /// - The caller must ensure exclusive access to the `UserCtx<T>` for the duration of `'a`. + #[inline] + unsafe fn from_raw_mut<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a mut Self> { + // SAFETY: The caller upholds the embedding, lifetime, pinning, and exclusivity invariants. + unsafe { + Pin::new_unchecked( + &mut *container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx).cast_mut(), + ) + } + } + + /// Returns a reference to the fwctl [`Device`] that owns this context. + #[inline] + fn device(self: Pin<&Self>) -> &Device<T> { + // SAFETY: fwctl initialises this pointer before any driver callback. + let raw_fwctl = unsafe { (*self.fwctl_uctx.get()).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device<T>` layout. + unsafe { Device::from_raw(raw_fwctl) } + } + + /// Returns a pinned reference to the driver context. + #[inline] + fn uctx(self: Pin<&Self>) -> Pin<&T> { + ::pin_init::assert_pinned!(UserCtx<T>, uctx, T, inline); + + // SAFETY: `uctx` is structurally pinned. + unsafe { self.map_unchecked(|ctx| &ctx.uctx) } + } +} + +/// Static vtable mapping Rust trait methods to C callbacks. +struct VTable<T: Operations>(PhantomData<T>); + +impl<T: Operations> VTable<T> { + /// The fwctl operations vtable for this driver type. + const VTABLE: bindings::fwctl_ops = bindings::fwctl_ops { + // CAST: `DeviceType` has the same `u32` representation as the C enum field. + device_type: T::DEVICE_TYPE as u32, + uctx_size: kmalloc_aligned_size::<UserCtx<T>>(), + open_uctx: Some(Self::open_uctx_callback), + close_uctx: Some(Self::close_uctx_callback), + info: Some(Self::info_callback), + fw_rpc: Some(Self::fw_rpc_callback), + }; + + /// Initialises a newly opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must be a valid `fwctl_uctx` embedded in a `UserCtx<T>` with + /// sufficient allocated space for the uctx field. + unsafe extern "C" fn open_uctx_callback(uctx: *mut bindings::fwctl_uctx) -> ffi::c_int { + const_assert!( + core::mem::offset_of!(UserCtx<T>, fwctl_uctx) == 0, + "struct fwctl_uctx must be at offset 0" + ); + + // SAFETY: fwctl sets this pointer before calling `open_uctx`. + let raw_fwctl = unsafe { (*uctx).fwctl }; + // SAFETY: Rust fwctl devices use the offset-0 `Device<T>` layout. + let device = unsafe { Device::<T>::from_raw(raw_fwctl) }; + + let uctx_offset = core::mem::offset_of!(UserCtx<T>, uctx); + // SAFETY: `uctx_size` reserves space for the full `UserCtx<T>`. + let uctx_ptr: *mut T = unsafe { uctx.byte_add(uctx_offset).cast() }; + + // SAFETY: `open_uctx` is called under `registration_lock` read, so the device is + // registered. `uctx_ptr` addresses the uninitialised pinned context reserved by + // `uctx_size`. + unsafe { + device.with_registration_data(|device, reg_data| { + match pin_init::raw_try_init(uctx_ptr, T::open(device, reg_data)) { + Ok(()) => 0, + Err(e) => e.to_errno(), + } + }) + } + } + + /// Closes and drops an opened Rust user context. + /// + /// # Safety + /// + /// `uctx` must point to a fully initialised `UserCtx<T>`. + unsafe extern "C" fn close_uctx_callback(uctx: *mut bindings::fwctl_uctx) { + // SAFETY: fwctl keeps the owning device live for this callback. + let device = unsafe { Device::<T>::from_raw((*uctx).fwctl) }; + + // SAFETY: close is called for an opened Rust user context. + let mut ctx = unsafe { UserCtx::<T>::from_raw_mut(uctx) }; + + // SAFETY: `close_uctx` is called under `registration_lock` write (from + // `fwctl_unregister`) or read (from `fwctl_fops_release`), so the device is registered. + unsafe { + device.with_registration_data(|device, reg_data| { + T::close(ctx.as_mut().project().uctx, device, reg_data); + }); + } + + // SAFETY: close is the last callback before fwctl frees the allocation. + unsafe { core::ptr::drop_in_place(ctx.project().uctx.get_unchecked_mut()) }; + } + + /// Returns device-specific information for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx<T>`. + /// - `length` must be a valid pointer. + unsafe extern "C" fn info_callback( + uctx: *mut bindings::fwctl_uctx, + length: *mut usize, + ) -> *mut ffi::c_void { + // SAFETY: info is called for an opened Rust user context. + let ctx = unsafe { UserCtx::<T>::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: `info` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| T::info(ctx.uctx(), device, reg_data)) + }; + + match result { + Ok(kvec) if kvec.is_empty() => { + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = 0 }; + // Return NULL for empty data; kfree(NULL) is safe. + core::ptr::null_mut() + } + Ok(kvec) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + // SAFETY: `length` is a valid out-parameter. + unsafe { *length = len }; + ptr.cast::<ffi::c_void>() + } + Err(e) => Error::to_ptr(e), + } + } + + /// Dispatches a firmware RPC for an opened Rust user context. + /// + /// # Safety + /// + /// - `uctx` must point to a fully initialised `UserCtx<T>`. + /// - `rpc_in` must be valid, initialised, and exclusively accessible for `in_len` bytes. + /// - `out_len` must be valid for reading and writing an initialised `usize`. + unsafe extern "C" fn fw_rpc_callback( + uctx: *mut bindings::fwctl_uctx, + scope: u32, + rpc_in: *mut ffi::c_void, + in_len: usize, + out_len: *mut usize, + ) -> *mut ffi::c_void { + let scope = match RpcScope::try_from(scope) { + Ok(s) => s, + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` points to an initialised `usize` supplied by fwctl. + let max_output_len = unsafe { *out_len }; + + // SAFETY: RPC is called for an opened Rust user context. + let ctx = unsafe { UserCtx::<T>::from_raw(uctx) }; + let device = ctx.device(); + + // SAFETY: fwctl passes an exclusively owned buffer that is valid and initialised for + // `in_len` bytes. It remains live for the duration of this callback. + let rpc_buf = unsafe { slice::from_raw_parts_mut(rpc_in.cast::<u8>(), in_len) }; + + // SAFETY: `fw_rpc` is called under `registration_lock` read, so the device is registered. + let result = unsafe { + device.with_registration_data(|device, reg_data| { + T::fw_rpc(ctx.uctx(), device, reg_data, scope, rpc_buf, max_output_len) + }) + }; + + let (response, response_len) = match result { + Ok(FwRpcResponse::InPlace(len)) => { + if len > in_len { + return Error::to_ptr(EINVAL); + } + + (rpc_in, len) + } + Ok(FwRpcResponse::NewBuffer(kvec)) if kvec.is_empty() => { + // Return NULL for empty data; kvfree(NULL) is safe. + (core::ptr::null_mut(), 0) + } + Ok(FwRpcResponse::NewBuffer(kvec)) => { + let (ptr, len, _cap) = kvec.into_raw_parts(); + (ptr.cast::<ffi::c_void>(), len) + } + Err(e) => return Error::to_ptr(e), + }; + + // SAFETY: `out_len` is a valid out-parameter. + unsafe { *out_len = response_len }; + response + } +} diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index 624b971ca8b0..0487bae811fb 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -65,10 +65,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// IdTable type for I2C @@ -77,14 +73,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a I2C `IdTable` with its alias for modpost. #[macro_export] macro_rules! i2c_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::i2c::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("i2c", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("i2c", $crate::i2c::DeviceId, $($tt)*); }; } @@ -142,7 +132,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { } // SAFETY: `idrv` is guaranteed to be a valid `DriverType`. - to_result(unsafe { bindings::i2c_register_driver(module.0, idrv.get()) }) + to_result(unsafe { bindings::i2c_register_driver(module.as_ptr(), idrv.get()) }) } unsafe fn unregister(idrv: &Opaque<Self::DriverType>) { @@ -159,8 +149,10 @@ impl<T: Driver> Adapter<T> { // INVARIANT: `idev` is valid for the duration of `probe_callback()`. let idev = unsafe { &*idev.cast::<I2cClient<device::CoreInternal<'_>>>() }; - let info = - Self::i2c_id_info(idev).or_else(|| <Self as driver::Adapter>::id_info(idev.as_ref())); + let info = Self::i2c_id_info(idev).or_else(|| { + // SAFETY: `idev` matched data is of type `Self::IdInfo`. + unsafe { <Self as driver::Adapter>::id_info(idev.as_ref()) } + }); from_result(|| { let data = T::probe(idev, info); @@ -218,7 +210,8 @@ impl<T: Driver> Adapter<T> { // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*raw_id.cast::<DeviceId>() }; - Some(table.info(<DeviceId as RawDeviceIdIndex>::index(id))) + // SAFETY: `id` comes from `table` which is of type `IdArray<_, Self::IdInfo>`. + Some(unsafe { id.info_unchecked::<T::IdInfo>() }) } } @@ -267,7 +260,6 @@ macro_rules! module_i2c_driver { /// /// kernel::acpi_device_table!( /// ACPI_TABLE, -/// MODULE_ACPI_TABLE, /// <MyDriver as i2c::Driver>::IdInfo, /// [ /// (acpi::DeviceId::new(c"LNUXBEEF"), ()) @@ -276,7 +268,6 @@ macro_rules! module_i2c_driver { /// /// kernel::i2c_device_table!( /// I2C_TABLE, -/// MODULE_I2C_TABLE, /// <MyDriver as i2c::Driver>::IdInfo, /// [ /// (i2c::DeviceId::new(c"rust_driver_i2c"), ()) @@ -285,7 +276,6 @@ macro_rules! module_i2c_driver { /// /// kernel::of_device_table!( /// OF_TABLE, -/// MODULE_OF_TABLE, /// <MyDriver as i2c::Driver>::IdInfo, /// [ /// (of::DeviceId::new(c"test,device"), ()) @@ -404,6 +394,7 @@ impl I2cAdapter { } /// Gets pointer to an `i2c_adapter` by index. + #[inline] pub fn get(index: i32) -> Result<ARef<Self>> { // SAFETY: `index` must refer to a valid I2C adapter; the kernel // guarantees that `i2c_get_adapter(index)` returns either a valid @@ -425,11 +416,13 @@ kernel::impl_device_context_into_aref!(I2cAdapter); // SAFETY: Instances of `I2cAdapter` are always reference-counted. unsafe impl AlwaysRefCounted for I2cAdapter { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::i2c_get_adapter(self.index()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) } diff --git a/rust/kernel/impl_flags.rs b/rust/kernel/impl_flags.rs index e2bd7639da12..fdf44d5eea9c 100644 --- a/rust/kernel/impl_flags.rs +++ b/rust/kernel/impl_flags.rs @@ -19,7 +19,10 @@ /// # Examples /// /// ``` -/// use kernel::impl_flags; +/// use kernel::{ +/// bits::bit_u32, +/// impl_flags, // +/// }; /// /// impl_flags!( /// /// Represents multiple permissions. @@ -30,13 +33,13 @@ /// #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// pub enum Permission { /// /// Read permission. -/// Read = 1 << 0, +/// Read = bit_u32(0), /// /// /// Write permission. -/// Write = 1 << 1, +/// Write = bit_u32(1), /// /// /// Execute permission. -/// Execute = 1 << 2, +/// Execute = bit_u32(2), /// } /// ); /// diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 05a12e869a57..1fdc3963e3e3 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -158,7 +158,9 @@ pub trait InPlaceInit<T>: Sized { { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + pin_init_from_closure(|slot| { + pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e)) + }) }; Self::try_pin_init(init, flags) } @@ -176,7 +178,7 @@ pub trait InPlaceInit<T>: Sized { { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + init_from_closure(|slot| pin_init::raw_try_init(slot, init).map_err(|e| Error::from(e))) }; Self::try_init(init, flags) } diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs new file mode 100644 index 000000000000..a880ec3b8538 --- /dev/null +++ b/rust/kernel/interrupt.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Interrupt controls +//! +//! This module allows Rust code to annotate areas of code where local processor interrupts should +//! be disabled, along with actually disabling local processor interrupts. +//! +//! # ⚠️ Warning! ⚠️ +//! +//! The usage of this module can be more complicated than meets the eye, especially surrounding +//! [preemptible kernels]. It's recommended to take care when using the functions and types defined +//! here and familiarize yourself with the various documentation we have before using them, along +//! with the various documents we link to here. +//! +//! # Reading material +//! +//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076) +//! +//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html + +use crate::types::NotThreadSafe; + +/// A guard that represents local processor interrupt disablement on preemptible kernels. +/// +/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have +/// been disabled on a preemptible kernel. +/// +/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require +/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels. +/// +/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local +/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the +/// state of interrupts are made by this type on non-preemptible kernels. +/// +/// # Invariants +/// +/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this +/// type exists. +pub struct LocalInterruptDisabled(NotThreadSafe); + +/// Disable local processor interrupts on a preemptible kernel. +/// +/// This function disables local processor interrupts on a preemptible kernel, and returns a +/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is +/// a no-op. +/// +/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are +/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local +/// processor interrupt state management on their own and managing this by hand is quite error +/// prone. +#[inline] +pub fn local_interrupt_disable() -> LocalInterruptDisabled { + // SAFETY: It's always safe to call `local_interrupt_disable()`. + unsafe { bindings::local_interrupt_disable() }; + + LocalInterruptDisabled(NotThreadSafe) +} + +impl Drop for LocalInterruptDisabled { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this + // object, hence calling the corresponding `local_interrupt_enable()` is safe. + unsafe { bindings::local_interrupt_enable() }; + } +} + +impl LocalInterruptDisabled { + /// Assume that local processor interrupts are disabled on preemptible kernels. + /// + /// This can be used for annotating code that is known to be run in contexts where local + /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local + /// interrupt state on its own. + /// + /// # Safety + /// + /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This + /// could be a context like, for example, an interrupt handler. + #[inline] + pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled { + const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe); + + // Confirm they're actually disabled if lockdep is available + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`. + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + ASSUME_DISABLED + } +} diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index fcc7678fd9e3..5ce9fd129068 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -4,11 +4,21 @@ //! //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + use crate::{ bindings, - prelude::*, // + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // }; +#[cfg(CONFIG_HAS_IOMEM)] pub mod mem; pub mod poll; pub mod register; @@ -31,128 +41,226 @@ pub type PhysAddr = bindings::phys_addr_t; /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. pub type ResourceSize = bindings::resource_size_t; +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +#[derive(FromBytes)] +pub struct Region<const SIZE: usize = 0> { + inner: [u8], +} + +impl<const SIZE: usize> Region<SIZE> { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE> + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl<const SIZE: usize> KnownSize for Region<SIZE> { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + /// Raw representation of an MMIO region. /// +/// `MmioRaw<T>` is equivalent to `T __iomem *` in C. +/// /// By itself, the existence of an instance of this structure does not provide any guarantees that /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` /// structure any guarantees are given. -pub struct MmioRaw<const SIZE: usize = 0> { - addr: usize, - maxsize: usize, +pub struct MmioRaw<T: ?Sized> { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, } -impl<const SIZE: usize> MmioRaw<SIZE> { - /// Returns a new `MmioRaw` instance on success, an error otherwise. - pub fn new(addr: usize, maxsize: usize) -> Result<Self> { - if maxsize < SIZE { - return Err(EINVAL); +impl<T: ?Sized> Copy for MmioRaw<T> {} +impl<T: ?Sized> Clone for MmioRaw<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl<T: ?Sized> Send for MmioRaw<T> {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl<T: ?Sized> Sync for MmioRaw<T> {} + +impl<T> MmioRaw<T> { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), } + } +} - Ok(Self { addr, maxsize }) +impl<const SIZE: usize> MmioRaw<Region<SIZE>> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result<Self> { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) } +} +impl<T: ?Sized + KnownSize> MmioRaw<T> { /// Returns the base address of the MMIO region. #[inline] pub fn addr(&self) -> usize { - self.addr + self.ptr.addr() } - /// Returns the maximum size of the MMIO region. + /// Returns the size of the MMIO region. #[inline] - pub fn maxsize(&self) -> usize { - self.maxsize + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) } } -/// IO-mapped memory region. -/// -/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the -/// mapping, performing an additional region request etc. -/// -/// # Invariant -/// -/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size -/// `maxsize`. -/// -/// # Examples -/// -/// ```no_run -/// use kernel::{ -/// bindings, -/// ffi::c_void, -/// io::{ -/// Io, -/// IoKnownSize, -/// Mmio, -/// MmioRaw, -/// PhysAddr, -/// }, -/// }; -/// use core::ops::Deref; -/// -/// // See also `pci::Bar` for a real example. -/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>); -/// -/// impl<const SIZE: usize> IoMem<SIZE> { -/// /// # Safety -/// /// -/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs -/// /// virtual address space. -/// unsafe fn new(paddr: usize) -> Result<Self>{ -/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is -/// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; -/// if addr.is_null() { -/// return Err(ENOMEM); -/// } -/// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) -/// } -/// } -/// -/// impl<const SIZE: usize> Drop for IoMem<SIZE> { -/// fn drop(&mut self) { -/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; -/// } -/// } -/// -/// impl<const SIZE: usize> Deref for IoMem<SIZE> { -/// type Target = Mmio<SIZE>; -/// -/// fn deref(&self) -> &Self::Target { -/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } -/// } -/// } -/// -///# fn no_run() -> Result<(), Error> { -/// // SAFETY: Invalid usage for example purposes. -/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? }; -/// iomem.write32(0x42, 0x0); -/// assert!(iomem.try_write32(0x42, 0x0).is_ok()); -/// assert!(iomem.try_write32(0x42, 0x4).is_err()); -/// # Ok(()) -/// # } -/// ``` -#[repr(transparent)] -pub struct Mmio<const SIZE: usize = 0>(MmioRaw<SIZE>); - -/// Checks whether an access of type `U` at the given `offset` +/// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. +/// +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. #[inline] -const fn offset_valid<U>(offset: usize, size: usize) -> bool { - let type_size = core::mem::size_of::<U>(); - if let Some(end) = offset.checked_add(type_size) { - end <= size && offset % type_size == 0 +const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::<U>()) { + end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0) } else { false } } +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> <IO::Backend as IoBackend>::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<<IO::Backend as IoBackend>::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. +/// +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). +/// +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. +/// +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; + + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; +} + /// Trait indicating that an I/O backend supports operations of a certain type and providing an /// implementation for these operations. /// @@ -161,20 +269,75 @@ const fn offset_valid<U>(offset: usize, size: usize) -> bool { /// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`, /// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit /// system might implement all four. -pub trait IoCapable<T> { - /// Performs an I/O read of type `T` at `address` and returns the result. +pub trait IoCapable<T>: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; + + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); +} + +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. /// /// # Safety /// - /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`. - unsafe fn io_read(&self, address: usize) -> T; + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); - /// Performs an I/O write of `value` at `address`. + /// Copy contents from `buffer` to `view`. /// /// # Safety /// - /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`. - unsafe fn io_write(&self, value: T, address: usize); + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); + + /// Copy from `view` and return the value. + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), + ) + }; + + let mut buf = MaybeUninit::<T>::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::<T>()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); + } } /// Describes a given I/O location: its offset, width, and type to convert the raw value from and @@ -186,15 +349,16 @@ pub trait IoCapable<T> { /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] /// macro). /// -/// An `IoLoc<T>` carries three pieces of information: +/// An `IoLoc<Base, T>` carries the following pieces of information: /// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), /// - The width of the access (determined by [`IoLoc::IoType`]), /// - The type `T` in which the raw data is returned or provided. /// /// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type /// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). -pub trait IoLoc<T> { +pub trait IoLoc<Base: ?Sized, T> { /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). type IoType: Into<T> + From<T>; @@ -202,12 +366,12 @@ pub trait IoLoc<T> { fn offset(self) -> usize; } -/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of -/// [`Io::read`] and [`Io::write`]. +/// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( - impl IoLoc<$ty> for usize { + impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize { type IoType = $ty; #[inline(always)] @@ -225,181 +389,430 @@ impl_usize_ioloc!(u8, u16, u32, u64); /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// -/// This is an abstract representation to be implemented by arbitrary I/O -/// backends (e.g. MMIO, PCI config space, etc.). +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. /// -/// The [`Io`] trait provides: -/// - Base address and size information +/// This should be implemented on cheaply copyable handles, such as references or view types. +pub trait IoBase<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + + /// Return a view that covers the full region. + fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. +/// +/// This trait provides: /// - Helper methods for offset validation and address calculation /// - Fallible (runtime checked) accessors for different data widths /// -/// Which I/O methods are available depends on which [`IoCapable<T>`] traits -/// are implemented for the type. -/// -/// # Examples -/// -/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically -/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. -pub trait Io { - /// Returns the base address of this mapping. - fn addr(&self) -> usize; +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. + #[inline] + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } - /// Returns the maximum size of this mapping. - fn maxsize(&self) -> usize; + /// Returns the length of the slice in number of elements. + #[inline] + fn len<T>(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty<T>(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } - /// Returns the absolute I/O address for a given `offset`, - /// performing runtime bound checks. + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` #[inline] - fn io_addr<U>(&self, offset: usize) -> Result<usize> { - if !offset_valid::<U>(offset, self.maxsize()) { + fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::<U>() > KnownSize::size(ptr) { return Err(EINVAL); } - // Probably no need to check, since the safety requirements of `Self::new` guarantee that - // this can't overflow. - self.addr().checked_add(offset).ok_or(EINVAL) + if ptr.addr() % align_of::<U>() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable<Self::Target>, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable<Self::Target>, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } } /// Fallible 8-bit read with runtime bounds check. #[inline(always)] - fn try_read8(&self, offset: usize) -> Result<u8> + fn try_read8(self, offset: usize) -> Result<u8> where - Self: IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.try_read(offset) } /// Fallible 16-bit read with runtime bounds check. #[inline(always)] - fn try_read16(&self, offset: usize) -> Result<u16> + fn try_read16(self, offset: usize) -> Result<u16> where - Self: IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.try_read(offset) } /// Fallible 32-bit read with runtime bounds check. #[inline(always)] - fn try_read32(&self, offset: usize) -> Result<u32> + fn try_read32(self, offset: usize) -> Result<u32> where - Self: IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.try_read(offset) } /// Fallible 64-bit read with runtime bounds check. #[inline(always)] - fn try_read64(&self, offset: usize) -> Result<u64> + fn try_read64(self, offset: usize) -> Result<u64> where - Self: IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.try_read(offset) } /// Fallible 8-bit write with runtime bounds check. #[inline(always)] - fn try_write8(&self, value: u8, offset: usize) -> Result + fn try_write8(self, value: u8, offset: usize) -> Result where - Self: IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.try_write(offset, value) } /// Fallible 16-bit write with runtime bounds check. #[inline(always)] - fn try_write16(&self, value: u16, offset: usize) -> Result + fn try_write16(self, value: u16, offset: usize) -> Result where - Self: IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.try_write(offset, value) } /// Fallible 32-bit write with runtime bounds check. #[inline(always)] - fn try_write32(&self, value: u32, offset: usize) -> Result + fn try_write32(self, value: u32, offset: usize) -> Result where - Self: IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.try_write(offset, value) } /// Fallible 64-bit write with runtime bounds check. #[inline(always)] - fn try_write64(&self, value: u64, offset: usize) -> Result + fn try_write64(self, value: u64, offset: usize) -> Result where - Self: IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.try_write(offset, value) } /// Infallible 8-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read8(&self, offset: usize) -> u8 + fn read8(self, offset: usize) -> u8 where - Self: IoKnownSize + IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.read(offset) } /// Infallible 16-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read16(&self, offset: usize) -> u16 + fn read16(self, offset: usize) -> u16 where - Self: IoKnownSize + IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.read(offset) } /// Infallible 32-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read32(&self, offset: usize) -> u32 + fn read32(self, offset: usize) -> u32 where - Self: IoKnownSize + IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.read(offset) } /// Infallible 64-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read64(&self, offset: usize) -> u64 + fn read64(self, offset: usize) -> u64 where - Self: IoKnownSize + IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.read(offset) } /// Infallible 8-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write8(&self, value: u8, offset: usize) + fn write8(self, value: u8, offset: usize) where - Self: IoKnownSize + IoCapable<u8>, + usize: IoLoc<Self::Target, u8, IoType = u8>, + Self::Backend: IoCapable<u8>, { self.write(offset, value) } /// Infallible 16-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write16(&self, value: u16, offset: usize) + fn write16(self, value: u16, offset: usize) where - Self: IoKnownSize + IoCapable<u16>, + usize: IoLoc<Self::Target, u16, IoType = u16>, + Self::Backend: IoCapable<u16>, { self.write(offset, value) } /// Infallible 32-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write32(&self, value: u32, offset: usize) + fn write32(self, value: u32, offset: usize) where - Self: IoKnownSize + IoCapable<u32>, + usize: IoLoc<Self::Target, u32, IoType = u32>, + Self::Backend: IoCapable<u32>, { self.write(offset, value) } /// Infallible 64-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write64(&self, value: u64, offset: usize) + fn write64(self, value: u64, offset: usize) where - Self: IoKnownSize + IoCapable<u64>, + usize: IoLoc<Self::Target, u64, IoType = u64>, + Self::Backend: IoCapable<u64>, { self.write(offset, value) } @@ -414,9 +827,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &Mmio) -> Result { + /// fn do_reads(io: Mmio<'_, Region>) -> Result { /// // 32-bit read from address `0x10`. /// let v: u32 = io.try_read(0x10)?; /// @@ -427,15 +841,13 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn try_read<T, L>(&self, location: L) -> Result<T> + fn try_read<T, L>(self, location: L) -> Result<T> where - L: IoLoc<T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr::<L::IoType>(location.offset())?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }.into()) + let view = io_view::<Self, L::IoType>(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) } /// Generic fallible write with runtime bounds check. @@ -448,9 +860,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &Mmio) -> Result { + /// fn do_writes(io: Mmio<'_, Region>) -> Result { /// // 32-bit write of value `1` at address `0x10`. /// io.try_write(0x10, 1u32)?; /// @@ -461,17 +874,14 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn try_write<T, L>(&self, location: L, value: T) -> Result + fn try_write<T, L>(self, location: L, value: T) -> Result where - L: IoLoc<T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr::<L::IoType>(location.offset())?; + let view = io_view::<Self, L::IoType>(self, location.offset())?; let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } - + Self::Backend::io_write(view, io_value); Ok(()) } @@ -486,6 +896,7 @@ pub trait Io { /// register, /// Io, /// Mmio, + /// Region, /// }; /// /// register! { @@ -501,17 +912,17 @@ pub trait Io { /// } /// } /// - /// fn do_write_reg(io: &Mmio) -> Result { + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { /// /// io.try_write_reg(VERSION::new(1, 0)) /// } /// ``` #[inline(always)] - fn try_write_reg<T, L, V>(&self, value: V) -> Result + fn try_write_reg<T, L, V>(self, value: V) -> Result where - L: IoLoc<T>, - V: LocatedRegister<Location = L, Value = T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + V: LocatedRegister<Self::Target, Location = L, Value = T>, + Self::Backend: IoCapable<L::IoType>, { let (location, value) = value.into_io_op(); @@ -531,29 +942,27 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) -> Result { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { /// io.try_update(0x10, |v: u32| { /// v + 1 /// }) /// } /// ``` #[inline(always)] - fn try_update<T, L, F>(&self, location: L, f: F) -> Result + fn try_update<T, L, F>(self, location: L, f: F) -> Result where - L: IoLoc<T>, - Self: IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, F: FnOnce(T) -> T, { - let address = self.io_addr::<L::IoType>(location.offset())?; + let view = io_view::<Self, L::IoType>(self, location.offset())?; - // SAFETY: `address` has been validated by `io_addr`. - let value: T = unsafe { self.io_read(address) }.into(); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); Ok(()) } @@ -568,9 +977,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &Mmio<0x1000>) { + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit read from address `0x10`. /// let v: u32 = io.read(0x10); /// @@ -579,15 +989,13 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn read<T, L>(&self, location: L) -> T + fn read<T, L>(self, location: L) -> T where - L: IoLoc<T>, - Self: IoKnownSize + IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr_assert::<L::IoType>(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) }.into() + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + Self::Backend::io_read(view).into() } /// Generic infallible write with compile-time bounds check. @@ -600,9 +1008,10 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &Mmio<0x1000>) { + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit write of value `1` at address `0x10`. /// io.write(0x10, 1u32); /// @@ -611,16 +1020,14 @@ pub trait Io { /// } /// ``` #[inline(always)] - fn write<T, L>(&self, location: L, value: T) + fn write<T, L>(self, location: L, value: T) where - L: IoLoc<T>, - Self: IoKnownSize + IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, { - let address = self.io_addr_assert::<L::IoType>(location.offset()); + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } /// Generic infallible write of a fully-located register value. @@ -634,6 +1041,7 @@ pub trait Io { /// register, /// Io, /// Mmio, + /// Region, /// }; /// /// register! { @@ -649,16 +1057,16 @@ pub trait Io { /// } /// } /// - /// fn do_write_reg(io: &Mmio<0x1000>) { + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { /// io.write_reg(VERSION::new(1, 0)); /// } /// ``` #[inline(always)] - fn write_reg<T, L, V>(&self, value: V) + fn write_reg<T, L, V>(self, value: V) where - L: IoLoc<T>, - V: LocatedRegister<Location = L, Value = T>, - Self: IoKnownSize + IoCapable<L::IoType>, + L: IoLoc<Self::Target, T>, + V: LocatedRegister<Self::Target, Location = L, Value = T>, + Self::Backend: IoCapable<L::IoType>, { let (location, value) = value.into_io_op(); @@ -678,143 +1086,208 @@ pub trait Io { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { /// io.update(0x10, |v: u32| { /// v + 1 /// }) /// } /// ``` #[inline(always)] - fn update<T, L, F>(&self, location: L, f: F) + fn update<T, L, F>(self, location: L, f: F) where - L: IoLoc<T>, - Self: IoKnownSize + IoCapable<L::IoType> + Sized, + L: IoLoc<Self::Target, T>, + Self::Backend: IoCapable<L::IoType>, F: FnOnce(T) -> T, { - let address = self.io_addr_assert::<L::IoType>(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - let value: T = unsafe { self.io_read(address) }.into(); + let view = io_view_assert::<Self, L::IoType>(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } } -/// Trait for types with a known size at compile time. +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} + +/// A view of memory-mapped I/O region. /// -/// This trait is implemented by I/O backends that have a compile-time known size, -/// enabling the use of infallible I/O accessors with compile-time bounds checking. +/// # Invariant /// -/// Types implementing this trait can use the infallible methods in [`Io`] trait -/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound. -pub trait IoKnownSize: Io { - /// Minimum usable size of this region. - const MIN_SIZE: usize; +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} - /// Returns the absolute I/O address for a given `offset`, - /// performing compile-time bound checks. - // Always inline to optimize out error path of `build_assert`. - #[inline(always)] - fn io_addr_assert<U>(&self, offset: usize) -> usize { - build_assert!(offset_valid::<U>(offset, Self::MIN_SIZE)); +impl<T: ?Sized> Copy for Mmio<'_, T> {} +impl<T: ?Sized> Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} - self.addr() + offset +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } } } -/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { + type Backend = MmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> Mmio<'a, T> { + self + } +} + +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_mmio_io_capable { - ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { - $(#[$attr])* - impl<const SIZE: usize> IoCapable<$ty> for $mmio<SIZE> { - unsafe fn io_read(&self, address: usize) -> $ty { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$read_fn(address as *const c_void) } + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { + #[inline] + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } } - unsafe fn io_write(&self, value: $ty, address: usize) { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$write_fn(value, address as *mut c_void) } + #[inline] + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } } } }; } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(Mmio, u8, readb, writeb); -impl_mmio_io_capable!(Mmio, u16, readw, writew); -impl_mmio_io_capable!(Mmio, u32, readl, writel); +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - Mmio, - #[cfg(CONFIG_64BIT)] - u64, - readq, - writeq -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); -impl<const SIZE: usize> Io for Mmio<SIZE> { - /// Returns the base address of this mapping. +impl IoCopyable for MmioBackend { #[inline] - fn addr(&self) -> usize { - self.0.addr() + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } } - /// Returns the maximum size of this mapping. #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() - } -} - -impl<const SIZE: usize> IoKnownSize for Mmio<SIZE> { - const MIN_SIZE: usize = SIZE; -} - -impl<const SIZE: usize> Mmio<SIZE> { - /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping. - /// - /// # Safety - /// - /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size - /// `maxsize`. - pub unsafe fn from_raw(raw: &MmioRaw<SIZE>) -> &Self { - // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. - unsafe { &*core::ptr::from_ref(raw).cast() } + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } } } -/// [`Mmio`] wrapper using relaxed accessors. +/// [`Mmio`] but using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of /// the regular ones. /// /// See [`Mmio::relaxed`] for a usage example. -#[repr(transparent)] -pub struct RelaxedMmio<const SIZE: usize = 0>(Mmio<SIZE>); +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); + +impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {} +impl<T: ?Sized> Clone for RelaxedMmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; -impl<const SIZE: usize> Io for RelaxedMmio<SIZE> { #[inline] - fn addr(&self) -> usize { - self.0.addr() + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) } #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) } } -impl<const SIZE: usize> IoKnownSize for RelaxedMmio<SIZE> { - const MIN_SIZE: usize = SIZE; +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self + } } -impl<const SIZE: usize> Mmio<SIZE> { - /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. /// /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses /// and can be used when such ordering is not required. @@ -825,31 +1298,457 @@ impl<const SIZE: usize> Mmio<SIZE> { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// RelaxedMmio, /// }; /// - /// fn do_io(io: &Mmio<0x100>) { + /// fn do_io(io: Mmio<'_, Region<0x100>>) { /// // The access is performed using `readl_relaxed` instead of `readl`. /// let v = io.relaxed().read32(0x10); /// } /// /// ``` - pub fn relaxed(&self) -> &RelaxedMmio<SIZE> { - // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio<SIZE>` and - // `RelaxedMmio<SIZE>` have identical layout. - unsafe { core::mem::transmute(self) } + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) } } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - RelaxedMmio, - #[cfg(CONFIG_64BIT)] - u64, - readq_relaxed, - writeq_relaxed -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl<T: ?Sized> Copy for SysMem<'_, T> {} +impl<T: ?Sized> Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. + /// + /// # Safety + /// + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> { + self + } +} + +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl<T: ?Sized> Copy for IoSysMap<'_, T> {} +impl<T: ?Sized> Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl<T> IoCapable<T> for IoSysMapBackend +where + MmioBackend: IoCapable<T>, + SysMemBackend: IoCapable<T>, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper<T>(pub T); + +impl<'a, T> ProjectHelper<T> +where + T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view<U: ?Sized + KnownSize>( + self, + ptr: *mut U, + ) -> <T::Backend as IoBackend>::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; +} +#[doc(inline)] +pub use crate::io_write; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index fc2a3e24f8d5..32a919099dcd 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -2,24 +2,28 @@ //! Generic memory-mapped IO. -use core::ops::Deref; - use crate::{ device::{ Bound, Device, // }, - devres::Devres, + devres::DevresLt, io::{ self, resource::{ Region, Resource, // }, + IoBase, Mmio, + MmioBackend, MmioRaw, // }, prelude::*, + types::{ + CovariantForLt, + ForLt, // + }, }; /// An IO request for a specific device and resource. @@ -172,6 +176,19 @@ pub struct ExclusiveIoMem<'a, const SIZE: usize> { _region: Region, } +impl<const SIZE: usize> ForLt for ExclusiveIoMem<'static, SIZE> { + type Of<'a> = ExclusiveIoMem<'a, SIZE>; +} + +// SAFETY: `ExclusiveIoMem<'a, SIZE>` is covariant over `'a`; it holds an `IoMem<'a, SIZE>`, +// which holds `&'a Device<Bound>`, which is covariant. +unsafe impl<const SIZE: usize> CovariantForLt for ExclusiveIoMem<'static, SIZE> {} + +/// A device-managed exclusive I/O memory region. +/// +/// See [`ExclusiveIoMem::into_devres`]. +pub type DevresExclusiveIoMem<const SIZE: usize> = DevresLt<ExclusiveIoMem<'static, SIZE>>; + impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { /// Creates a new `ExclusiveIoMem` instance. fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> { @@ -198,23 +215,23 @@ impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { /// Consume the `ExclusiveIoMem` and register it as a device-managed resource. /// - /// The returned `Devres<ExclusiveIoMem<'static, SIZE>>` can outlive the original lifetime - /// `'a`. Access to the I/O memory is revoked when the device is unbound. - pub fn into_devres(self) -> Result<Devres<ExclusiveIoMem<'static, SIZE>>> { - // SAFETY: Casting to `'static` is sound because `Devres` guarantees the - // `ExclusiveIoMem` does not actually outlive the device -- access is revoked and the - // resource is released when the device is unbound. - let iomem: ExclusiveIoMem<'static, SIZE> = unsafe { core::mem::transmute(self) }; - let dev = iomem.iomem.dev; - Devres::new(dev, iomem) + /// The returned [`DevresExclusiveIoMem`] can outlive the original borrow and be stored in + /// driver data. Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result<DevresExclusiveIoMem<SIZE>> { + let dev = self.iomem.dev; + // SAFETY: `ExclusiveIoMem` only holds a device reference and an I/O mapping, both of + // which remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } } } -impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region<SIZE>; - fn deref(&self) -> &Self::Target { - &self.iomem + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() } } @@ -229,9 +246,22 @@ impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> { /// start of the I/O memory mapped region. pub struct IoMem<'a, const SIZE: usize = 0> { dev: &'a Device<Bound>, - io: MmioRaw<SIZE>, + io: MmioRaw<super::Region<SIZE>>, } +impl<const SIZE: usize> ForLt for IoMem<'static, SIZE> { + type Of<'a> = IoMem<'a, SIZE>; +} + +// SAFETY: `IoMem<'a, SIZE>` is covariant over `'a`; it holds `&'a Device<Bound>`, +// which is covariant. +unsafe impl<const SIZE: usize> CovariantForLt for IoMem<'static, SIZE> {} + +/// A device-managed I/O memory region. +/// +/// See [`IoMem::into_devres`]. +pub type DevresIoMem<const SIZE: usize = 0> = DevresLt<IoMem<'static, SIZE>>; + impl<'a, const SIZE: usize> IoMem<'a, SIZE> { fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> { // Note: Some ioremap() implementations use types that depend on the CPU @@ -264,23 +294,19 @@ impl<'a, const SIZE: usize> IoMem<'a, SIZE> { return Err(ENOMEM); } - let io = MmioRaw::new(addr as usize, size)?; - + let io = MmioRaw::new_region(addr as usize, size)?; Ok(IoMem { dev, io }) } /// Consume the `IoMem` and register it as a device-managed resource. /// - /// The returned `Devres<IoMem<'static, SIZE>>` can outlive the original - /// lifetime `'a`. Access to the I/O memory is revoked when the device - /// is unbound. - pub fn into_devres(self) -> Result<Devres<IoMem<'static, SIZE>>> { - // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `IoMem` does not - // actually outlive the device -- access is revoked and the resource is released when the - // device is unbound. - let iomem: IoMem<'static, SIZE> = unsafe { core::mem::transmute(self) }; - let dev = iomem.dev; - Devres::new(dev, iomem) + /// The returned [`DevresIoMem`] can outlive the original borrow and be stored in driver data. + /// Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result<DevresIoMem<SIZE>> { + let dev = self.dev; + // SAFETY: `IoMem` only holds a device reference and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } } } @@ -291,11 +317,13 @@ impl<const SIZE: usize> Drop for IoMem<'_, SIZE> { } } -impl<const SIZE: usize> Deref for IoMem<'_, SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region<SIZE>; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: Safe as by the invariant of `IoMem`. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 75d1b3e8596c..d75f2fcf46f2 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -48,13 +48,14 @@ use crate::{ /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result { +/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result { /// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), @@ -135,13 +136,14 @@ where /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout_atomic, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result { +/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result { /// read_poll_timeout_atomic( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index f924c7c7c1db..03dfd2ff48c7 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -58,7 +58,7 @@ //! }, //! num::Bounded, //! }; -//! # use kernel::io::Mmio; +//! # use kernel::io::{Mmio, Region}; //! # register! { //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; @@ -66,7 +66,7 @@ //! # 3:0 minor_revision; //! # } //! # } -//! # fn test(io: &Mmio<0x1000>) { +//! # fn test(io: Mmio<'_, Region<0x1000>>) { //! # fn obtain_vendor_id() -> u8 { 0xff } //! //! // Read from the register's defined offset (0x100). @@ -113,6 +113,8 @@ use crate::{ io::IoLoc, // }; +use super::Region; + /// Trait implemented by all registers. pub trait Register: Sized { /// Backing primitive type of the register. @@ -129,7 +131,7 @@ pub trait FixedRegister: Register {} /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. -impl<T> IoLoc<T> for () +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for () where T: FixedRegister, { @@ -143,7 +145,7 @@ where /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used /// as an [`IoLoc`]. -impl<T> IoLoc<T> for T +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T where T: FixedRegister, { @@ -168,7 +170,7 @@ impl<T: FixedRegister> FixedRegisterLoc<T> { } } -impl<T> IoLoc<T> for FixedRegisterLoc<T> +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T> where T: FixedRegister, { @@ -239,7 +241,7 @@ where } } -impl<T, B> IoLoc<T> for RelativeRegisterLoc<T, B> +impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B> where T: RelativeRegister, B: RegisterBase<T::BaseFamily> + ?Sized, @@ -283,7 +285,7 @@ impl<T: RegisterArray> RegisterArrayLoc<T> { } } -impl<T> IoLoc<T> for RegisterArrayLoc<T> +impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T> where T: RegisterArray, { @@ -370,7 +372,7 @@ where } } -impl<T, B> IoLoc<T> for RelativeRegisterArrayLoc<T, B> +impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B> where T: RelativeRegisterArray, B: RegisterBase<T::BaseFamily> + ?Sized, @@ -387,18 +389,18 @@ where /// which to write it. /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). -pub trait LocatedRegister { +pub trait LocatedRegister<Base: ?Sized> { /// Register value to write. type Value: Register; /// Full location information at which to write the value. - type Location: IoLoc<Self::Value>; + type Location: IoLoc<Base, Self::Value>; /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write /// operation. fn into_io_op(self) -> (Self::Location, Self::Value); } -impl<T> LocatedRegister for T +impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T where T: FixedRegister, { @@ -444,16 +446,16 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// /// register! { /// FIXED_REG(u32) @ 0x100 { -/// 16:8 high_byte; +/// 15:8 high_byte; /// 7:0 low_byte; /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// let val = io.read(FIXED_REG); /// /// // Write from an already-existing value. @@ -557,7 +559,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -582,7 +584,7 @@ where /// } /// } /// -/// # fn test(io: Mmio<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// // Read the status of `Cpu0`. /// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>()); /// @@ -599,7 +601,7 @@ where /// } /// } /// -/// # fn test2(io: Mmio<0x1000>) { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { /// // Start the aliased `CPU0`, leaving its other fields untouched. /// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true)); /// # } @@ -636,7 +638,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -649,7 +651,7 @@ where /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) +/// # fn test(io: Mmio<'_, Region<0x1000>>) /// # -> Result<(), Error>{ /// // Read scratch register 0, i.e. I/O address `0x80`. /// let scratch_0 = io.read(SCRATCH::at(0)).value(); @@ -722,7 +724,7 @@ where /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -750,7 +752,7 @@ where /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// // Read scratch register 0 of CPU0. /// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0)); /// @@ -792,7 +794,7 @@ where /// } /// } /// -/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status(); /// # Ok(()) /// # } @@ -802,6 +804,9 @@ macro_rules! register { // Entry point for the macro, allowing multiple registers to be defined in one call. // It matches all possible register declaration patterns to dispatch them to corresponding // `@reg` rule that defines a single register. + // + // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O + // projections. ( $( $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @@ -830,12 +835,12 @@ macro_rules! register { ) => { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); }; // Creates an alias register of fixed offset register `alias` with its own fields. ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path { $($fields:tt)* } ) => { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); @@ -843,7 +848,7 @@ macro_rules! register { @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); }; // Creates a register at a relative offset from a base address provider. @@ -853,7 +858,7 @@ macro_rules! register { ) => { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative $vis $name($storage) @ $base); + $crate::register!(@io_relative $name @ $base); }; // Creates an alias register of relative offset register `alias` with its own fields. @@ -865,7 +870,7 @@ macro_rules! register { $crate::register!( @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET ); - $crate::register!(@io_relative $vis $name($storage) @ $base); + $crate::register!(@io_relative $name @ $base); }; // Creates an array of registers at a fixed offset of the MMIO space. @@ -877,7 +882,7 @@ macro_rules! register { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_array $vis $name($storage) [ $size, stride = $stride ]); + $crate::register!(@io_array $name [ $size, stride = $stride ]); }; // Shortcut for contiguous array of registers (stride == size of element). @@ -886,14 +891,15 @@ macro_rules! register { { $($fields:tt)* } ) => { $crate::register!( - $(#[$attr])* $vis $name($storage) [ $size, stride = ::core::mem::size_of::<$storage>() ] + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] @ $offset { $($fields)* } ); }; // Creates an alias of register `idx` of array of registers `alias` with its own fields. ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident [ $idx:expr ] + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] { $($fields:tt)* } ) => { $crate::build_assert::static_assert!( @@ -906,7 +912,7 @@ macro_rules! register { <$alias as $crate::io::register::Register>::OFFSET + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); }; // Creates an array of registers at a relative offset from a base address provider. @@ -919,9 +925,7 @@ macro_rules! register { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!( - @io_relative_array $vis $name($storage) [ $size, stride = $stride ] @ $base + $offset - ); + $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); }; // Shortcut for contiguous array of relative registers (stride == size of element). @@ -930,7 +934,8 @@ macro_rules! register { @ $base:ident + $offset:literal { $($fields:tt)* } ) => { $crate::register!( - $(#[$attr])* $vis $name($storage) [ $size, stride = ::core::mem::size_of::<$storage>() ] + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] @ $base + $offset { $($fields)* } ); }; @@ -951,7 +956,7 @@ macro_rules! register { <$alias as $crate::io::register::Register>::OFFSET + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE ); - $crate::register!(@io_relative $vis $name($storage) @ $base); + $crate::register!(@io_relative $name @ $base); }; // Generates the bitfield for the register. @@ -977,7 +982,7 @@ macro_rules! register { }; // Implementations of fixed registers. - (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)) => { + (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { impl $crate::io::register::FixedRegister for $name {} $(#[$attr])* @@ -986,7 +991,7 @@ macro_rules! register { }; // Implementations of relative registers. - (@io_relative $vis:vis $name:ident ($storage:ty) @ $base:ident) => { + (@io_relative $name:ident @ $base:ident) => { impl $crate::io::register::WithBase for $name { type BaseFamily = $base; } @@ -995,7 +1000,7 @@ macro_rules! register { }; // Implementations of register arrays. - (@io_array $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ]) => { + (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { impl $crate::io::register::Array for $name {} impl $crate::io::register::RegisterArray for $name { @@ -1006,8 +1011,7 @@ macro_rules! register { // Implementations of relative array registers. ( - @io_relative_array $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ] - @ $base:ident + $offset:literal + @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident ) => { impl $crate::io::register::WithBase for $name { type BaseFamily = $base; diff --git a/rust/kernel/iommu/pgtable.rs b/rust/kernel/iommu/pgtable.rs index c88e38fd938a..5f9b42ca92c2 100644 --- a/rust/kernel/iommu/pgtable.rs +++ b/rust/kernel/iommu/pgtable.rs @@ -16,7 +16,6 @@ use crate::{ Bound, Device, // }, - devres::Devres, error::to_result, io::PhysAddr, prelude::*, // @@ -59,15 +58,16 @@ pub struct Config { /// # Invariants /// /// The pointer references a valid io page table. -pub struct IoPageTable<F: IoPageTableFmt> { +pub struct IoPageTable<'a, F: IoPageTableFmt> { ptr: NonNull<bindings::io_pgtable_ops>, + _dev: PhantomData<&'a Device<Bound>>, _marker: PhantomData<F>, } // SAFETY: `struct io_pgtable_ops` is not restricted to a single thread. -unsafe impl<F: IoPageTableFmt> Send for IoPageTable<F> {} +unsafe impl<F: IoPageTableFmt> Send for IoPageTable<'_, F> {} // SAFETY: `struct io_pgtable_ops` may be accessed concurrently. -unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<F> {} +unsafe impl<F: IoPageTableFmt> Sync for IoPageTable<'_, F> {} /// The format used by this page table. pub trait IoPageTableFmt: 'static { @@ -75,25 +75,10 @@ pub trait IoPageTableFmt: 'static { const FORMAT: io_pgtable_fmt; } -impl<F: IoPageTableFmt> IoPageTable<F> { - /// Create a new `IoPageTable` as a device resource. - #[inline] - pub fn new( - dev: &Device<Bound>, - config: Config, - ) -> impl PinInit<Devres<IoPageTable<F>>, Error> + '_ { - // SAFETY: Devres ensures that the value is dropped during device unbind. - Devres::new(dev, unsafe { Self::new_raw(dev, config) }) - } - +impl<'a, F: IoPageTableFmt> IoPageTable<'a, F> { /// Create a new `IoPageTable`. - /// - /// # Safety - /// - /// If successful, then the returned `IoPageTable` must be dropped before the device is - /// unbound. #[inline] - pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable<F>> { + pub fn new(dev: &'a Device<Bound>, config: Config) -> Result<IoPageTable<'a, F>> { let mut raw_cfg = bindings::io_pgtable_cfg { quirks: config.quirks, pgsize_bitmap: config.pgsize_bitmap, @@ -102,8 +87,7 @@ impl<F: IoPageTableFmt> IoPageTable<F> { coherent_walk: config.coherent_walk, tlb: &raw const NOOP_FLUSH_OPS, iommu_dev: dev.as_raw(), - // SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`. - ..unsafe { core::mem::zeroed() } + ..Zeroable::zeroed() }; // SAFETY: @@ -118,6 +102,7 @@ impl<F: IoPageTableFmt> IoPageTable<F> { // INVARIANT: We successfully created a valid page table. Ok(IoPageTable { ptr: NonNull::new(ops).ok_or(ENOMEM)?, + _dev: PhantomData, _marker: PhantomData, }) } @@ -240,7 +225,7 @@ extern "C" fn rust_tlb_flush_walk_noop( ) { } -impl<F: IoPageTableFmt> Drop for IoPageTable<F> { +impl<F: IoPageTableFmt> Drop for IoPageTable<'_, F> { fn drop(&mut self) { // SAFETY: The caller of `Self::ttbr()` promised that the page table is not live when this // destructor runs. @@ -255,7 +240,7 @@ impl IoPageTableFmt for ARM64LPAES1 { const FORMAT: io_pgtable_fmt = bindings::io_pgtable_fmt_ARM_64_LPAE_S1 as io_pgtable_fmt; } -impl IoPageTable<ARM64LPAES1> { +impl IoPageTable<'_, ARM64LPAES1> { /// Access the `ttbr` field of the configuration. /// /// This is the physical address of the page table, which may be passed to the device that diff --git a/rust/kernel/irq.rs b/rust/kernel/irq.rs index 20abd4056655..09ef1e7f853c 100644 --- a/rust/kernel/irq.rs +++ b/rust/kernel/irq.rs @@ -8,7 +8,7 @@ //! The current abstractions handle IRQ requests and handlers, i.e.: it allows //! drivers to register a handler for a given IRQ line. //! -//! C header: [`include/linux/device.h`](srctree/include/linux/interrupt.h) +//! C header: [`include/linux/interrupt.h`](srctree/include/linux/interrupt.h) /// Flags to be used when registering IRQ handlers. mod flags; diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index f425fe12f7c8..c1c6525a676a 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -5,16 +5,21 @@ //! [`ThreadedRegistration`], which allow users to register handlers for a given //! IRQ line. -use core::marker::PhantomPinned; - -use crate::alloc::Allocator; -use crate::device::{Bound, Device}; -use crate::devres::Devres; -use crate::error::to_result; -use crate::irq::flags::Flags; -use crate::prelude::*; -use crate::str::CStr; -use crate::sync::Arc; +use core::marker::{ + PhantomData, + PhantomPinned, // +}; + +use crate::{ + device::{ + Bound, + Device, // + }, + error::to_result, + irq::flags::Flags, + prelude::*, + str::CStr, +}; /// The value that can be returned from a [`Handler`] or a [`ThreadedHandler`]. #[repr(u32)] @@ -27,7 +32,7 @@ pub enum IrqReturn { } /// Callbacks for an IRQ handler. -pub trait Handler: Sync + 'static { +pub trait Handler: Sync { /// The hard IRQ handler. /// /// This is executed in interrupt context, hence all corresponding @@ -36,73 +41,20 @@ pub trait Handler: Sync + 'static { /// All work that does not necessarily need to be executed from /// interrupt context, should be deferred to a threaded handler. /// See also [`ThreadedRegistration`]. - fn handle(&self, device: &Device<Bound>) -> IrqReturn; -} - -impl<T: ?Sized + Handler + Send> Handler for Arc<T> { - fn handle(&self, device: &Device<Bound>) -> IrqReturn { - T::handle(self, device) - } -} - -impl<T: ?Sized + Handler, A: Allocator + 'static> Handler for Box<T, A> { - fn handle(&self, device: &Device<Bound>) -> IrqReturn { - T::handle(self, device) - } + fn handle(&self) -> IrqReturn; } -/// # Invariants -/// -/// - `self.irq` is the same as the one passed to `request_{threaded}_irq`. -/// - `cookie` was passed to `request_{threaded}_irq` as the cookie. It is guaranteed to be unique -/// by the type system, since each call to `new` will return a different instance of -/// `Registration`. -#[pin_data(PinnedDrop)] -struct RegistrationInner { - irq: u32, - cookie: *mut c_void, -} - -impl RegistrationInner { - fn synchronize(&self) { - // SAFETY: safe as per the invariants of `RegistrationInner` - unsafe { bindings::synchronize_irq(self.irq) }; - } -} - -#[pinned_drop] -impl PinnedDrop for RegistrationInner { - fn drop(self: Pin<&mut Self>) { - // SAFETY: - // - // Safe as per the invariants of `RegistrationInner` and: - // - // - The containing struct is `!Unpin` and was initialized using - // pin-init, so it occupied the same memory location for the entirety of - // its lifetime. - // - // Notice that this will block until all handlers finish executing, - // i.e.: at no point will &self be invalid while the handler is running. - unsafe { bindings::free_irq(self.irq, self.cookie) }; - } -} - -// SAFETY: We only use `inner` on drop, which called at most once with no -// concurrent access. -unsafe impl Sync for RegistrationInner {} - -// SAFETY: It is safe to send `RegistrationInner` across threads. -unsafe impl Send for RegistrationInner {} - /// A request for an IRQ line for a given device. /// /// # Invariants /// /// - `ìrq` is the number of an interrupt source of `dev`. -/// - `irq` has not been registered yet. +/// - `irq` has not been registered yet; this is consumed by [`Registration::new()`]. pub struct IrqRequest<'a> { - dev: &'a Device<Bound>, irq: u32, + /// Proves the device is bound at registration time and ties `'a` to the device's bound + /// lifetime, ensuring the [`Registration`] cannot outlive it. + _dev: PhantomData<&'a Device<Bound>>, } impl<'a> IrqRequest<'a> { @@ -111,12 +63,16 @@ impl<'a> IrqRequest<'a> { /// # Safety /// /// - `irq` should be a valid IRQ number for `dev`. - pub(crate) unsafe fn new(dev: &'a Device<Bound>, irq: u32) -> Self { + pub(crate) unsafe fn new(_dev: &'a Device<Bound>, irq: u32) -> Self { // INVARIANT: `irq` is a valid IRQ number for `dev`. - IrqRequest { dev, irq } + IrqRequest { + irq, + _dev: PhantomData, + } } /// Returns the IRQ number of an [`IrqRequest`]. + #[inline] pub fn irq(&self) -> u32 { self.irq } @@ -139,10 +95,18 @@ impl<'a> IrqRequest<'a> { /// [`Completion::wait_for_completion()`]: kernel::sync::Completion::wait_for_completion /// /// ``` -/// use kernel::device::{Bound, Device}; -/// use kernel::irq::{self, Flags, IrqRequest, IrqReturn, Registration}; -/// use kernel::prelude::*; -/// use kernel::sync::{Arc, Completion}; +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// Registration, +/// }, +/// prelude::*, +/// sync::Completion, +/// }; /// /// // Data shared between process and IRQ context. /// #[pin_data] @@ -153,7 +117,7 @@ impl<'a> IrqRequest<'a> { /// /// impl irq::Handler for Data { /// // Executed in IRQ context. -/// fn handle(&self, _dev: &Device<Bound>) -> IrqReturn { +/// fn handle(&self) -> IrqReturn { /// self.completion.complete_all(); /// IrqReturn::Handled /// } @@ -163,12 +127,21 @@ impl<'a> IrqRequest<'a> { /// // /// // This runs in process context and assumes `request` was previously acquired from a device. /// fn register_irq( -/// handler: impl PinInit<Data, Error>, /// request: IrqRequest<'_>, -/// ) -> Result<Arc<Registration<Data>>> { -/// let registration = Registration::new(request, Flags::SHARED, c"my_device", handler); +/// ) -> Result<Pin<KBox<Registration<'_, Data>>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// Registration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// completion <- Completion::new(), +/// }? Error), +/// ) +/// }; /// -/// let registration = Arc::pin_init(registration, GFP_KERNEL)?; +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; /// /// registration.handler().completion.wait_for_completion(); /// @@ -179,11 +152,10 @@ impl<'a> IrqRequest<'a> { /// /// # Invariants /// -/// * We own an irq handler whose cookie is a pointer to `Self`. -#[pin_data] -pub struct Registration<T: Handler> { - #[pin] - inner: Devres<RegistrationInner>, +/// * We own an irq handler registered via `request_irq` whose cookie is a pointer to `Self`. +#[pin_data(PinnedDrop)] +pub struct Registration<'a, T: Handler> { + request: IrqRequest<'a>, #[pin] handler: T, @@ -194,44 +166,46 @@ pub struct Registration<T: Handler> { _pin: PhantomPinned, } -impl<T: Handler> Registration<T> { +impl<'a, T: Handler> Registration<'a, T> { /// Registers the IRQ handler with the system for the given IRQ number. - pub fn new<'a>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn new( request: IrqRequest<'a>, flags: Flags, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<Self, Error> + 'a { + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_irq` whose cookie is a pointer to `Self`. try_pin_init!(&this in Self { handler <- handler, - inner <- Devres::new( - request.dev, - try_pin_init!(RegistrationInner { - // INVARIANT: `this` is a valid pointer to the `Registration` instance - cookie: this.as_ptr().cast::<c_void>(), - irq: { - // SAFETY: - // - The callbacks are valid for use with request_irq. - // - If this succeeds, the slot is guaranteed to be valid until the - // destructor of Self runs, which will deregister the callbacks - // before the memory location becomes invalid. - // - When request_irq is called, everything that handle_irq_callback will - // touch has already been initialized, so it's safe for the callback to - // be called immediately. - to_result(unsafe { - bindings::request_irq( - request.irq, - Some(handle_irq_callback::<T>), - flags.into_inner(), - name.as_char_ptr(), - this.as_ptr().cast::<c_void>(), - ) - })?; - request.irq - } - }) - ), + request, _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callback to be + // called immediately. + to_result(unsafe { + bindings::request_irq( + request.irq, + Some(handle_irq_callback::<T>), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::<c_void>(), + ) + })?; + }, }) } @@ -241,19 +215,25 @@ impl<T: Handler> Registration<T> { } /// Wait for pending IRQ handlers on other CPUs. - /// - /// This will attempt to access the inner [`Devres`] container. - pub fn try_synchronize(&self) -> Result { - let inner = self.inner.try_access().ok_or(ENODEV)?; - inner.synchronize(); - Ok(()) + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; } +} - /// Wait for pending IRQ handlers on other CPUs. - pub fn synchronize(&self, dev: &Device<Bound>) -> Result { - let inner = self.inner.access(dev)?; - inner.synchronize(); - Ok(()) +#[pinned_drop] +impl<T: Handler> PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `Registration::new()`. This blocks + // until all in-flight handlers complete, so no references to `self` remain after this + // returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::<Self>(self.get_unchecked_mut()).cast::<c_void>(), + ) + }; } } @@ -261,13 +241,11 @@ impl<T: Handler> Registration<T> { /// /// This function should be only used as the callback in `request_irq`. unsafe extern "C" fn handle_irq_callback<T: Handler>(_irq: i32, ptr: *mut c_void) -> c_uint { - // SAFETY: `ptr` is a pointer to `Registration<T>` set in `Registration::new` - let registration = unsafe { &*(ptr as *const Registration<T>) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::<Registration<'_, T>>(); + // SAFETY: `ptr` is a pointer to `Registration<'_, T>` set in `Registration::new()`. + let registration = unsafe { &*ptr }; - T::handle(®istration.handler, device) as c_uint + T::handle(®istration.handler) as c_uint } /// The value that can be returned from [`ThreadedHandler::handle`]. @@ -284,7 +262,7 @@ pub enum ThreadedIrqReturn { } /// Callbacks for a threaded IRQ handler. -pub trait ThreadedHandler: Sync + 'static { +pub trait ThreadedHandler: Sync { /// The hard IRQ handler. /// /// This is executed in interrupt context, hence all corresponding @@ -293,8 +271,7 @@ pub trait ThreadedHandler: Sync + 'static { /// handler, i.e. [`ThreadedHandler::handle_threaded`]. /// /// The default implementation returns [`ThreadedIrqReturn::WakeThread`]. - #[expect(unused_variables)] - fn handle(&self, device: &Device<Bound>) -> ThreadedIrqReturn { + fn handle(&self) -> ThreadedIrqReturn { ThreadedIrqReturn::WakeThread } @@ -302,27 +279,7 @@ pub trait ThreadedHandler: Sync + 'static { /// /// This is executed in process context. The kernel creates a dedicated /// `kthread` for this purpose. - fn handle_threaded(&self, device: &Device<Bound>) -> IrqReturn; -} - -impl<T: ?Sized + ThreadedHandler + Send> ThreadedHandler for Arc<T> { - fn handle(&self, device: &Device<Bound>) -> ThreadedIrqReturn { - T::handle(self, device) - } - - fn handle_threaded(&self, device: &Device<Bound>) -> IrqReturn { - T::handle_threaded(self, device) - } -} - -impl<T: ?Sized + ThreadedHandler, A: Allocator + 'static> ThreadedHandler for Box<T, A> { - fn handle(&self, device: &Device<Bound>) -> ThreadedIrqReturn { - T::handle(self, device) - } - - fn handle_threaded(&self, device: &Device<Bound>) -> IrqReturn { - T::handle_threaded(self, device) - } + fn handle_threaded(&self) -> IrqReturn; } /// A registration of a threaded IRQ handler for a given IRQ line. @@ -339,13 +296,20 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator + 'static> ThreadedHandler for Bo /// [`Mutex`](kernel::sync::Mutex) to provide interior mutability. /// /// ``` -/// use kernel::device::{Bound, Device}; -/// use kernel::irq::{ -/// self, Flags, IrqRequest, IrqReturn, ThreadedHandler, ThreadedIrqReturn, -/// ThreadedRegistration, +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// ThreadedHandler, +/// ThreadedIrqReturn, +/// ThreadedRegistration, +/// }, +/// prelude::*, +/// sync::Mutex, /// }; -/// use kernel::prelude::*; -/// use kernel::sync::{Arc, Mutex}; /// /// // Declare a struct that will be passed in when the interrupt fires. The u32 /// // merely serves as an example of some internal data. @@ -363,7 +327,7 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator + 'static> ThreadedHandler for Bo /// // This will run (in a separate kthread) if and only if /// // [`ThreadedHandler::handle`] returns [`WakeThread`], which it does by /// // default. -/// fn handle_threaded(&self, _dev: &Device<Bound>) -> IrqReturn { +/// fn handle_threaded(&self) -> IrqReturn { /// let mut data = self.value.lock(); /// *data += 1; /// IrqReturn::Handled @@ -375,13 +339,21 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator + 'static> ThreadedHandler for Bo /// // This is executing in process context and assumes that `request` was /// // previously acquired from a device. /// fn register_threaded_irq( -/// handler: impl PinInit<Data, Error>, /// request: IrqRequest<'_>, -/// ) -> Result<Arc<ThreadedRegistration<Data>>> { -/// let registration = -/// ThreadedRegistration::new(request, Flags::SHARED, c"my_device", handler); +/// ) -> Result<Pin<KBox<ThreadedRegistration<'_, Data>>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// ThreadedRegistration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// value <- kernel::new_mutex!(0), +/// }? Error), +/// ) +/// }; /// -/// let registration = Arc::pin_init(registration, GFP_KERNEL)?; +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; /// /// { /// // The data can be accessed from process context too. @@ -396,11 +368,11 @@ impl<T: ?Sized + ThreadedHandler, A: Allocator + 'static> ThreadedHandler for Bo /// /// # Invariants /// -/// * We own an irq handler whose cookie is a pointer to `Self`. -#[pin_data] -pub struct ThreadedRegistration<T: ThreadedHandler> { - #[pin] - inner: Devres<RegistrationInner>, +/// * We own an irq handler registered via `request_threaded_irq` whose cookie is a pointer to +/// `Self`. +#[pin_data(PinnedDrop)] +pub struct ThreadedRegistration<'a, T: ThreadedHandler> { + request: IrqRequest<'a>, #[pin] handler: T, @@ -411,45 +383,47 @@ pub struct ThreadedRegistration<T: ThreadedHandler> { _pin: PhantomPinned, } -impl<T: ThreadedHandler> ThreadedRegistration<T> { +impl<'a, T: ThreadedHandler> ThreadedRegistration<'a, T> { /// Registers the IRQ handler with the system for the given IRQ number. - pub fn new<'a>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`ThreadedRegistration`] or otherwise prevent + /// its [`Drop`] implementation from running. + pub unsafe fn new( request: IrqRequest<'a>, flags: Flags, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<Self, Error> + 'a { + ) -> impl PinInit<Self, Error> + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_threaded_irq` whose cookie is a pointer to `Self`. try_pin_init!(&this in Self { handler <- handler, - inner <- Devres::new( - request.dev, - try_pin_init!(RegistrationInner { - // INVARIANT: `this` is a valid pointer to the `ThreadedRegistration` instance. - cookie: this.as_ptr().cast::<c_void>(), - irq: { - // SAFETY: - // - The callbacks are valid for use with request_threaded_irq. - // - If this succeeds, the slot is guaranteed to be valid until the - // destructor of Self runs, which will deregister the callbacks - // before the memory location becomes invalid. - // - When request_threaded_irq is called, everything that the two callbacks - // will touch has already been initialized, so it's safe for the - // callbacks to be called immediately. - to_result(unsafe { - bindings::request_threaded_irq( - request.irq, - Some(handle_threaded_irq_callback::<T>), - Some(thread_fn_callback::<T>), - flags.into_inner(), - name.as_char_ptr(), - this.as_ptr().cast::<c_void>(), - ) - })?; - request.irq - } - }) - ), + request, _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_threaded_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callbacks to be + // called immediately. + to_result(unsafe { + bindings::request_threaded_irq( + request.irq, + Some(handle_threaded_irq_callback::<T>), + Some(thread_fn_callback::<T>), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::<c_void>(), + ) + })?; + }, }) } @@ -459,19 +433,25 @@ impl<T: ThreadedHandler> ThreadedRegistration<T> { } /// Wait for pending IRQ handlers on other CPUs. - /// - /// This will attempt to access the inner [`Devres`] container. - pub fn try_synchronize(&self) -> Result { - let inner = self.inner.try_access().ok_or(ENODEV)?; - inner.synchronize(); - Ok(()) + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; } +} - /// Wait for pending IRQ handlers on other CPUs. - pub fn synchronize(&self, dev: &Device<Bound>) -> Result { - let inner = self.inner.access(dev)?; - inner.synchronize(); - Ok(()) +#[pinned_drop] +impl<T: ThreadedHandler> PinnedDrop for ThreadedRegistration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `ThreadedRegistration::new()`. This + // blocks until all in-flight handlers complete, so no references to `self` remain after + // this returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::<Self>(self.get_unchecked_mut()).cast::<c_void>(), + ) + }; } } @@ -482,24 +462,22 @@ unsafe extern "C" fn handle_threaded_irq_callback<T: ThreadedHandler>( _irq: i32, ptr: *mut c_void, ) -> c_uint { - // SAFETY: `ptr` is a pointer to `ThreadedRegistration<T>` set in `ThreadedRegistration::new` - let registration = unsafe { &*(ptr as *const ThreadedRegistration<T>) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::<ThreadedRegistration<'_, T>>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; - T::handle(®istration.handler, device) as c_uint + T::handle(®istration.handler) as c_uint } /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. unsafe extern "C" fn thread_fn_callback<T: ThreadedHandler>(_irq: i32, ptr: *mut c_void) -> c_uint { - // SAFETY: `ptr` is a pointer to `ThreadedRegistration<T>` set in `ThreadedRegistration::new` - let registration = unsafe { &*(ptr as *const ThreadedRegistration<T>) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::<ThreadedRegistration<'_, T>>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; - T::handle_threaded(®istration.handler, device) as c_uint + T::handle_threaded(®istration.handler) as c_uint } diff --git a/rust/kernel/jump_label.rs b/rust/kernel/jump_label.rs index 4e974c768dbd..f54cedcb6fd5 100644 --- a/rust/kernel/jump_label.rs +++ b/rust/kernel/jump_label.rs @@ -44,6 +44,7 @@ const _: &str = include!(concat!( #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(CONFIG_JUMP_LABEL)] macro_rules! arch_static_branch { ($key:path, $keytyp:ty, $field:ident, $branch:expr) => {'my_label: { @@ -61,6 +62,17 @@ macro_rules! arch_static_branch { }}; } +#[macro_export] +#[doc(hidden)] +#[cfg(testlib)] +#[cfg(CONFIG_JUMP_LABEL)] +macro_rules! arch_static_branch { + ($key:path, $keytyp:ty, $field:ident, $branch:expr) => { + // The asm falls through until patched, which never happens on the host. + false + }; +} + #[cfg(CONFIG_JUMP_LABEL)] pub use arch_static_branch; diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index cdee5f27bd7f..91eaff8c186a 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -288,6 +288,7 @@ macro_rules! kunit_unsafe_test_suite { log: ::core::ptr::null_mut(), suite_init_err: 0, is_init: false, + status: kernel::bindings::kunit_status_KUNIT_SUCCESS, }; #[used(compiler)] diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 9512af7156df..4d5c96ddc49c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,9 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] +// // Stable since Rust 1.89.0. #![feature(generic_arg_infer)] // @@ -73,6 +76,8 @@ pub mod faux; pub mod firmware; pub mod fmt; pub mod fs; +#[cfg(CONFIG_RUST_FWCTL_ABSTRACTIONS)] +pub mod fwctl; #[cfg(CONFIG_GPU_BUDDY = "y")] pub mod gpu; #[cfg(CONFIG_I2C = "y")] @@ -82,6 +87,7 @@ pub mod id_pool; pub mod impl_flags; pub mod init; pub mod interop; +pub mod interrupt; pub mod io; pub mod ioctl; pub mod iommu; @@ -94,6 +100,7 @@ pub mod list; pub mod maple_tree; pub mod miscdevice; pub mod mm; +pub mod module; pub mod module_param; #[cfg(CONFIG_NET)] pub mod net; @@ -119,6 +126,8 @@ pub mod safety; pub mod scatterlist; pub mod security; pub mod seq_file; +#[cfg(CONFIG_RUST_SERIAL_DEV_BUS_ABSTRACTIONS)] +pub mod serdev; pub mod sizes; #[cfg(CONFIG_SOC_BUS)] pub mod soc; @@ -140,77 +149,29 @@ pub mod xarray; #[doc(hidden)] pub use bindings; pub use macros; +pub use module::{ + InPlaceModule, + Module, + ModuleMetadata, + ThisModule, // +}; pub use uapi; /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; -/// The top level entrypoint to implementing a kernel module. -/// -/// For any teardown or cleanup operations, your type may implement [`Drop`]. -pub trait Module: Sized + Sync + Send { - /// Called at module initialization time. - /// - /// Use this method to perform whatever setup or registration your module - /// should do. - /// - /// Equivalent to the `module_init` macro in the C API. - fn init(module: &'static ThisModule) -> error::Result<Self>; -} - -/// A module that is pinned and initialised in-place. -pub trait InPlaceModule: Sync + Send { - /// Creates an initialiser for the module. - /// - /// It is called when the module is loaded. - fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error>; -} - -impl<T: Module> InPlaceModule for T { - fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, error::Error> { - let initer = move |slot: *mut Self| { - let m = <Self as Module>::init(module)?; - - // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. - unsafe { slot.write(m) }; - Ok(()) - }; - - // SAFETY: On success, `initer` always fully initialises an instance of `Self`. - unsafe { pin_init::pin_init_from_closure(initer) } - } -} - -/// Metadata attached to a [`Module`] or [`InPlaceModule`]. -pub trait ModuleMetadata { - /// The name of the module as specified in the `module!` macro. - const NAME: &'static crate::str::CStr; -} - -/// Equivalent to `THIS_MODULE` in the C API. -/// -/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) -pub struct ThisModule(*mut bindings::module); +/// Dummy module type for `#[vtable]` `impl` blocks within the `kernel` crate (e.g. KUnit tests). +// The `allow` is needed since it may be unused (e.g. KUnit tests may be disabled). +#[allow(dead_code)] +struct LocalModule; -// SAFETY: `THIS_MODULE` may be used from all threads within a module. -unsafe impl Sync for ThisModule {} +impl ModuleMetadata for LocalModule { + const NAME: &'static str::CStr = c"rust_kernel"; -impl ThisModule { - /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. - /// - /// # Safety - /// - /// The pointer must be equal to the right `THIS_MODULE`. - pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { - ThisModule(ptr) - } - - /// Access the raw pointer for this module. - /// - /// It is up to the user to use it correctly. - pub const fn as_ptr(&self) -> *mut bindings::module { - self.0 - } + const THIS_MODULE: ThisModule = { + // SAFETY: `try_module_get`/`module_put` handle null module pointers gracefully. + unsafe { ThisModule::from_ptr(core::ptr::null_mut()) } + }; } #[cfg(not(testlib))] diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 406e3a028c55..0f367264ee2e 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -249,7 +249,7 @@ pub use self::arc_field::{ /// assert_eq!(list.iter().count(), 3); /// } /// -/// // Pop the items from the list using `pop_front()` and verify the content. +/// // Pop the items from the list using `pop_back()` and verify the content. /// { /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); diff --git a/rust/kernel/miscdevice.rs b/rust/kernel/miscdevice.rs index 83ce50def5ac..8d4b719bd83f 100644 --- a/rust/kernel/miscdevice.rs +++ b/rust/kernel/miscdevice.rs @@ -24,12 +24,13 @@ use crate::{ IovIterSource, // }, mm::virt::VmaNew, + module::this_module, prelude::*, seq_file::SeqFile, types::{ ForeignOwnable, Opaque, // - }, + }, // }; use core::marker::PhantomData; @@ -289,7 +290,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { /// # Safety /// /// `kiocb` must be correspond to a valid file that is associated with a - /// `MiscDeviceRegistration<T>`. `iter` must be a valid `struct iov_iter` for writing. + /// `MiscDeviceRegistration<T>`. `iter` must be a valid `struct iov_iter` for reading. unsafe extern "C" fn write_iter( kiocb: *mut bindings::kiocb, iter: *mut bindings::iov_iter, @@ -430,6 +431,7 @@ impl<T: MiscDevice> MiscdeviceVTable<T> { } else { None }, + owner: this_module::<T::OwnerModule>().as_ptr(), ..pin_init::zeroed() }; diff --git a/rust/kernel/module.rs b/rust/kernel/module.rs new file mode 100644 index 000000000000..d71370598447 --- /dev/null +++ b/rust/kernel/module.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Module-related types and helpers. + +/// The entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait Module: Sized + Sync + Send { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init(module: &'static ThisModule) -> crate::error::Result<Self>; +} + +/// A module that is pinned and initialised in-place. +pub trait InPlaceModule: Sync + Send { + /// Creates an initialiser for the module. + /// + /// It is called when the module is loaded. + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, crate::error::Error>; +} + +impl<T: Module> InPlaceModule for T { + fn init(module: &'static ThisModule) -> impl pin_init::PinInit<Self, crate::error::Error> { + let initer = move |slot: *mut Self| { + let m = <Self as Module>::init(module)?; + + // SAFETY: `slot` is valid for write per the contract with `pin_init_from_closure`. + unsafe { slot.write(m) }; + Ok(()) + }; + + // SAFETY: On success, `initer` always fully initialises an instance of `Self`. + unsafe { pin_init::pin_init_from_closure(initer) } + } +} + +/// Metadata attached to a [`Module`] or [`InPlaceModule`]. +pub trait ModuleMetadata { + /// The name of the module as specified in the `module!` macro. + const NAME: &'static crate::str::CStr; + + /// The module's `THIS_MODULE` pointer. + const THIS_MODULE: ThisModule; +} + +/// Returns a reference to the `THIS_MODULE` of the given module type. +#[inline] +pub const fn this_module<M: ModuleMetadata>() -> &'static ThisModule { + &M::THIS_MODULE +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: [`include/linux/init.h`](srctree/include/linux/init.h) +pub struct ThisModule(*mut crate::bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut crate::bindings::module) -> ThisModule { + ThisModule(ptr) + } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut crate::bindings::module { + self.0 + } +} diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs index 6541af218390..f9a14765a926 100644 --- a/rust/kernel/module_param.rs +++ b/rust/kernel/module_param.rs @@ -5,7 +5,7 @@ //! C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) use crate::prelude::*; -use crate::str::BStr; +use crate::str::{kstrtobool_bytes, BStr}; use bindings; use kernel::sync::SetOnce; @@ -105,6 +105,12 @@ impl_int_module_param!(u64); impl_int_module_param!(isize); impl_int_module_param!(usize); +impl ModuleParam for bool { + fn try_from_param_arg(arg: &BStr) -> Result<Self> { + kstrtobool_bytes(arg) + } +} + /// A wrapper for kernel parameters. /// /// This type is instantiated by the [`module!`] macro when module parameters are @@ -130,10 +136,26 @@ impl<T> ModuleParamAccess<T> { } } + /// Get a copy of the parameter value. + /// + /// Returns the value supplied at module load time, or the default value + /// if the parameter has not been set. + #[inline] + pub fn value(&self) -> T + where + T: Copy, + { + self.value.copy().unwrap_or(self.default) + } + /// Get a shared reference to the parameter value. + /// + /// Returns a reference to the value supplied at module load time, or a + /// reference to the default value if the parameter has not been set. // Note: When sysfs access to parameters are enabled, we have to pass in a // held lock guard here. - pub fn value(&self) -> &T { + #[inline] + pub fn value_ref(&self) -> &T { self.value.as_ref().unwrap_or(&self.default) } @@ -179,3 +201,4 @@ make_param_ops!(PARAM_OPS_I64, i64); make_param_ops!(PARAM_OPS_U64, u64); make_param_ops!(PARAM_OPS_ISIZE, isize); make_param_ops!(PARAM_OPS_USIZE, usize); +make_param_ops!(PARAM_OPS_BOOL, bool); diff --git a/rust/kernel/net.rs b/rust/kernel/net/mod.rs index fe415cb369d3..8ecae7577ed2 100644 --- a/rust/kernel/net.rs +++ b/rust/kernel/net/mod.rs @@ -4,3 +4,5 @@ #[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] pub mod phy; + +pub mod netlink; diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs new file mode 100644 index 000000000000..22ef3dde36fa --- /dev/null +++ b/rust/kernel/net/netlink.rs @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2026 Google LLC. + +//! Rust support for generic netlink. +//! +//! Currently only supports exposing multicast groups. +//! +//! C header: [`include/net/genetlink.h`](srctree/include/net/genetlink.h) + +use kernel::{ + alloc::{self, AllocError}, + error::to_result, + prelude::*, + transmute::AsBytes, + types::Opaque, + ThisModule, +}; + +use core::{ + mem::ManuallyDrop, + ptr::NonNull, // +}; + +/// The default netlink message size. +pub const GENLMSG_DEFAULT_SIZE: usize = bindings::GENLMSG_DEFAULT_SIZE; + +/// A wrapper around `struct sk_buff` for generic netlink messages. +/// +/// This type is intended to be specific for buffers used with netlink only, and other usecases for +/// `struct sk_buff` are out-of-scope for this abstraction. +/// +/// # Invariants +/// +/// The pointer has ownership over a valid `sk_buff`. +pub struct NetlinkSkBuff { + skb: NonNull<kernel::bindings::sk_buff>, +} + +impl NetlinkSkBuff { + /// Creates a new `NetlinkSkBuff` with the given size. + pub fn new(size: usize, flags: alloc::Flags) -> Result<NetlinkSkBuff, AllocError> { + // SAFETY: `genlmsg_new` only requires its arguments to be valid integers. + let skb = unsafe { bindings::genlmsg_new(size, flags.as_raw()) }; + let skb = NonNull::new(skb).ok_or(AllocError)?; + Ok(NetlinkSkBuff { skb }) + } + + /// Puts a generic netlink header into the `NetlinkSkBuff`. + pub fn genlmsg_put( + self, + portid: u32, + seq: u32, + family: &'static Family, + cmd: u8, + ) -> Result<GenlMsg, AllocError> { + let skb = self.skb.as_ptr(); + // SAFETY: The skb and family pointers are valid. + let hdr = unsafe { bindings::genlmsg_put(skb, portid, seq, family.as_raw(), 0, cmd) }; + let hdr = NonNull::new(hdr).ok_or(AllocError)?; + Ok(GenlMsg { skb: self, hdr }) + } +} + +impl Drop for NetlinkSkBuff { + fn drop(&mut self) { + // SAFETY: We have ownership over the `sk_buff`, so we may free it. + unsafe { bindings::nlmsg_free(self.skb.as_ptr()) } + } +} + +/// A generic netlink message being constructed. +/// +/// # Invariants +/// +/// `hdr` references the header in this netlink message. +pub struct GenlMsg { + skb: NetlinkSkBuff, + hdr: NonNull<c_void>, +} + +impl GenlMsg { + /// Puts an attribute into the message. + #[inline] + fn put<T>(&mut self, attrtype: c_int, value: &T) -> Result + where + T: ?Sized + AsBytes, + { + let skb = self.skb.skb.as_ptr(); + let len = size_of_val(value); + let ptr = core::ptr::from_ref(value).cast::<c_void>(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is + // readable and initialized for its `size_of` bytes. + to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) }) + } + + /// Puts a `u32` attribute into the message. + #[inline] + pub fn put_u32(&mut self, attrtype: c_int, value: u32) -> Result { + self.put(attrtype, &value) + } + + /// Puts a string attribute into the message. + #[inline] + pub fn put_string(&mut self, attrtype: c_int, value: &CStr) -> Result { + self.put(attrtype, value.to_bytes_with_nul()) + } + + /// Puts a flag attribute into the message. + #[inline] + pub fn put_flag(&mut self, attrtype: c_int) -> Result { + let skb = self.skb.skb.as_ptr(); + // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and a null pointer is valid + // when the length is zero. + to_result(unsafe { bindings::nla_put(skb, attrtype, 0, core::ptr::null()) }) + } + + /// Sends the generic netlink message as a multicast message. + #[inline] + pub fn multicast( + self, + family: &'static Family, + portid: u32, + group: u32, + flags: alloc::Flags, + ) -> Result { + let me = ManuallyDrop::new(self); + // SAFETY: The `skb` and `family` pointers are valid. We pass ownership of the `skb` to + // `genlmsg_multicast` by not dropping `self`. + unsafe { + bindings::genlmsg_end(me.skb.skb.as_ptr(), me.hdr.as_ptr()); + to_result(bindings::genlmsg_multicast( + family.as_raw(), + me.skb.skb.as_ptr(), + portid, + group, + flags.as_raw(), + )) + } + } +} +impl Drop for GenlMsg { + fn drop(&mut self) { + // SAFETY: The `hdr` pointer references the header of this generic netlink message. + unsafe { bindings::genlmsg_cancel(self.skb.skb.as_ptr(), self.hdr.as_ptr()) }; + } +} + +/// Flags for a generic netlink family. +struct FamilyFlags { + /// Whether the family supports network namespaces. + netnsok: bool, + /// Whether the family supports parallel operations. + parallel_ops: bool, +} + +impl FamilyFlags { + /// Converts the flags to the bitfield representation used by `genl_family`. + const fn into_bitfield(self) -> bindings::__BindgenBitfieldUnit<[u8; 1]> { + // The below shifts are verified correct by test_family_flags_bitfield() below. + // + // Although bindgen generates helpers to change bitfields based on the C headers, these + // helpers unfortunately can't be used in const context. Since `Family` needs to be filled + // out at build-time, we use this helper instead. + let mut bits = 0; + if self.netnsok { + bits |= 1 << 0; + } + if self.parallel_ops { + bits |= 1 << 1; + } + // Convert from little endian to the target's endianness. + bits = u8::from_le(bits); + // SAFETY: This bitfield is represented as an u8. + unsafe { core::mem::transmute::<u8, bindings::__BindgenBitfieldUnit<[u8; 1]>>(bits) } + } +} + +/// A generic netlink family. +#[repr(transparent)] +pub struct Family { + inner: Opaque<bindings::genl_family>, +} + +// SAFETY: The `Family` type is thread safe. +unsafe impl Sync for Family {} + +impl Family { + /// Creates a new `Family` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + /// + /// The name must be a nul-terminated string, but it is taken as `&[u8]` so that it can be used + /// more conveniently with the strings generated by bindgen. + pub const fn const_new( + module: &ThisModule, + name: &[u8], + version: u32, + mcgrps: &'static [MulticastGroup], + ) -> Family { + let n_mcgrps = mcgrps.len() as u8; + if n_mcgrps as usize != mcgrps.len() { + panic!("too many mcgrps"); + } + let mut genl_family = bindings::genl_family { + version, + _bitfield_1: FamilyFlags { + netnsok: true, + parallel_ops: true, + } + .into_bitfield(), + module: module.as_ptr(), + mcgrps: mcgrps.as_ptr().cast(), + n_mcgrps, + ..pin_init::zeroed() + }; + if CStr::from_bytes_with_nul(name).is_err() { + panic!("genl_family name not nul-terminated"); + } + if genl_family.name.len() < name.len() { + panic!("genl_family name too long"); + } + let mut i = 0; + while i < name.len() { + genl_family.name[i] = name[i]; + i += 1; + } + Family { + inner: Opaque::new(genl_family), + } + } + + /// Checks if there are any listeners for the given multicast group. + pub fn has_listeners(&self, group: u32) -> bool { + // SAFETY: The family and init_net pointers are valid. + unsafe { + bindings::genl_has_listeners(self.as_raw(), &raw mut bindings::init_net, group) != 0 + } + } + + /// Returns a raw pointer to the underlying `genl_family` structure. + pub fn as_raw(&self) -> *mut bindings::genl_family { + self.inner.get() + } +} + +/// A generic netlink multicast group. +#[repr(transparent)] +pub struct MulticastGroup { + // No Opaque because fully immutable + group: bindings::genl_multicast_group, +} + +// SAFETY: Pure data so thread safe. +unsafe impl Sync for MulticastGroup {} + +impl MulticastGroup { + /// Creates a new `MulticastGroup` instance. + /// + /// Intended to be used from const context only. Will panic if provided with invalid arguments. + pub const fn const_new(name: &CStr) -> MulticastGroup { + let mut group: bindings::genl_multicast_group = pin_init::zeroed(); + + let name = name.to_bytes_with_nul(); + if group.name.len() < name.len() { + panic!("genl_multicast_group name too long"); + } + let mut i = 0; + while i < name.len() { + group.name[i] = name[i]; + i += 1; + } + + MulticastGroup { group } + } +} + +/// A registration of a generic netlink family. +/// +/// This type represents the registration of a [`Family`]. When an instance of this type is +/// dropped, its respective generic netlink family will be unregistered from the system. +/// +/// # Invariants +/// +/// `self.family` always holds a valid reference to an initialized and registered [`Family`]. +pub struct Registration { + family: &'static Family, +} + +impl Family { + /// Registers the generic netlink family with the kernel. + pub fn register(&'static self) -> Result<Registration> { + // SAFETY: `self.as_raw()` is a valid pointer to a `genl_family` struct. + // The `genl_family` struct is static, so it will outlive the registration. + to_result(unsafe { bindings::genl_register_family(self.as_raw()) })?; + Ok(Registration { family: self }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: `self.family.as_raw()` is a valid pointer to a registered `genl_family` struct. + // The `Registration` struct ensures that `genl_unregister_family` is called exactly once + // for this family when it goes out of scope. + unsafe { bindings::genl_unregister_family(self.family.as_raw()) }; + } +} + +#[macros::kunit_tests(rust_netlink)] +mod tests { + use super::*; + + #[test] + fn test_family_flags_bitfield() { + for netnsok in [false, true] { + for parallel_ops in [false, true] { + let mut b_fam = bindings::genl_family { + ..Default::default() + }; + b_fam.set_netnsok(if netnsok { 1 } else { 0 }); + b_fam.set_parallel_ops(if parallel_ops { 1 } else { 0 }); + + let c_bitfield = FamilyFlags { + netnsok, + parallel_ops, + } + .into_bitfield(); + + // SAFETY: The bit field is stored as u8. + let b_val: u8 = unsafe { core::mem::transmute(b_fam._bitfield_1) }; + // SAFETY: The bit field is stored as u8. + let c_val: u8 = unsafe { core::mem::transmute(c_bitfield) }; + assert_eq!(b_val, c_val); + } + } + } +} diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 3ca99db5cccf..956cda573ddb 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -659,7 +659,11 @@ impl Registration { // the `drivers` slice are initialized properly. `drivers` will not be moved. // So it's just an FFI call. to_result(unsafe { - bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0) + bindings::phy_drivers_register( + drivers[0].0.get(), + drivers.len().try_into()?, + module.as_ptr(), + ) })?; // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. Ok(Registration { drivers }) @@ -800,62 +804,6 @@ impl DeviceMask { /// } /// # } /// ``` -/// -/// This expands to the following code: -/// -/// ```ignore -/// use kernel::net::phy::{self, DeviceId}; -/// use kernel::prelude::*; -/// -/// struct Module { -/// _reg: ::kernel::net::phy::Registration, -/// } -/// -/// module! { -/// type: Module, -/// name: "rust_sample_phy", -/// authors: ["Rust for Linux Contributors"], -/// description: "Rust sample PHYs driver", -/// license: "GPL", -/// } -/// -/// struct PhySample; -/// -/// #[vtable] -/// impl phy::Driver for PhySample { -/// const NAME: &'static CStr = c"PhySample"; -/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); -/// } -/// -/// const _: () = { -/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] = -/// [::kernel::net::phy::create_phy_driver::<PhySample>()]; -/// -/// impl ::kernel::Module for Module { -/// fn init(module: &'static ::kernel::ThisModule) -> Result<Self> { -/// let drivers = unsafe { &mut DRIVERS }; -/// let mut reg = ::kernel::net::phy::Registration::register( -/// module, -/// ::core::pin::Pin::static_mut(drivers), -/// )?; -/// Ok(Module { _reg: reg }) -/// } -/// } -/// }; -/// -/// const N: usize = 1; -/// -/// const TABLE: ::kernel::device_id::IdArray<::kernel::net::phy::DeviceId, (), N> = -/// ::kernel::device_id::IdArray::new_without_index([ -/// ::kernel::net::phy::DeviceId( -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0x00000001, -/// phy_id_mask: 0xffffffff, -/// }), -/// ]); -/// -/// ::kernel::module_device_table!("mdio", phydev, TABLE); -/// ``` #[macro_export] macro_rules! module_phy_driver { (@replace_expr $_t:tt $sub:expr) => {$sub}; @@ -865,12 +813,10 @@ macro_rules! module_phy_driver { }; (@device_table [$($dev:expr),+]) => { - const N: usize = $crate::module_phy_driver!(@count_devices $($dev),+); - - const TABLE: $crate::device_id::IdArray<$crate::net::phy::DeviceId, (), N> = - $crate::device_id::IdArray::new_without_index([ $(($dev,())),+, ]); - - $crate::module_device_table!("mdio", phydev, TABLE); + $crate::module_device_table!( + "mdio", $crate::net::phy::DeviceId, + TABLE, @none, [$($dev),+] + ); }; (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { diff --git a/rust/kernel/num.rs b/rust/kernel/num.rs index 8532b511384c..dbe848e30efe 100644 --- a/rust/kernel/num.rs +++ b/rust/kernel/num.rs @@ -5,6 +5,8 @@ use core::ops; pub mod bounded; +pub mod casts; + pub use bounded::*; /// Designates unsigned primitive types. diff --git a/rust/kernel/num/bounded.rs b/rust/kernel/num/bounded.rs index dafe77782d79..2a2b0a4bca5e 100644 --- a/rust/kernel/num/bounded.rs +++ b/rust/kernel/num/bounded.rs @@ -13,7 +13,10 @@ use core::{ }; use kernel::{ - num::Integer, + num::{ + Integer, + Unsigned, // + }, prelude::*, // }; @@ -174,13 +177,16 @@ fn fits_within<T: Integer>(value: T, num_bits: u32) -> bool { /// // `u8` (regardless of the passed value). /// // let _ = Bounded::<u32, 6>::from(10u8); /// -/// // Booleans can be converted into single-bit `Bounded`s. +/// // Booleans can be converted into unsigned `Bounded`s. /// /// let v = Bounded::<u64, 1>::from(false); /// assert_eq!(v.get(), 0); /// /// let v = Bounded::<u64, 1>::from(true); /// assert_eq!(v.get(), 1); +/// +/// // This does not build because `i8` is signed. +/// // let _ = Bounded::<i8, 2>::from(true); /// ``` /// /// Infallible conversions from a [`Bounded`] to a primitive integer are also supported, and @@ -203,12 +209,16 @@ fn fits_within<T: Integer>(value: T, num_bits: u32) -> bool { /// let _v = Bounded::<u32, 10>::new::<10>(); /// // assert_eq!(u8::from(_v), 10); /// -/// // Single-bit `Bounded`s can be converted into a boolean. +/// // Unsigned single-bit `Bounded`s can be converted into a boolean. /// let v = Bounded::<u8, 1>::new::<1>(); /// assert_eq!(bool::from(v), true); /// /// let v = Bounded::<u8, 1>::new::<0>(); /// assert_eq!(bool::from(v), false); +/// +/// // This does not build because `i8` is signed. +/// // let v = Bounded::<i8, 1>::new::<-1>(); +/// // let _ = bool::from(v); /// ``` /// /// Fallible conversions from any primitive integer to any [`Bounded`] are also supported using the @@ -485,13 +495,45 @@ where /// assert_eq!(v_shifted.get(), 0xff); /// ``` pub fn shr<const SHIFT: u32, const RES: u32>(self) -> Bounded<T, RES> { - const { assert!(RES + SHIFT >= N) } + const_assert!(SHIFT < T::BITS); + const_assert!(RES + SHIFT >= N); // SAFETY: We shift the value right by `SHIFT`, reducing the number of bits needed to // represent the shifted value by as much, and just asserted that `RES >= N - SHIFT`. unsafe { Bounded::__new(self.0 >> SHIFT) } } + /// Right-shifts `self` by `SHIFT` if that loses no set bits, and returns the result as a + /// `Bounded<_, RES>`, where `RES >= N - SHIFT`. + /// + /// Returns [`None`] if any of the `SHIFT` least significant bits of `self` is set. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::Bounded; + /// + /// let v = Bounded::<u32, 16>::new::<0xff00>(); + /// let v_shifted: Option<Bounded<u32, 8>> = v.shr_exact::<8, _>(); + /// + /// assert_eq!(v_shifted.map(|v| v.get()), Some(0xff)); + /// + /// // A set bit would be shifted out. + /// let v = Bounded::<u32, 16>::new::<0xff01>(); + /// let v_shifted: Option<Bounded<u32, 8>> = v.shr_exact::<8, _>(); + /// + /// assert!(v_shifted.is_none()); + /// ``` + #[inline] + pub fn shr_exact<const SHIFT: u32, const RES: u32>(self) -> Option<Bounded<T, RES>> { + let shifted = self.shr::<SHIFT, RES>(); + if shifted.get() << SHIFT == self.0 { + Some(shifted) + } else { + None + } + } + /// Left-shifts `self` by `SHIFT` and returns the result as a `Bounded<_, RES>`, where `RES >= /// N + SHIFT`. /// @@ -506,7 +548,7 @@ where /// assert_eq!(v_shifted.get(), 0xff00); /// ``` pub fn shl<const SHIFT: u32, const RES: u32>(self) -> Bounded<T, RES> { - const { assert!(RES >= N + SHIFT) } + const_assert!(RES >= N + SHIFT); // SAFETY: We shift the value left by `SHIFT`, augmenting the number of bits needed to // represent the shifted value by as much, and just asserted that `RES >= N + SHIFT`. @@ -1077,31 +1119,33 @@ impl_into_primitive!( i8 i16 i32 i64 isize ); -// Single-bit `Bounded`s can be converted from/to a boolean. +// Unsigned single-bit `Bounded`s can be converted to a boolean. impl<T> From<Bounded<T, 1>> for bool where - T: Integer + Zeroable, + T: Integer<Signedness = Unsigned> + Zeroable, { fn from(value: Bounded<T, 1>) -> Self { value.get() != Zeroable::zeroed() } } +// Booleans can be converted to unsigned `Bounded`s. + impl<T, const N: u32> From<bool> for Bounded<T, N> where - T: Integer + From<bool>, + T: Integer<Signedness = Unsigned> + From<bool>, { fn from(value: bool) -> Self { - // SAFETY: A boolean can be represented using a single bit, and thus fits within any - // integer type for any `N` > 0. + // SAFETY: A boolean is represented by `0` or `1`, so it fits within any valid unsigned + // `Bounded` width. unsafe { Self::__new(T::from(value)) } } } impl<T> Bounded<T, 1> where - T: Integer + Zeroable, + T: Integer<Signedness = Unsigned> + Zeroable, { /// Converts this [`Bounded`] into a [`bool`]. /// diff --git a/rust/kernel/num/casts.rs b/rust/kernel/num/casts.rs new file mode 100644 index 000000000000..7e6c7dec747d --- /dev/null +++ b/rust/kernel/num/casts.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Helpers for performing lossless integer casts. +//! +//! The `as` keyword can be used to perform casts between integer types, but it unfortunately makes +//! no distinction between casts that are lossless, and casts from a larger type into a smaller one +//! that might silently strip data away. Thus, its use in the kernel is discouraged in favor of +//! [`From`] implementations. +//! +//! Conversely, there are casts that are lossless depending on the build architecture (such as +//! casting [`usize`] to [`u64`] on 32 or 64 bit archs), but not supported by [`From`] +//! implementations in the standard library because they are not portable. It does however make +//! sense for the kernel to support these, if only for code that is architecture-specific. +//! +//! This module provides ways to perform such conversions safely: +//! +//! - A series of const functions (e.g. [`usize_as_u64`]) supporting safe conversions in const +//! context. Conversions supported by [`From`] implementations in the standard library are also +//! covered as the [`From`] trait cannot be used in const context. +//! - Two extension traits, [`FromSafeCast`] and [`IntoSafeCast`], providing conversion methods +//! similar to [`From`] and [`Into`] for conversions that are safe to perform in the kernel, but +//! not supported by the standard library. +//! - Another series of const functions (e.g. [`u64_into_u8`]) supporting the conversion of a const +//! value from a larger type into a smaller one, provided the value fits into the destination +//! type. This is useful if a constant is defined as a larger type, but needs to be used as a +//! smaller one. +//! - An [`arch`] sub-module, defining more conversion functions that are only guaranteed to be +//! lossless for a given pointer size. These can only be used in code that is specific to a +//! given pointer size. +//! +//! # Examples +//! +//! ``` +//! use kernel::num::casts::{self, FromSafeCast, IntoSafeCast}; +//! +//! // Conversion from const context. +//! const USIZED_CONST: usize = casts::u8_as_usize(255u8); +//! +//! // Non-const conversions. +//! let a = u64::from_safe_cast(4096usize); +//! let b: u64 = 4096usize.into_safe_cast(); +//! ``` + +use crate::prelude::*; + +/// Implements safe `as` conversion functions from a given type into a series of target types. +/// +/// These functions can be used in place of `as`, with the guarantee that they will be lossless. +macro_rules! impl_safe_as { + ($from:ty as { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Losslessly converts a [`", + ::core::stringify!($from), + "`] into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This conversion is allowed as it is always lossless. Prefer this over the `as` + /// keyword to ensure no lossy casts are performed. + /// + /// This is for use from a `const` context. For non `const` use, prefer the + /// [`FromSafeCast`] and [`IntoSafeCast`] traits. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_as_", + ::core::stringify!($into), + "(1", + ::core::stringify!($from), + "), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _as_ $into>](value: $from) -> $into { + $crate::static_assert!(size_of::<$into>() >= size_of::<$from>()); + + value as $into + } + } + )* + }; +} + +// Valid `Into` transformations. +impl_safe_as!(u8 as { u16, u32, u64, usize }); +impl_safe_as!(u16 as { u32, u64, usize }); +impl_safe_as!(u32 as { u64 }); +// A `usize` fits into a `u64` on all supported platforms. +impl_safe_as!(usize as { u64 }); +// A `u32` fits into a `usize` on all supported platforms. +impl_safe_as!(u32 as { usize }); + +/// Extension trait providing guaranteed lossless cast to [`Self`] from `T`. +/// +/// The standard library's [`From`] implementations do not cover conversions that are not portable +/// or future-proof. For instance, even though it is safe today, [`From<usize>`] is not implemented +/// for [`u64`] because of the possibility of needing to support larger-than-64bit architectures in +/// the future. +/// +/// The workaround is to either deal with the error handling of [`TryFrom`] for an operation that +/// technically cannot fail, or to use the `as` keyword, which can silently strip data if the +/// destination type is smaller than the source. +/// +/// Both options are hardly acceptable for the kernel. It is also a much more architecture +/// dependent environment, supporting only 32 and 64 bit architectures, with some modules +/// explicitly depending on a specific bus width that could greatly benefit from infallible +/// conversion operations. +/// +/// Thus this extension trait that provides, for all architectures supported by the kernel, +/// conversion methods between types for which such a cast is lossless. +/// +/// In other words, this trait is implemented if, for all supported targets and with `t: T`, the +/// `t as Self` operation is completely lossless. +/// +/// Prefer this over the `as` keyword to guarantee that no lossy casts are performed. +/// +/// If you need to perform a conversion in `const` context, use [`u32_as_usize`], [`usize_as_u64`], +/// etc. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::FromSafeCast; +/// +/// assert_eq!(usize::from_safe_cast(0xf00u32), 0xf00usize); +/// ``` +pub trait FromSafeCast<T> { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast(value: T) -> Self; +} + +// A `usize` fits into a `u64` on all supported platforms. +impl FromSafeCast<usize> for u64 { + #[inline] + fn from_safe_cast(value: usize) -> Self { + usize_as_u64(value) + } +} + +// A `u32` fits into a `usize` on all supported platforms. +impl FromSafeCast<u32> for usize { + #[inline] + fn from_safe_cast(value: u32) -> Self { + u32_as_usize(value) + } +} + +/// Counterpart to the [`FromSafeCast`] trait, i.e. this trait is to [`FromSafeCast`] what [`Into`] +/// is to [`From`]. +/// +/// See the documentation of [`FromSafeCast`] for the motivation. +/// +/// # Examples +/// +/// ``` +/// use kernel::num::casts::IntoSafeCast; +/// +/// assert_eq!(0xf00usize, 0xf00u32.into_safe_cast()); +/// ``` +pub trait IntoSafeCast<T> { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast(self) -> T; +} + +/// Reverse operation for types implementing [`FromSafeCast`]. +impl<S, T> IntoSafeCast<T> for S +where + T: FromSafeCast<S>, +{ + #[inline] + fn into_safe_cast(self) -> T { + T::from_safe_cast(self) + } +} + +/// Implements lossless conversion of a constant from a larger type into a smaller one. +macro_rules! impl_const_into { + ($from:ty => { $($into:ty),* }) => { + $( + $crate::macros::paste! { + #[doc = ::core::concat!( + "Performs a build-time safe conversion of a [`", + ::core::stringify!($from), + "`] constant value into a [`", + ::core::stringify!($into), + "`].")] + /// + /// This checks at compile-time that the conversion is lossless, and triggers a build + /// error if it isn't. + /// + /// # Examples + /// + /// ``` + /// use kernel::num::casts; + /// + /// // Succeeds because the value of the source fits into the destination's type. + #[doc = ::core::concat!( + "assert_eq!(casts::", + ::core::stringify!($from), + "_into_", + ::core::stringify!($into), + "::<1", + ::core::stringify!($from), + ">(), 1", + ::core::stringify!($into), + ");")] + /// ``` + #[inline] + pub const fn [<$from _into_ $into>]<const N: $from>() -> $into { + // Make sure that the target type is smaller than the source one. + $crate::static_assert!($from::BITS >= $into::BITS); + // CAST: we statically enforced above that `$from` is larger than `$into`, so the + // `as` conversion will be lossless. + $crate::const_assert!(N >= $into::MIN as $from && N <= $into::MAX as $from); + + N as $into + } + } + )* + }; +} + +impl_const_into!(usize => { u8, u16, u32 }); +impl_const_into!(u64 => { u8, u16, u32 }); +impl_const_into!(u32 => { u8, u16 }); +impl_const_into!(u16 => { u8 }); + +/// Conversions that are only lossless for the current architecture. +/// +/// # Portability +/// +/// Callers of this module become dependent on the setting of `CONFIG_64BIT`. Use with caution, and +/// never in code that is portable across pointer sizes. +pub mod arch { + /// Trait identical to [`FromSafeCast`](super::FromSafeCast), but for conversions that are not + /// available on all architectures. + pub trait FromSafeCastArch<T> { + /// Create a [`Self`] from `value`. This operation is guaranteed to be lossless. + fn from_safe_cast_arch(value: T) -> Self; + } + + /// Trait identical to [`IntoSafeCast`](super::IntoSafeCast), but for conversions that are not + /// available on all architectures. + pub trait IntoSafeCastArch<T> { + /// Convert `self` into a `T`. This operation is guaranteed to be lossless. + fn into_safe_cast_arch(self) -> T; + } + + /// Reverse operation for types implementing [`FromSafeCastArch`]. + impl<S, T> IntoSafeCastArch<T> for S + where + T: FromSafeCastArch<S>, + { + #[inline] + fn into_safe_cast_arch(self) -> T { + T::from_safe_cast_arch(self) + } + } + + /// A [`u64`] fits into a [`usize`] on 64-bit platforms. + #[cfg(CONFIG_64BIT)] + #[inline] + pub const fn u64_as_usize(value: u64) -> usize { + value as usize + } + + #[cfg(CONFIG_64BIT)] + impl FromSafeCastArch<u64> for usize { + #[inline] + fn from_safe_cast_arch(value: u64) -> Self { + u64_as_usize(value) + } + } + + /// A [`usize`] fits into a [`u32`] on 32-bit platforms. + #[cfg(not(CONFIG_64BIT))] + #[inline] + pub const fn usize_as_u32(value: usize) -> u32 { + value as u32 + } + + #[cfg(not(CONFIG_64BIT))] + impl FromSafeCastArch<usize> for u32 { + #[inline] + fn from_safe_cast_arch(value: usize) -> Self { + usize_as_u32(value) + } + } +} diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 58b20c367f99..d0318f62afd7 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -25,10 +25,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); - - fn index(&self) -> usize { - self.0.data as usize - } } impl DeviceId { @@ -53,13 +49,7 @@ impl DeviceId { /// Create an OF `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! of_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::of::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("of", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("of", $crate::of::DeviceId, $($tt)*); }; } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 5071cae6543f..3ec897709e89 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -25,6 +25,7 @@ use crate::{ use core::{ marker::PhantomData, mem::offset_of, + num::NonZero, ptr::{ addr_of_mut, NonNull, // @@ -43,15 +44,16 @@ pub use self::id::{ pub use self::io::{ Bar, ConfigSpace, - ConfigSpaceKind, ConfigSpaceSize, + DevresBar, Extended, Normal, // }; pub use self::irq::{ IrqType, IrqTypes, - IrqVector, // + IrqVector, + IrqVectorRegistration, // }; /// An adapter for the registration of PCI drivers. @@ -86,7 +88,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__pci_register_driver(pdrv.get(), module.0, name.as_char_ptr()) + bindings::__pci_register_driver(pdrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -110,7 +112,11 @@ impl<T: Driver> Adapter<T> { // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; - let info = T::ID_TABLE.info(id.index()); + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>` or + // `pci_device_id_any` which has 0 as driver_data. It can also come from dynamic IDs, which + // will ensure that `driver_data` exists in `T::ID_TABLE`. + let info = unsafe { id.info_unchecked_opt::<T::IdInfo>() }; from_result(|| { let data = T::probe(pdev, info); @@ -233,10 +239,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// `IdTable` type for PCI. @@ -245,14 +247,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a PCI `IdTable` with its alias for modpost. #[macro_export] macro_rules! pci_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::pci::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("pci", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("pci", $crate::pci::DeviceId, $($tt)*); }; } @@ -267,7 +263,6 @@ macro_rules! pci_device_table { /// /// kernel::pci_device_table!( /// PCI_TABLE, -/// MODULE_PCI_TABLE, /// <MyDriver as pci::Driver>::IdInfo, /// [ /// ( @@ -284,7 +279,7 @@ macro_rules! pci_device_table { /// /// fn probe<'bound>( /// _pdev: &'bound pci::Device<Core<'_>>, -/// _id_info: &'bound Self::IdInfo, +/// _id_info: Option<&'bound Self::IdInfo>, /// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { /// Err(ENODEV) /// } @@ -313,7 +308,7 @@ pub trait Driver { /// attempt to initialize the device here. fn probe<'bound>( dev: &'bound Device<device::Core<'_>>, - id_info: &'bound Self::IdInfo, + id_info: Option<&'bound Self::IdInfo>, ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// PCI driver unbind. @@ -453,6 +448,18 @@ impl Device { } impl<'a> Device<device::Core<'a>> { + /// Returns the total number of VFs, or [`None`] if SR-IOV is not available. + #[inline] + pub fn sriov_get_totalvfs(&self) -> Option<NonZero<u16>> { + // SAFETY: `self.as_raw()` is a valid pointer to a `struct pci_dev`. + let total_vfs = unsafe { bindings::pci_sriov_get_totalvfs(self.as_raw()) }; + + // CAST: The C function returns `unsigned int`, but the value originates + // from TotalVFs/driver_max_VFs (which are defined as `u16`), so this cast + // cannot truncate. + NonZero::new(total_vfs as u16) + } + /// Enable memory resources for this device. pub fn enable_device_mem(&self) -> Result { // SAFETY: `self.as_raw` is guaranteed to be a pointer to a valid `struct pci_dev`. @@ -482,11 +489,13 @@ impl<'a> crate::dma::Device<'a> for Device<device::Core<'a>> {} // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::pci_dev_get(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) } diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 0461e01aaa20..953e16735c6e 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -6,19 +6,22 @@ use super::Device; use crate::{ bindings, device, - devres::Devres, + devres::DevresLt, io::{ - Io, + IoBackend, + IoBase, IoCapable, - IoKnownSize, Mmio, - MmioRaw, // + MmioBackend, + MmioRaw, + Region, // }, - prelude::*, // -}; -use core::{ - marker::PhantomData, - ops::Deref, // + prelude::*, + ptr::KnownSize, + types::{ + CovariantForLt, + ForLt, // + }, // }; /// Represents the size of a PCI configuration space. @@ -46,68 +49,95 @@ impl ConfigSpaceSize { } } -/// Marker type for normal (256-byte) PCI configuration space. -pub struct Normal; +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; -/// Marker type for extended (4096-byte) PCIe configuration space. -pub struct Extended; +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; -/// Trait for PCI configuration space size markers. +/// A view of PCI configuration space of a device. +/// +/// Provides typed read and write accessors for configuration registers +/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. +/// +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). /// -/// This trait is implemented by [`Normal`] and [`Extended`] to provide -/// compile-time knowledge of the configuration space size. -pub trait ConfigSpaceKind { - /// The size of this configuration space in bytes. - const SIZE: usize; +/// # Invariants +/// +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { + pub(crate) pdev: &'a Device<device::Bound>, + ptr: *mut T, } -impl ConfigSpaceKind for Normal { - const SIZE: usize = 256; +impl<T: ?Sized> Copy for ConfigSpace<'_, T> {} +impl<T: ?Sized> Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } } -impl ConfigSpaceKind for Extended { - const SIZE: usize = 4096; -} +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Send for ConfigSpace<'_, T> {} -/// The PCI configuration space of a device. -/// -/// Provides typed read and write accessors for configuration registers -/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. -/// -/// The generic parameter `S` indicates the maximum size of the configuration space. -/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for -/// 4096-byte PCIe extended configuration space (default). -pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { - pub(crate) pdev: &'a Device<device::Bound>, - _marker: PhantomData<S>, +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl<T: ?Sized + Sync> Sync for ConfigSpace<'_, T> {} + +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } + } } /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { - unsafe fn io_read(&self, address: usize) -> $ty { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + let mut val: $ty = 0; // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) }; - + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; val } - unsafe fn io_write(&self, value: $ty, address: usize) { + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) }; + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; } } }; @@ -118,24 +148,16 @@ impl_config_space_io_capable!(u8, pci_read_config_byte, pci_write_config_byte); impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { - /// Returns the base address of the I/O region. It is always 0 for configuration space. - #[inline] - fn addr(&self) -> usize { - 0 - } +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; + type Target = T; - /// Returns the maximum size of the configuration space. #[inline] - fn maxsize(&self) -> usize { - self.pdev.cfg_size().into_raw() + fn as_view(self) -> ConfigSpace<'a, T> { + self } } -impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { - const MIN_SIZE: usize = S::SIZE; -} - /// A PCI BAR to perform I/O-Operations on. /// /// I/O backend assumes that the device is little-endian and will automatically @@ -147,10 +169,23 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { /// memory mapped PCI BAR and its size. pub struct Bar<'a, const SIZE: usize = 0> { pdev: &'a Device<device::Bound>, - io: MmioRaw<SIZE>, + io: MmioRaw<crate::io::Region<SIZE>>, num: i32, } +impl<const SIZE: usize> ForLt for Bar<'static, SIZE> { + type Of<'a> = Bar<'a, SIZE>; +} + +// SAFETY: `Bar<'a, SIZE>` is covariant over `'a`; it holds `&'a Device<Bound>`, +// which is covariant. +unsafe impl<const SIZE: usize> CovariantForLt for Bar<'static, SIZE> {} + +/// A device-managed PCI BAR mapping. +/// +/// See [`Bar::into_devres`]. +pub type DevresBar<const SIZE: usize = 0> = DevresLt<Bar<'static, SIZE>>; + impl<'a, const SIZE: usize> Bar<'a, SIZE> { pub(super) fn new( pdev: &'a Device<device::Bound>, @@ -187,7 +222,7 @@ impl<'a, const SIZE: usize> Bar<'a, SIZE> { return Err(ENOMEM); } - let io = match MmioRaw::new(ioptr, len as usize) { + let io = match MmioRaw::new_region(ioptr, len as usize) { Ok(io) => io, Err(err) => { // SAFETY: @@ -223,15 +258,13 @@ impl<'a, const SIZE: usize> Bar<'a, SIZE> { /// Consume the `Bar` and register it as a device-managed resource. /// - /// The returned `Devres<Bar<'static, SIZE>>` can outlive the original lifetime `'a`. Access - /// to the BAR is revoked when the device is unbound. - pub fn into_devres(self) -> Result<Devres<Bar<'static, SIZE>>> { - // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `Bar` does not - // actually outlive the device -- access is revoked and the resource is released when the - // device is unbound. - let bar: Bar<'static, SIZE> = unsafe { core::mem::transmute(self) }; - let pdev = bar.pdev; - Devres::new(pdev.as_ref(), bar) + /// The returned [`DevresBar`] can outlive the original borrow and be stored in driver data. + /// Access to the BAR is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result<DevresBar<SIZE>> { + let pdev = self.pdev; + // SAFETY: `Bar` only holds a reference to the device and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(pdev.as_ref(), self) } } } @@ -249,12 +282,14 @@ impl<const SIZE: usize> Drop for Bar<'_, SIZE> { } } -impl<const SIZE: usize> Deref for Bar<'_, SIZE> { - type Target = Mmio<SIZE>; +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region<SIZE>; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } @@ -289,23 +324,25 @@ impl Device<device::Bound> { } } - /// Return an initialized normal (256-byte) config space object. + /// Return a view of the normal (256-byte) config space. pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), } } - /// Return an initialized extended (4096-byte) config space object. + /// Return a view of the extended (4096-byte) config space. pub fn config_space_extended<'a>(&'a self) -> Result<ConfigSpace<'a, Extended>> { if self.cfg_size() != ConfigSpaceSize::Extended { return Err(EINVAL); } + // INVARIANT: null is aligned and we just checked the `cfg_size`. Ok(ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), }) } } diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs index d9230e105541..6741046ec1c0 100644 --- a/rust/kernel/pci/irq.rs +++ b/rust/kernel/pci/irq.rs @@ -7,17 +7,11 @@ use crate::{ bindings, device, device::Bound, - devres, error::to_result, - irq::{ - self, - IrqRequest, // - }, - prelude::*, - str::CStr, - sync::aref::ARef, // + irq::IrqRequest, + prelude::*, // }; -use core::ops::RangeInclusive; +use core::num::NonZero; /// IRQ type flags for PCI interrupt allocation. #[derive(Debug, Clone, Copy)] @@ -39,6 +33,16 @@ impl IrqType { IrqType::MsiX => bindings::PCI_IRQ_MSIX, } } + + /// Construct from raw value. + #[inline] + const fn from_raw(raw: u32) -> Self { + match raw { + bindings::PCI_IRQ_MSIX => IrqType::MsiX, + bindings::PCI_IRQ_MSI => IrqType::Msi, + _ => IrqType::Intx, + } + } } /// Set of IRQ types that can be used for PCI interrupt allocation. @@ -71,148 +75,113 @@ impl IrqTypes { } } -/// Represents an allocated IRQ vector for a specific PCI device. +/// A resolved IRQ vector from a PCI interrupt vector allocation. /// -/// This type ties an IRQ vector to the device it was allocated for, -/// ensuring the vector is only used with the correct device. -#[derive(Clone, Copy)] +/// Created by [`IrqVectorRegistration::index`]. Convert to [`IrqRequest`] via [`From`] to register +/// a handler with [`irq::Registration::new`](crate::irq::Registration::new). pub struct IrqVector<'a> { - dev: &'a Device<Bound>, - index: u32, + request: IrqRequest<'a>, + reg: &'a IrqVectorRegistration<'a>, } impl<'a> IrqVector<'a> { - /// Creates a new [`IrqVector`] for the given device and index. + /// Creates a new [`IrqVector`] with an already resolved [`IrqRequest`]. /// /// # Safety /// - /// - `index` must be a valid IRQ vector index for `dev`. - /// - `dev` must point to a [`Device`] that has successfully allocated IRQ vectors. - unsafe fn new(dev: &'a Device<Bound>, index: u32) -> Self { - Self { dev, index } + /// `request` must have been resolved from `reg`. + #[inline] + unsafe fn new(request: IrqRequest<'a>, reg: &'a IrqVectorRegistration<'a>) -> Self { + Self { request, reg } } - /// Returns the raw vector index. - fn index(&self) -> u32 { - self.index + /// Returns the [`IrqVectorRegistration`] this vector was derived from. + #[inline] + pub fn vectors(&self) -> &'a IrqVectorRegistration<'a> { + self.reg } -} -impl<'a> TryInto<IrqRequest<'a>> for IrqVector<'a> { - type Error = Error; + /// Returns the interrupt type the PCI core selected for this vector's allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + self.reg.irq_type() + } +} - fn try_into(self) -> Result<IrqRequest<'a>> { - // SAFETY: `self.as_raw` returns a valid pointer to a `struct pci_dev`. - let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), self.index()) }; - if irq < 0 { - return Err(crate::error::Error::from_errno(irq)); - } - // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. - Ok(unsafe { IrqRequest::new(self.dev.as_ref(), irq as u32) }) +impl<'a> From<IrqVector<'a>> for IrqRequest<'a> { + #[inline] + fn from(vector: IrqVector<'a>) -> Self { + vector.request } } -/// Represents an IRQ vector allocation for a PCI device. +/// An allocation of PCI interrupt vectors for a device. /// -/// This type ensures that IRQ vectors are properly allocated and freed by -/// tying the allocation to the lifetime of this registration object. +/// This type owns the vector allocation; dropping it frees the vectors. IRQ handlers borrow from +/// this registration and must be dropped before it is. /// /// # Invariants /// -/// The [`Device`] has successfully allocated IRQ vectors. -struct IrqVectorRegistration { - dev: ARef<Device>, +/// `dev` has an allocation of `len` interrupt vectors. +pub struct IrqVectorRegistration<'a> { + dev: &'a Device<Bound>, + len: NonZero<usize>, } -impl IrqVectorRegistration { - /// Allocate and register IRQ vectors for the given PCI device. +impl<'a> IrqVectorRegistration<'a> { + /// Returns the number of allocated vectors. /// - /// Allocates IRQ vectors and registers them with devres for automatic cleanup. - /// Returns a range of valid IRQ vectors. - fn register<'a>( - dev: &'a Device<Bound>, - min_vecs: u32, - max_vecs: u32, - irq_types: IrqTypes, - ) -> Result<RangeInclusive<IrqVector<'a>>> { - // SAFETY: - // - `dev.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` - // by the type invariant of `Device`. - // - `pci_alloc_irq_vectors` internally validates all other parameters - // and returns error codes. - let ret = unsafe { - bindings::pci_alloc_irq_vectors(dev.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) - }; - - to_result(ret)?; - let count = ret as u32; + /// This is at least the `min_vecs` that [`Device::alloc_irq_vectors`] was asked for. + #[inline] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.len.get() + } - // SAFETY: - // - `pci_alloc_irq_vectors` returns the number of allocated vectors on success. - // - Vectors are 0-based, so valid indices are [0, count-1]. - // - `pci_alloc_irq_vectors` guarantees `count >= min_vecs > 0`, so both `0` and - // `count - 1` are valid IRQ vector indices for `dev`. - let range = unsafe { IrqVector::new(dev, 0)..=IrqVector::new(dev, count - 1) }; + /// Returns the interrupt type the PCI core selected for this allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + IrqType::from_raw(unsafe { bindings::pci_irq_type(self.dev.as_raw()) }) + } - // INVARIANT: The IRQ vector allocation for `dev` above was successful. - let irq_vecs = Self { dev: dev.into() }; - devres::register(dev.as_ref(), irq_vecs, GFP_KERNEL)?; + /// Returns the [`IrqVector`] at `index`. + /// + /// Returns [`EINVAL`] if the `index` is out of bounds for the length reported by + /// [`Self::len()`]. + #[inline] + pub fn index(&self, index: usize) -> Result<IrqVector<'_>> { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index as u32) }; + if irq < 0 { + return Err(Error::from_errno(irq)); + } - Ok(range) + // SAFETY: `irq` is a valid IRQ number for `self.dev`, resolved from this registration. + Ok(unsafe { IrqVector::new(IrqRequest::new(self.dev.as_ref(), irq as u32), self) }) } } -impl Drop for IrqVectorRegistration { +impl Drop for IrqVectorRegistration<'_> { + #[inline] fn drop(&mut self) { - // SAFETY: - // - By the type invariant, `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. - // - `self.dev` has successfully allocated IRQ vectors. + // SAFETY: By the type invariant, `self.dev.as_raw()` is a valid pointer to a + // `struct pci_dev` that has successfully allocated IRQ vectors. unsafe { bindings::pci_free_irq_vectors(self.dev.as_raw()) }; } } impl Device<device::Bound> { - /// Returns a [`kernel::irq::Registration`] for the given IRQ vector. - pub fn request_irq<'a, T: crate::irq::Handler + 'static>( - &'a self, - vector: IrqVector<'a>, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::Registration<T>, Error> + 'a { - pin_init::pin_init_scope(move || { - let request = vector.try_into()?; - - Ok(irq::Registration::<T>::new(request, flags, name, handler)) - }) - } - - /// Returns a [`kernel::irq::ThreadedRegistration`] for the given IRQ vector. - pub fn request_threaded_irq<'a, T: crate::irq::ThreadedHandler + 'static>( - &'a self, - vector: IrqVector<'a>, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::ThreadedRegistration<T>, Error> + 'a { - pin_init::pin_init_scope(move || { - let request = vector.try_into()?; - - Ok(irq::ThreadedRegistration::<T>::new( - request, flags, name, handler, - )) - }) - } - - /// Allocate IRQ vectors for this PCI device with automatic cleanup. + /// Allocate IRQ vectors for this PCI device. /// /// Allocates between `min_vecs` and `max_vecs` interrupt vectors for the device. /// The allocation will use MSI-X, MSI, or INTx interrupts based on the `irq_types` /// parameter and hardware capabilities. When multiple types are specified, the kernel /// will try them in order of preference: MSI-X first, then MSI, then INTx interrupts. /// - /// The allocated vectors are automatically freed when the device is unbound, using the - /// devres (device resource management) system. + /// The allocated vectors are freed when the returned [`IrqVectorRegistration`] is dropped. + /// Use [`IrqVectorRegistration::index`] to obtain an [`IrqVector`] for a given vector + /// index. /// /// # Arguments /// @@ -222,8 +191,8 @@ impl Device<device::Bound> { /// /// # Returns /// - /// Returns a range of IRQ vectors that were successfully allocated, or an error if the - /// allocation fails or cannot meet the minimum requirement. + /// Returns the IRQ vector registration, or an error if `min_vecs` vectors cannot be + /// allocated. /// /// # Examples /// @@ -246,7 +215,20 @@ impl Device<device::Bound> { min_vecs: u32, max_vecs: u32, irq_types: IrqTypes, - ) -> Result<RangeInclusive<IrqVector<'_>>> { - IrqVectorRegistration::register(self, min_vecs, max_vecs, irq_types) + ) -> Result<IrqVectorRegistration<'_>> { + // SAFETY: + // - `self.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` + // by the type invariant of `Device`. + // - `pci_alloc_irq_vectors` internally validates all other parameters + // and returns error codes. + let ret = unsafe { + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + }; + to_result(ret)?; + + let len = NonZero::new(ret as usize).ok_or(EINVAL)?; + + // INVARIANT: `pci_alloc_irq_vectors()` allocated `len` vectors for `self`. + Ok(IrqVectorRegistration { dev: self, len }) } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 9b362e0495d3..ac0a012ae1bb 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -17,10 +17,7 @@ use crate::{ from_result, to_result, // }, - io::{ - mem::IoRequest, - Resource, // - }, + io::Resource, irq::{ self, IrqRequest, // @@ -31,6 +28,9 @@ use crate::{ ThisModule, // }; +#[cfg(CONFIG_HAS_IOMEM)] +use crate::io::mem::IoRequest; + use core::{ marker::PhantomData, mem::offset_of, @@ -83,7 +83,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::__platform_driver_register(pdrv.get(), module.0, name.as_char_ptr()) + bindings::__platform_driver_register(pdrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -100,7 +100,8 @@ impl<T: Driver> Adapter<T> { // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. let pdev = unsafe { &*pdev.cast::<Device<device::CoreInternal<'_>>>() }; - let info = <Self as driver::Adapter>::id_info(pdev.as_ref()); + // SAFETY: `pdev` matched data is of type `Self::IdInfo`. + let info = unsafe { <Self as driver::Adapter>::id_info(pdev.as_ref()) }; from_result(|| { let data = T::probe(pdev, info); @@ -176,7 +177,6 @@ macro_rules! module_platform_driver { /// /// kernel::of_device_table!( /// OF_TABLE, -/// MODULE_OF_TABLE, /// <MyDriver as platform::Driver>::IdInfo, /// [ /// (of::DeviceId::new(c"test,device"), ()) @@ -185,7 +185,6 @@ macro_rules! module_platform_driver { /// /// kernel::acpi_device_table!( /// ACPI_TABLE, -/// MODULE_ACPI_TABLE, /// <MyDriver as platform::Driver>::IdInfo, /// [ /// (acpi::DeviceId::new(c"LNUXBEEF"), ()) @@ -307,6 +306,7 @@ impl<Ctx: device::DeviceContext> Device<Ctx> { } } +#[cfg(CONFIG_HAS_IOMEM)] impl Device<Bound> { /// Returns an `IoRequest` for the resource at `index`, if any. pub fn io_request_by_index(&self, index: u32) -> Option<IoRequest<'_>> { @@ -339,22 +339,30 @@ macro_rules! define_irq_accessor_by_index { $handler_trait:ident ) => { $(#[$meta])* - pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( &'a self, flags: irq::Flags, index: u32, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::$reg_type<T>, Error> + 'a { + ) -> impl PinInit<irq::$reg_type<'a, T>, Error> + 'a { pin_init::pin_init_scope(move || { let request = self.$request_fn(index)?; - Ok(irq::$reg_type::<T>::new( - request, - flags, - name, - handler, - )) + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::<T>::new( + request, + flags, + name, + handler, + ) + }) }) } }; @@ -368,22 +376,30 @@ macro_rules! define_irq_accessor_by_name { $handler_trait:ident ) => { $(#[$meta])* - pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( &'a self, flags: irq::Flags, irq_name: &'a CStr, name: &'static CStr, handler: impl PinInit<T, Error> + 'a, - ) -> impl PinInit<irq::$reg_type<T>, Error> + 'a { + ) -> impl PinInit<irq::$reg_type<'a, T>, Error> + 'a { pin_init::pin_init_scope(move || { let request = self.$request_fn(irq_name)?; - Ok(irq::$reg_type::<T>::new( - request, - flags, - name, - handler, - )) + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::<T>::new( + request, + flags, + name, + handler, + ) + }) }) } }; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 6fd84389a858..0d62beeedca5 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -99,7 +99,7 @@ pub mod format_strings { /// The format string must be one of the ones in [`format_strings`], and /// the module name must be null-terminated. /// -/// [`_printk`]: srctree/include/linux/_printk.h +/// [`_printk`]: srctree/include/linux/printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub unsafe fn call_printk( diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 3f3e529e9f58..82acb531b17b 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -235,11 +235,20 @@ impl_alignable_uint!(u8, u16, u32, u64, usize); /// /// This is a generalization of [`size_of`] that works for dynamically sized types. pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. fn size(p: *const Self) -> usize; } impl<T> KnownSize for T { + const MIN_SIZE: usize = size_of::<T>(); + const MIN_ALIGN: Alignment = Alignment::of::<T>(); + #[inline(always)] fn size(_: *const Self) -> usize { size_of::<T>() @@ -247,6 +256,9 @@ impl<T> KnownSize for T { } impl<T> KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::<T>(); + #[inline(always)] fn size(p: *const Self) -> usize { p.len() * size_of::<T>() diff --git a/rust/kernel/pwm.rs b/rust/kernel/pwm.rs index 6c9d667009ef..8aa47304bec3 100644 --- a/rust/kernel/pwm.rs +++ b/rust/kernel/pwm.rs @@ -494,9 +494,7 @@ impl PwmOpsVTable { /// This is used to bridge Rust trait implementations to the C `struct pwm_ops` /// expected by the kernel. pub const fn create_pwm_ops<T: PwmOps>() -> PwmOpsVTable { - // SAFETY: `core::mem::zeroed()` is unsafe. For `pwm_ops`, all fields are - // `Option<extern "C" fn(...)>` or data, so a zeroed pattern (None/0) is valid initially. - let mut ops: bindings::pwm_ops = unsafe { core::mem::zeroed() }; + let mut ops: bindings::pwm_ops = pin_init::zeroed(); ops.request = Some(Adapter::<T>::request_callback); ops.capture = Some(Adapter::<T>::capture_callback); @@ -600,7 +598,7 @@ impl<T: PwmOps> Chip<T> { let drvdata_ptr = unsafe { bindings::pwmchip_get_drvdata(c_chip_ptr) }; // SAFETY: We construct the `T` object in-place in the allocated private memory. - unsafe { data.__pinned_init(drvdata_ptr.cast()) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(drvdata_ptr.cast(), data) }.inspect_err(|_| { // SAFETY: It is safe to call `pwmchip_put()` with a valid pointer obtained // from `pwmchip_alloc()`. We will not use pointer after this. unsafe { bindings::pwmchip_put(c_chip_ptr) } diff --git a/rust/kernel/revocable.rs b/rust/kernel/revocable.rs index 0f4ae673256d..0e55e2a0fb37 100644 --- a/rust/kernel/revocable.rs +++ b/rust/kernel/revocable.rs @@ -7,12 +7,21 @@ use pin_init::Wrapper; -use crate::{bindings, prelude::*, sync::rcu, types::Opaque}; +use crate::{ + prelude::*, + sync::{ + atomic::{ + AtomicFlag, + Relaxed, // + }, + rcu, // + }, + types::Opaque, // +}; use core::{ marker::PhantomData, ops::Deref, - ptr::drop_in_place, - sync::atomic::{AtomicBool, Ordering}, + ptr::drop_in_place, // }; /// An object that can become inaccessible at runtime. @@ -65,7 +74,7 @@ use core::{ /// ``` #[pin_data(PinnedDrop)] pub struct Revocable<T> { - is_available: AtomicBool, + is_available: AtomicFlag, #[pin] data: Opaque<T>, } @@ -84,7 +93,7 @@ impl<T> Revocable<T> { /// Creates a new revocable instance of the given data. pub fn new<E>(data: impl PinInit<T, E>) -> impl PinInit<Self, E> { try_pin_init!(Self { - is_available: AtomicBool::new(true), + is_available: AtomicFlag::new(true), data <- Opaque::pin_init(data), }? E) } @@ -98,7 +107,7 @@ impl<T> Revocable<T> { /// because another CPU may be waiting to complete the revocation of this object. pub fn try_access(&self) -> Option<RevocableGuard<'_, T>> { let guard = rcu::read_lock(); - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // Since `self.is_available` is true, data is initialised and has to remain valid // because the RCU read side lock prevents it from being dropped. Some(RevocableGuard::new(self.data.get(), guard)) @@ -116,7 +125,7 @@ impl<T> Revocable<T> { /// allowed to sleep because another CPU may be waiting to complete the revocation of this /// object. pub fn try_access_with_guard<'a>(&'a self, _guard: &'a rcu::Guard) -> Option<&'a T> { - if self.is_available.load(Ordering::Relaxed) { + if self.is_available.load(Relaxed) { // SAFETY: Since `self.is_available` is true, data is initialised and has to remain // valid because the RCU read side lock prevents it from being dropped. Some(unsafe { &*self.data.get() }) @@ -157,12 +166,11 @@ impl<T> Revocable<T> { /// /// Callers must ensure that there are no more concurrent users of the revocable object. unsafe fn revoke_internal<const SYNC: bool>(&self) -> bool { - let revoke = self.is_available.swap(false, Ordering::Relaxed); + let revoke = self.is_available.xchg(false, Relaxed); if revoke { if SYNC { - // SAFETY: Just an FFI call, there are no further requirements. - unsafe { bindings::synchronize_rcu() }; + rcu::synchronize_rcu(); } // SAFETY: We know `self.data` is valid because only one CPU can succeed the diff --git a/rust/kernel/serdev.rs b/rust/kernel/serdev.rs new file mode 100644 index 000000000000..17ca504b7f8d --- /dev/null +++ b/rust/kernel/serdev.rs @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the serial device bus. +//! +//! C header: [`include/linux/serdev.h`](srctree/include/linux/serdev.h) + +use crate::{ + acpi, + device, + driver, + error::{ + from_result, + to_result, + VTABLE_DEFAULT_ERROR, // + }, + new_mutex, + of, + prelude::*, + sync::{ + aref::AlwaysRefCounted, + Mutex, // + }, + time::Jiffies, + types::{ + Opaque, + ScopeGuard, // + }, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{offset_of, MaybeUninit}, + ptr::NonNull, // +}; + +/// Parity bit to use with a serial device. +#[repr(u32)] +pub enum Parity { + /// No parity bit. + None = bindings::serdev_parity_SERDEV_PARITY_NONE, + /// Even partiy. + Even = bindings::serdev_parity_SERDEV_PARITY_EVEN, + /// Odd parity. + Odd = bindings::serdev_parity_SERDEV_PARITY_ODD, +} + +/// An adapter for the registration of serial device bus device drivers. +pub struct Adapter<T: Driver>(T); + +// SAFETY: +// - `bindings::serdev_device_driver` is a C type declared as `repr(C)`. +// - `PrivateData<'bound, T>` is the type of the driver's device private data. +// - `struct serdev_device_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl<T: Driver> driver::DriverLayout for Adapter<T> { + type DriverType = bindings::serdev_device_driver; + type DriverData<'bound> = PrivateData<'bound, T>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { + unsafe fn register( + sdrv: &Opaque<Self::DriverType>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct serdev_device_driver` on initialization. + unsafe { + (*sdrv.get()).driver.name = name.as_char_ptr(); + (*sdrv.get()).probe = Some(Self::probe_callback); + (*sdrv.get()).remove = Some(Self::remove_callback); + (*sdrv.get()).driver.of_match_table = of_table; + (*sdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::__serdev_device_driver_register(sdrv.get(), module.as_ptr()) }) + } + + unsafe fn unregister(sdrv: &Opaque<Self::DriverType>) { + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::serdev_device_driver_unregister(sdrv.get()) }; + } +} + +#[doc(hidden)] +#[pin_data(PinnedDrop)] +pub struct PrivateData<'bound, T: Driver> { + sdev: &'bound Device<device::Bound>, + #[pin] + driver: UnsafeCell<MaybeUninit<T::Data<'bound>>>, + open: UnsafeCell<bool>, + /// Whether `receive_buf_callback` is allowed to call `Driver::receive`. + /// + /// If locked, the receive_buf_callback will be blocked on data reception. + /// This is the case while the driver is being probed or while [`PrivateData`] is being dropped. + /// This is necessary, because we need to open the serdev device before the driver has been + /// probed in order to allow it to be configured, which allows `receive_buf_callback` to be + /// called. Thus we need to block data until probe completes and the driver data becomes + /// initialized. + /// + /// If unlocked and true, the receive_buf_callback will forward the data to + /// `Driver::receive`. This is the normal state of operation. + /// + /// If unlocked and false, the receive_buf_callback will throw away the data. + /// This is only the case, if the serdev device is open and + /// - the driver returned an error in probe + /// or + /// - the driver data already has been dropped, because it was unbound. + #[pin] + active: Mutex<bool>, +} + +#[pinned_drop] +impl<T: Driver> PinnedDrop for PrivateData<'_, T> { + fn drop(self: Pin<&mut Self>) { + let mut active = self.active.lock(); + if *active { + // SAFETY: + // - We have exclusive access to `self.driver`. + // - `self.driver` is guaranteed to be initialized. + unsafe { (*self.driver.get()).assume_init_drop() }; + *active = false; + } + drop(active); + + // SAFETY: We have exclusive access to `self.open`. + if unsafe { *self.open.get() } { + // SAFETY: `self.sdev.as_raw()` is guaranteed to be a pointer to a valid + // `struct serdev_device`. + unsafe { bindings::serdev_device_close(self.sdev.as_raw()) }; + } + } +} + +impl<T: Driver> Adapter<T> { + const OPS: &'static bindings::serdev_device_ops = &bindings::serdev_device_ops { + receive_buf: if T::HAS_RECEIVE { + Some(Self::receive_buf_callback) + } else { + None + }, + write_wakeup: Some(bindings::serdev_device_write_wakeup), + }; + + extern "C" fn probe_callback(sdev: *mut bindings::serdev_device) -> kernel::ffi::c_int { + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer to + // a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `probe_callback()`. + let sdev = unsafe { &*sdev.cast::<Device<device::CoreInternal<'_>>>() }; + // SAFETY: `sdev` matched data is of type `Self::IdInfo`. + let info = unsafe { <Self as driver::Adapter>::id_info(sdev.as_ref()) }; + + from_result(|| { + sdev.as_ref().set_drvdata(try_pin_init!(PrivateData::<T> { + sdev: &**sdev, + driver: MaybeUninit::<T::Data<'_>>::zeroed().into(), + open: false.into(), + active <- new_mutex!(false), + }))?; + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::<PrivateData<'_, T>>() }; + let private_data = ScopeGuard::new_with_data(private_data, |_| { + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + drop(unsafe { sdev.as_ref().drvdata_obtain::<PrivateData<'_, T>>() }); + }); + let mut active = private_data.active.lock(); + + // SAFETY: `sdev.as_raw()` is guaranteed to be a valid pointer to `serdev_device`. + unsafe { bindings::serdev_device_set_client_ops(sdev.as_raw(), Self::OPS) }; + + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer + // to a `serdev_device`. + to_result(unsafe { bindings::serdev_device_open(sdev.as_raw()) })?; + + // SAFETY: We have exclusive access to `private_data.open`. + unsafe { *private_data.open.get() = true }; + + let data = T::probe(sdev, info); + + // SAFETY: We have exclusive access to `private_data.driver`. + let driver = unsafe { &mut *private_data.driver.get() }; + // SAFETY: + // - `driver.as_mut_ptr()` is a valid pointer to uninitialized data. + // - `private_data.driver` is pinned. + let result = unsafe { pin_init::raw_try_init(driver.as_mut_ptr(), data) }; + + *active = result.is_ok(); + + drop(active); + + result.map(|()| { + private_data.dismiss(); + 0 + }) + }) + } + + extern "C" fn remove_callback(sdev: *mut bindings::serdev_device) { + // SAFETY: The serial device bus only ever calls the remove callback with a valid pointer + // to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `remove_callback()`. + let sdev = unsafe { &*sdev.cast::<Device<device::CoreInternal<'_>>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<PrivateData<'_, T>>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::<PrivateData<'_, T>>() }; + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `remove_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + T::unbind(sdev, data_pinned); + } + + extern "C" fn receive_buf_callback( + sdev: *mut bindings::serdev_device, + buf: *const u8, + length: usize, + ) -> usize { + // SAFETY: The serial device bus only ever calls the receive buf callback with a valid + // pointer to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `receive_buf_callback()`. + let sdev = unsafe { &*sdev.cast::<Device<device::BoundInternal>>() }; + + // SAFETY: `receive_buf_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin<KBox<PrivateData<'_, T>>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::<PrivateData<'_, T>>() }; + let active = private_data.active.lock(); + + if !*active { + return length; + } + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `receive_buf_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + // SAFETY: `buf` is guaranteed to be non-null and has the size of `length`. + let buf = unsafe { core::slice::from_raw_parts(buf, length) }; + + T::receive(sdev, data_pinned, buf) + } +} + +impl<T: Driver> driver::Adapter for Adapter<T> { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option<of::IdTable<Self::IdInfo>> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option<acpi::IdTable<Self::IdInfo>> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single serial device bus device driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_serdev_device_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_serdev_device_driver { + ($($f:tt)*) => { + $crate::module_driver!(<T>, $crate::serdev::Adapter<T>, { $($f)* }); + }; +} + +/// The serial device bus device driver trait. +/// +/// Drivers must implement this trait in order to get a serial device bus device driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// acpi, +/// bindings, +/// device::{ +/// Bound, +/// Core, // +/// }, +/// of, +/// serdev, // +/// }; +/// +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// <MyDriver as serdev::Driver>::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// <MyDriver as serdev::Driver>::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// #[vtable] +/// impl serdev::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// sdev: &'bound serdev::Device<Core<'_>>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { +/// sdev.set_baudrate(115200); +/// sdev.write_all(b"Hello\n", 0)?; +/// Ok(MyDriver) +/// } +/// } +///``` +#[vtable] +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + Sync + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option<acpi::IdTable<Self::IdInfo>> = None; + + /// Serial device bus device driver probe. + /// + /// Called when a new serial device bus device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + sdev: &'bound Device<device::Core<'_>>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; + + /// Serial device bus device driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device<Core>` or `&Device<Bound>` reference. For instance. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(sdev: &'bound Device<device::Core<'_>>, this: Pin<&Self::Data<'bound>>) { + let _ = (sdev, this); + } + + /// Serial device bus device data receive callback. + /// + /// Called when data got received from device. + /// + /// Returns the number of bytes accepted. + fn receive<'bound>( + sdev: &'bound Device<device::Bound>, + this: Pin<&Self::Data<'bound>>, + data: &[u8], + ) -> usize { + let _ = (sdev, this, data); + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// The serial device bus device representation. +/// +/// This structure represents the Rust abstraction for a C `struct serdev_device`. The +/// implementation abstracts the usage of an already existing C `struct serdev_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct serdev_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device<Ctx: device::DeviceContext = device::Normal>( + Opaque<bindings::serdev_device>, + PhantomData<Ctx>, +); + +impl<Ctx: device::DeviceContext> Device<Ctx> { + #[inline] + fn as_raw(&self) -> *mut bindings::serdev_device { + self.0.get() + } +} + +impl Device<device::Bound> { + /// Set the baudrate in bits per second. + /// + /// Common baudrates are 115200, 9600, 19200, 57600, 4800. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_baudrate(&self, speed: u32) -> Result<(), u32> { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + let ret = unsafe { bindings::serdev_device_set_baudrate(self.as_raw(), speed) }; + if ret == speed { + Ok(()) + } else { + Err(ret) + } + } + + /// Set if flow control should be enabled. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_flow_control(&self, enable: bool) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_set_flow_control(self.as_raw(), enable) }; + } + + /// Set parity to use. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_parity(&self, parity: Parity) -> Result { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + to_result(unsafe { bindings::serdev_device_set_parity(self.as_raw(), parity as u32) }) + } + + /// Write data to the serial device until the controller has accepted all the data or has + /// been interrupted by a timeout or signal. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Use a timeout of 0 to wait indefinitely. + /// + /// Returns the number of bytes written (less than `data.len()` if interrupted). + /// [`kernel::error::code::ETIMEDOUT`] or [`kernel::error::code::ERESTARTSYS`] if interrupted + /// before any bytes were written. [`kernel::error::code::EINVAL`] if `data.len() > i32::MAX`. + #[inline] + pub fn write_all(&self, data: &[u8], timeout: Jiffies) -> Result<usize> { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = unsafe { + bindings::serdev_device_write( + self.as_raw(), + data.as_ptr(), + data.len(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + // CAST: negative return values are guaranteed to be between `-MAX_ERRNO` and `-1`, + // which always fit into a `i32`. + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Write data to the serial device. + /// + /// If you want to write until the controller has accepted all the data, use + /// [`Device::write_all`]. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Returns the number of bytes written (less than `data.len()` if not enough room in the + /// write buffer). + #[inline] + pub fn write(&self, data: &[u8]) -> Result<u32> { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = + unsafe { bindings::serdev_device_write_buf(self.as_raw(), data.as_ptr(), data.len()) }; + + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Send data to the serial device immediately. + /// + /// Note that this doesn't guarantee that the data has been transmitted. + /// Use [`Device::wait_until_sent`] for this purpose. + #[inline] + pub fn write_flush(&self) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_write_flush(self.as_raw()) }; + } + + /// Wait for the data to be sent. + /// + /// After this function, the write buffer of the controller should be empty or the timeout + /// elapsed. + /// + /// Use a timeout of 0 to wait indefinitely. + #[inline] + pub fn wait_until_sent(&self, timeout: Jiffies) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { + bindings::serdev_device_wait_until_sent( + self.as_raw(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + } +} + +// SAFETY: `serdev::Device` is a transparent wrapper of `struct serdev_device`. +// The offset is guaranteed to point to a valid device field inside `serdev::Device`. +unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> { + const OFFSET: usize = offset_of!(bindings::serdev_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + self.as_ref().inc_ref(); + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::serdev_device_put(obj.cast().as_ptr()) } + } +} + +impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> { + fn as_ref(&self) -> &device::Device<Ctx> { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct serdev_device`. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device<Normal>) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device<Normal>` -- the underlying `struct serdev_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device<device::Bound> {} diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..3e55de3b1636 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -21,16 +21,25 @@ pub mod poll; pub mod rcu; mod refcount; mod set_once; +pub mod srcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy}; pub use lock::mutex::{new_mutex, Mutex, MutexGuard}; -pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard}; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, + SpinLock, + SpinLockGuard, + SpinLockIrq, + SpinLockIrqGuard, // +}; pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; +pub use srcu::Srcu; /// Represents a lockdep class. /// diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 5ac4961b7cd2..8ae0fe6f19ec 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -154,7 +154,7 @@ impl<T: ?Sized> ArcInner<T> { /// /// # Safety /// - /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the [`Arc`] must /// not yet have been destroyed. unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { let refcount_layout = Layout::new::<Refcount>(); @@ -253,7 +253,7 @@ impl<T: ?Sized> Arc<T> { /// Convert the [`Arc`] into a raw pointer. /// - /// The raw pointer has ownership of the refcount that this Arc object owned. + /// The raw pointer has ownership of the refcount that this [`Arc`] object owned. pub fn into_raw(self) -> *const T { let ptr = self.ptr.as_ptr(); core::mem::forget(self); @@ -261,7 +261,7 @@ impl<T: ?Sized> Arc<T> { unsafe { core::ptr::addr_of!((*ptr).data) } } - /// Return a raw pointer to the data in this arc. + /// Return a raw pointer to the data in this [`Arc`]. pub fn as_ptr(this: &Self) -> *const T { let ptr = this.ptr.as_ptr(); @@ -305,7 +305,7 @@ impl<T: ?Sized> Arc<T> { /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. /// - /// When this destroys the `Arc`, it does so while properly avoiding races. This means that + /// When this destroys the [`Arc`], it does so while properly avoiding races. This means that /// this method will never call the destructor of the value. /// /// # Examples @@ -345,11 +345,11 @@ impl<T: ?Sized> Arc<T> { // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will // return without further touching the `Arc`. If the refcount reaches zero, then there are - // no other arcs, and we can create a `UniqueArc`. + // no other `Arc`s, and we can create a `UniqueArc`. if refcount.dec_and_test() { refcount.set(1); - // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We + // INVARIANT: We own the only refcount to this `Arc`, so we may create a `UniqueArc`. We // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin // their values. Some(Pin::from(UniqueArc { @@ -717,7 +717,7 @@ impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. - unsafe { init.__init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }) } @@ -727,7 +727,7 @@ impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { pin_init::raw_try_init(slot, init)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } @@ -795,7 +795,7 @@ impl<T> UniqueArc<MaybeUninit<T>> { #[inline] pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { // SAFETY: The supplied pointer is valid for initialization. - match unsafe { init.__init(self.as_mut_ptr()) } { + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { // SAFETY: Initialization completed successfully. Ok(()) => Ok(unsafe { self.assume_init() }), Err(err) => Err(err), @@ -810,7 +810,7 @@ impl<T> UniqueArc<MaybeUninit<T>> { ) -> core::result::Result<Pin<UniqueArc<T>>, E> { // SAFETY: The supplied pointer is valid for initialization and we will later pin the value // to ensure it does not move. - match unsafe { init.__pinned_init(self.as_mut_ptr()) } { + match unsafe { pin_init::raw_try_init(self.as_mut_ptr(), init) } { // SAFETY: Initialization completed successfully. Ok(()) => Ok(unsafe { self.assume_init() }.into()), Err(err) => Err(err), diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs index b721b2e00b98..9983ee085248 100644 --- a/rust/kernel/sync/aref.rs +++ b/rust/kernel/sync/aref.rs @@ -24,6 +24,11 @@ use core::{ ptr::NonNull, // }; +use crate::{ + prelude::*, + types::ForeignOwnable, // +}; + /// Types that are _always_ reference counted. /// /// It allows such types to define their own custom ref increment and decrement functions. @@ -188,6 +193,51 @@ where } impl<T: AlwaysRefCounted + Eq> Eq for ARef<T> {} +// SAFETY: `into_foreign` returns a pointer from `NonNull::as_ptr`, so it's non-null. The +// `ARef` invariant guarantees that `ptr` points to a valid `T`, so it's aligned to `T`. +unsafe impl<T: AlwaysRefCounted> ForeignOwnable for ARef<T> { + const FOREIGN_ALIGN: usize = core::mem::align_of::<T>(); + + type Borrowed<'a> + = &'a T + where + Self: 'a; + type BorrowedMut<'a> + = &'a T + where + Self: 'a; + + #[inline] + fn into_foreign(self) -> *mut c_void { + ARef::into_raw(self).as_ptr().cast() + } + + #[inline] + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: The safety requirements of this function ensure that `ptr` comes from a previous + // call to `Self::into_foreign`. + let ptr = unsafe { NonNull::new_unchecked(ptr.cast()) }; + + // SAFETY: `ptr` came from `into_foreign`, which consumed an `ARef` without decrementing + // the refcount, so we can transfer the ownership to the new `ARef`. + unsafe { ARef::from_raw(ptr) } + } + + #[inline] + unsafe fn borrow<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements of this method ensure that the object remains alive and + // immutable for the duration of 'a. + unsafe { &*ptr.cast() } + } + + #[inline] + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> &'a T { + // SAFETY: The safety requirements for `borrow_mut` are a superset of the safety + // requirements for `borrow`. + unsafe { <Self as ForeignOwnable>::borrow(ptr) } + } +} + impl<T, U> PartialEq<&'_ U> for ARef<T> where T: AlwaysRefCounted + PartialEq<U>, diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the following memory accesses. //! - It provides ordering between all the preceding memory accesses and all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..1180695d533a 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,38 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{ + Full, + Read, + Write, // +}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses across the barrier. @@ -19,43 +51,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } -/// A full memory barrier. +trait MemoryBarrier<Flavour = ()> { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. -#[inline(always)] -pub fn smp_mb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; - } else { - barrier(); - } +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb<T: MemoryBarrier>(_: T) { + T::run() } -/// A write-write memory barrier. +/// Memory barrier between CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory write accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; + T::run() } else { - barrier(); + barrier() } } -/// A read-read memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory read accesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) { + T::run() } diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index ec2dd84316fc..ebb10521d8bd 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) => { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index ef76fa07ca3a..aafc80125f59 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -3,6 +3,11 @@ //! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super::*; +use crate::{ + interrupt::LocalInterruptDisabled, + prelude::*, // +}; /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// @@ -82,7 +87,7 @@ pub use new_spinlock; /// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock<T> = super::Lock<T, SpinLockBackend>; +pub type SpinLock<T> = Lock<T, SpinLockBackend>; /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; @@ -91,13 +96,11 @@ pub struct SpinLockBackend; /// /// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>; +pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>; // SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend { type State = bindings::spinlock_t; type GuardState = (); @@ -144,3 +147,319 @@ unsafe impl super::Backend for SpinLockBackend { unsafe { bindings::spin_assert_is_held(ptr) } } } + +/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock_irq { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLockIrq::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_spinlock_irq; + +/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. +/// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// +/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, +/// [see the interrupt module](kernel::interrupt). +/// +/// # Examples +/// +/// The following example shows how to declare, allocate initialise and access a struct (`Example`) +/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local +/// processor interrupts to be disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// c: SpinLockIrq<Inner>, +/// #[pin] +/// d: SpinLockIrq<Inner>, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }), +/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example` +/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// +/// // Accessing an `Example` from a context where interrupts may not be disabled already. +/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount +/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount +/// +/// assert_eq!(c_guard.a, 0); +/// assert_eq!(c_guard.b, 10); +/// assert_eq!(d_guard.a, 20); +/// assert_eq!(d_guard.b, 30); +/// +/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is +/// // still in scope. +/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled +/// // now +/// # Ok::<(), Error>(()) +/// ``` +/// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq<Inner>, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// +/// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with +pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>; + +/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts. +pub struct SpinLockIrqBackend; + +/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`]. +/// +/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using +/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt +/// disablement refcount upon being dropped. +/// +/// [`lock()`]: SpinLockIrq::lock +pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. +unsafe impl Backend for SpinLockIrqBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + #[inline] + unsafe fn init( + ptr: *mut Self::State, + name: *const crate::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + #[inline] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock_irq_disable(ptr) } + } + + #[inline] + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the spinlock. + unsafe { bindings::spin_unlock_irq_enable(ptr) } + } + + #[inline] + unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + let result = unsafe { bindings::spin_trylock_irq_disable(ptr) }; + + if result != 0 { + Some(()) + } else { + None + } + } + + #[inline] + unsafe fn assert_is_held(ptr: *mut Self::State) { + // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. + unsafe { bindings::spin_assert_is_held(ptr) } + } +} + +impl<T: ?Sized> Lock<T, SpinLockIrqBackend> { + /// Casts the lock as a `Lock<T, SpinLockBackend>`. + #[inline] + fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> { + // SAFETY: + // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data + // layouts. + // - As long as local interrupts are disabled (which is proven to be true by _context), it + // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. + unsafe { core::mem::transmute(self) } + } + + /// Acquires the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::lock()`] function when + /// possible for [`SpinLockIrq`] locks. + #[inline] + pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> { + self.as_lock_in_interrupt(context).lock() + } + + /// Tries to acquire the lock without modifying local interrupt state. + /// + /// This function should be used in place of the more expensive [`Lock::try_lock()`] function + /// when possible for [`SpinLockIrq`] locks. + /// + /// Returns a guard that can be used to access the data protected by the lock if successful. + #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] + pub fn try_lock_with<'a>( + &'a self, + context: &'a LocalInterruptDisabled, + ) -> Option<SpinLockGuard<'a, T>> { + self.as_lock_in_interrupt(context).try_lock() + } +} + +#[kunit_tests(rust_spinlock_irq_condvar)] +mod tests { + use super::*; + use crate::{ + sync::*, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, + }; + + struct TestState { + value: u32, + waiter_ready: bool, + } + + #[pin_data] + struct Test { + #[pin] + state: SpinLockIrq<TestState>, + + #[pin] + state_changed: CondVar, + + #[pin] + waiter_state_changed: CondVar, + + #[pin] + wait_work: Work<Self>, + } + + impl_has_work! { + impl HasWork<Self> for Test { self.wait_work } + } + + impl Test { + pub(crate) fn new() -> Result<Arc<Self>> { + Arc::try_pin_init( + try_pin_init!( + Self { + state <- new_spinlock_irq!(TestState { + value: 1, + waiter_ready: false + }), + state_changed <- new_condvar!(), + waiter_state_changed <- new_condvar!(), + wait_work <- new_work!("IrqCondvarTest::wait_work") + } + ), + GFP_KERNEL, + ) + } + } + + impl WorkItem for Test { + type Pointer = Arc<Self>; + + fn run(this: Arc<Self>) { + // Wait for the test to be ready to wait for us + let mut state = this.state.lock(); + + // Make sure the interrupts actually turned off + // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()` + unsafe { bindings::lockdep_assert_irqs_disabled() }; + + while !state.waiter_ready { + this.waiter_state_changed.wait(&mut state); + } + + // Deliver the exciting value update our test has been waiting for + state.value += 1; + this.state_changed.notify_sync(); + } + } + + #[test] + fn spinlock_irq_condvar() -> Result { + let testdata = Test::new()?; + + let _ = workqueue::system().enqueue(testdata.clone()); + + // Let the updater know when we're ready to wait + let mut state = testdata.state.lock(); + state.waiter_ready = true; + testdata.waiter_state_changed.notify_sync(); + + // Wait for the exciting value update + testdata.state_changed.wait(&mut state); + assert_eq!(state.value, 2); + Ok(()) + } +} diff --git a/rust/kernel/sync/poll.rs b/rust/kernel/sync/poll.rs index 0ec985d560c8..dc40bfaa57e6 100644 --- a/rust/kernel/sync/poll.rs +++ b/rust/kernel/sync/poll.rs @@ -5,12 +5,22 @@ //! Utilities for working with `struct poll_table`. use crate::{ + alloc::AllocError, bindings, fs::File, prelude::*, - sync::{CondVar, LockClassKey}, + sync::{ + rcu::synchronize_rcu, + CondVar, + LockClassKey, // + }, // + types::Opaque, // +}; +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, // }; -use core::{marker::PhantomData, ops::Deref}; /// Creates a [`PollCondVar`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -66,6 +76,7 @@ impl<'a> PollTable<'a> { /// /// [`CondVar`]: crate::sync::CondVar #[pin_data(PinnedDrop)] +#[repr(transparent)] pub struct PollCondVar { #[pin] inner: CondVar, @@ -99,8 +110,70 @@ impl PinnedDrop for PollCondVar { unsafe { bindings::__wake_up_pollfree(self.inner.wait_queue_head.get()) }; // Wait for epoll items to be properly removed. - // - // SAFETY: Just an FFI call. - unsafe { bindings::synchronize_rcu() }; + synchronize_rcu(); + } +} + +/// A [`KBox<PollCondVar>`] that uses `kfree_rcu`. +/// +/// [`KBox<PollCondVar>`]: PollCondVar +pub struct PollCondVarBox { + inner: ManuallyDrop<Pin<KBox<PollCondVarBoxInner>>>, +} + +#[pin_data] +#[repr(C)] +struct PollCondVarBoxInner { + #[pin] + inner: PollCondVar, + rcu: Opaque<bindings::kvfree_rcu_head>, +} + +// SAFETY: PollCondVar is Send +unsafe impl Send for PollCondVarBoxInner {} +// SAFETY: PollCondVar is Sync +unsafe impl Sync for PollCondVarBoxInner {} + +impl PollCondVarBox { + /// Constructs a new boxed [`PollCondVar`]. + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> Result<Self, AllocError> { + let b = KBox::pin_init( + pin_init!(PollCondVarBoxInner { + inner <- PollCondVar::new(name, key), + rcu: Opaque::uninit(), + }), + GFP_KERNEL, + ) + .map_err(|_| AllocError)?; + + Ok(PollCondVarBox { + inner: ManuallyDrop::new(b), + }) + } +} + +impl Deref for PollCondVarBox { + type Target = PollCondVar; + fn deref(&self) -> &PollCondVar { + &self.inner.inner + } +} + +impl Drop for PollCondVarBox { + #[inline] + fn drop(&mut self) { + // SAFETY: ManuallyDrop::take ok because not already taken. + let boxed = unsafe { ManuallyDrop::take(&mut self.inner) }; + + // SAFETY: The code below frees the box without calling the actual destructor of the type, + // but it's okay because it re-implements the destructor using `kfree_rcu()` in place of + // `synchronize_rcu()`. + let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(boxed) }); + + // SAFETY: The pointer points at a valid `wait_queue_head`. + unsafe { bindings::__wake_up_pollfree((*ptr).inner.inner.wait_queue_head.get()) }; + + // SAFETY: This was allocated using `KBox::pin_init`, so it can be freed with `kvfree`. + unsafe { bindings::kvfree_call_rcu((*ptr).rcu.get(), ptr.cast::<ffi::c_void>()) }; } } diff --git a/rust/kernel/sync/rcu.rs b/rust/kernel/sync/rcu.rs index a32bef6e490b..0daa1ac87d81 100644 --- a/rust/kernel/sync/rcu.rs +++ b/rust/kernel/sync/rcu.rs @@ -50,3 +50,39 @@ impl Drop for Guard { pub fn read_lock() -> Guard { Guard::new() } + +/// Wait until all in-flight `call_rcu()` callbacks complete. +/// +/// Note that this primitive does not necessarily wait for an RCU grace period +/// to complete. For example, if there are no RCU callbacks queued anywhere +/// in the system, then [`rcu_barrier()`] is within its rights to return +/// immediately, without waiting for anything, much less an RCU grace period. +/// In fact, [`rcu_barrier()`] will normally not result in any RCU grace periods +/// beyond those that were already destined to be executed. +/// +/// In kernels built with `CONFIG_RCU_LAZY=y`, this function also hurries all +/// pending lazy RCU callbacks. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn rcu_barrier() { + // SAFETY: `rcu_barrier()` is always safe to be called. It just might wait for a grace period. + unsafe { bindings::rcu_barrier() }; +} + +/// Wait for one RCU grace period. +/// +/// Waits for all RCU read-side critical sections (such as those established by +/// a [`Guard`]) at the moment of the function call to finish. +/// +/// Does not prevent new read-side critical sections from starting, which may +/// begin and run while this call is blocking. +/// +/// Note that this is one of the RCU primitives which must not be called in +/// atomic context. +#[inline] +pub fn synchronize_rcu() { + // SAFETY: `synchronize_rcu()` is always safe to be called from process context. + unsafe { bindings::synchronize_rcu() }; +} diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000000..723e5e277fd6 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque<bindings::srcu_struct>, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit<Self, Error> { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +} diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 38273f4eedb5..3336df493dec 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -210,20 +210,20 @@ impl Task { unsafe { *ptr::addr_of!((*self.as_ptr()).pid) } } - /// Returns the UID of the given task. + /// Returns the TGID (Thread Group ID / Process ID) of the given task. + pub fn tgid(&self) -> Pid { + // SAFETY: The tgid of a task never changes after initialization, so reading this field is + // not a data race. + unsafe { *ptr::addr_of!((*self.as_ptr()).tgid) } + } + + /// Returns the objective real UID of the given task. #[inline] pub fn uid(&self) -> Kuid { // SAFETY: It's always safe to call `task_uid` on a valid task. Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) }) } - /// Returns the effective UID of the given task. - #[inline] - pub fn euid(&self) -> Kuid { - // SAFETY: It's always safe to call `task_euid` on a valid task. - Kuid::from_raw(unsafe { bindings::task_euid(self.as_ptr()) }) - } - /// Determines whether the given task has pending signals. #[inline] pub fn signal_pending(&self) -> bool { @@ -371,7 +371,7 @@ impl PartialEq for Task { impl Eq for Task {} impl Kuid { - /// Get the current euid. + /// Get the current subjective effective UID. #[inline] pub fn current_euid() -> Kuid { // SAFETY: Just an FFI call. diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 363e93cbb139..6c0a5e8090d0 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -246,7 +246,7 @@ impl<C: ClockSource> ops::Sub for Instant<C> { #[inline] fn sub(self, other: Instant<C>) -> Delta { Delta { - nanos: self.inner - other.inner, + value: self.inner - other.inner, } } } @@ -258,7 +258,7 @@ impl<T: ClockSource> ops::Add<Delta> for Instant<T> { fn add(self, rhs: Delta) -> Self::Output { // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow // (e.g. go above `KTIME_MAX`) - let res = self.inner + rhs.nanos; + let res = self.inner + rhs.value; // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] @@ -278,7 +278,7 @@ impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { fn sub(self, rhs: Delta) -> Self::Output { // INVARIANT: With arithmetic over/underflow checks enabled, this will panic if we overflow // (e.g. go above `KTIME_MAX`) - let res = self.inner - rhs.nanos; + let res = self.inner - rhs.value; // INVARIANT: With overflow checks enabled, we verify here that the value is >= 0 #[cfg(CONFIG_RUST_OVERFLOW_CHECKS)] @@ -291,14 +291,64 @@ impl<T: ClockSource> ops::Sub<Delta> for Instant<T> { } } +mod private { + pub trait Sealed {} + + impl Sealed for super::Nsec {} + impl Sealed for super::Jiffy {} +} + +/// A trait for time units. +pub trait TimeUnit: private::Sealed { + /// The underlying representation of the time unit. + type Repr: Copy + Clone + PartialEq + PartialOrd + Eq + Ord + core::fmt::Debug; +} + +/// A time unit of nanoseconds. +/// +/// A [`Delta<Nsec>`] stores its value as [`i64`] nanoseconds and can represent +/// any [`i64`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Nsec {} + +impl TimeUnit for Nsec { + type Repr = i64; +} + +/// A time unit of jiffies. +/// +/// A [`Delta<Jiffy>`] stores its value as [`isize`] jiffies and can represent +/// any [`isize`] value, including negative, zero, and positive numbers. +#[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] +pub enum Jiffy {} + +impl TimeUnit for Jiffy { + type Repr = isize; +} + /// A span of time. /// -/// This struct represents a span of time, with its value stored as nanoseconds. -/// The value can represent any valid i64 value, including negative, zero, and -/// positive numbers. +/// The span is stored in the unit given by the type parameter `U` (see +/// [`TimeUnit`]); its value has type `U::Repr`. `U` defaults to [`Nsec`], so a +/// plain [`Delta`] is a span in nanoseconds. The value can be negative, zero, or +/// positive. #[derive(Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Debug)] -pub struct Delta { - nanos: i64, +pub struct Delta<U: TimeUnit = Nsec> { + value: U::Repr, +} + +impl Delta<Jiffy> { + /// Create a new [`Delta`] from a number of jiffies. + #[inline] + pub const fn from_jiffies(jiffies: isize) -> Self { + Self { value: jiffies } + } + + /// Return the number of jiffies in the [`Delta`]. + #[inline] + pub const fn as_jiffies(self) -> isize { + self.value + } } impl ops::Add for Delta { @@ -307,7 +357,7 @@ impl ops::Add for Delta { #[inline] fn add(self, rhs: Self) -> Self { Self { - nanos: self.nanos + rhs.nanos, + value: self.value + rhs.value, } } } @@ -315,7 +365,7 @@ impl ops::Add for Delta { impl ops::AddAssign for Delta { #[inline] fn add_assign(&mut self, rhs: Self) { - self.nanos += rhs.nanos; + self.value += rhs.value; } } @@ -325,7 +375,7 @@ impl ops::Sub for Delta { #[inline] fn sub(self, rhs: Self) -> Self::Output { Self { - nanos: self.nanos - rhs.nanos, + value: self.value - rhs.value, } } } @@ -333,7 +383,7 @@ impl ops::Sub for Delta { impl ops::SubAssign for Delta { #[inline] fn sub_assign(&mut self, rhs: Self) { - self.nanos -= rhs.nanos; + self.value -= rhs.value; } } @@ -343,7 +393,7 @@ impl ops::Mul<i64> for Delta { #[inline] fn mul(self, rhs: i64) -> Self::Output { Self { - nanos: self.nanos * rhs, + value: self.value * rhs, } } } @@ -351,7 +401,7 @@ impl ops::Mul<i64> for Delta { impl ops::MulAssign<i64> for Delta { #[inline] fn mul_assign(&mut self, rhs: i64) { - self.nanos *= rhs; + self.value *= rhs; } } @@ -362,25 +412,25 @@ impl ops::Div for Delta { fn div(self, rhs: Self) -> Self::Output { #[cfg(CONFIG_64BIT)] { - self.nanos / rhs.nanos + self.value / rhs.value } #[cfg(not(CONFIG_64BIT))] { // SAFETY: This function is always safe to call regardless of the input values - unsafe { bindings::div64_s64(self.nanos, rhs.nanos) } + unsafe { bindings::div64_s64(self.value, rhs.value) } } } } impl Delta { /// A span of time equal to zero. - pub const ZERO: Self = Self { nanos: 0 }; + pub const ZERO: Self = Self { value: 0 }; /// Create a new [`Delta`] from a number of nanoseconds. #[inline] pub const fn from_nanos(nanos: i64) -> Self { - Self { nanos } + Self { value: nanos } } /// Create a new [`Delta`] from a number of microseconds. @@ -391,7 +441,7 @@ impl Delta { #[inline] pub const fn from_micros(micros: i64) -> Self { Self { - nanos: micros.saturating_mul(NSEC_PER_USEC), + value: micros.saturating_mul(NSEC_PER_USEC), } } @@ -403,7 +453,7 @@ impl Delta { #[inline] pub const fn from_millis(millis: i64) -> Self { Self { - nanos: millis.saturating_mul(NSEC_PER_MSEC), + value: millis.saturating_mul(NSEC_PER_MSEC), } } @@ -415,7 +465,7 @@ impl Delta { #[inline] pub const fn from_secs(secs: i64) -> Self { Self { - nanos: secs.saturating_mul(NSEC_PER_SEC), + value: secs.saturating_mul(NSEC_PER_SEC), } } @@ -434,22 +484,32 @@ impl Delta { /// Return the number of nanoseconds in the [`Delta`]. #[inline] pub const fn as_nanos(self) -> i64 { - self.nanos + self.value } /// Return the smallest number of microseconds greater than or equal /// to the value in the [`Delta`]. #[inline] pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + #[cfg(CONFIG_64BIT)] { - self.as_nanos().saturating_add(NSEC_PER_USEC - 1) / NSEC_PER_USEC + n / NSEC_PER_USEC + add } #[cfg(not(CONFIG_64BIT))] // SAFETY: It is always safe to call `ktime_to_us()` with any value. unsafe { - bindings::ktime_to_us(self.as_nanos().saturating_add(NSEC_PER_USEC - 1)) + bindings::ktime_to_us(n) + add } } @@ -468,6 +528,32 @@ impl Delta { } } + /// Return the smallest number of milliseconds greater than or equal + /// to the value in the [`Delta`]. + #[inline] + pub fn as_millis_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + + #[cfg(CONFIG_64BIT)] + { + n / NSEC_PER_MSEC + add + } + + #[cfg(not(CONFIG_64BIT))] + // SAFETY: It is always safe to call `ktime_to_ms()` with any value. + unsafe { + bindings::ktime_to_ms(n) + add + } + } + /// Return `self % dividend` where `dividend` is in nanoseconds. /// /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is @@ -477,7 +563,7 @@ impl Delta { #[cfg(CONFIG_64BIT)] { Self { - nanos: self.as_nanos() % i64::from(dividend), + value: self.as_nanos() % i64::from(dividend), } } @@ -489,7 +575,7 @@ impl Delta { unsafe { bindings::div_s64_rem(self.as_nanos(), dividend, &mut rem) }; Self { - nanos: i64::from(rem), + value: i64::from(rem), } } } diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index ac316fd7b538..132dd428c1f6 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -13,7 +13,10 @@ use pin_init::{PinInit, Wrapper, Zeroable}; #[doc(hidden)] pub mod for_lt; -pub use for_lt::ForLt; +pub use for_lt::{ + CovariantForLt, + ForLt, // +}; /// Used to transfer ownership to and from foreign (non-Rust) languages. /// @@ -417,13 +420,13 @@ impl<T> Opaque<T> { impl<T> Wrapper<T> for Opaque<T> { /// Create an opaque pin-initializer from the given pin-initializer. - fn pin_init<E>(slot: impl PinInit<T, E>) -> impl PinInit<Self, E> { - Self::try_ffi_init(|ptr: *mut T| { + fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { + Self::try_ffi_init(|slot: *mut T| { // SAFETY: - // - `ptr` is a valid pointer to uninitialized memory, + // - `slot` is a valid pointer to uninitialized memory, // - `slot` is not accessed on error, // - `slot` is pinned in memory. - unsafe { PinInit::<T, E>::__pinned_init(slot, ptr) } + unsafe { pin_init::raw_try_init(slot, init) } }) } } diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs index d44323c28e8d..b8f422c802dc 100644 --- a/rust/kernel/types/for_lt.rs +++ b/rust/kernel/types/for_lt.rs @@ -1,22 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Provide implementation and test of the `ForLt` trait and macro. +//! Provide implementation and test of the [`trait@ForLt`] and [`trait@CovariantForLt`] traits and +//! macros. //! -//! This module is hidden and user should just use `ForLt!` directly. +//! This module is hidden and users should just use [`ForLt!`](macro@ForLt) / +//! [`CovariantForLt!`](macro@CovariantForLt) directly. use core::marker::PhantomData; /// Representation of types generic over a lifetime. /// -/// The type must be covariant over the generic lifetime, i.e. the lifetime parameter -/// can be soundly shortened. -/// -/// The lifetime involved must be covariant. -/// /// # Macro /// -/// It is not recommended to implement this trait directly. `ForLt!` macro is provided to obtain a -/// type that implements this trait. +/// It is not recommended to implement this trait directly. [`ForLt!`](macro@ForLt) macro is +/// provided to obtain a type that implements this trait. /// /// The full syntax is /// @@ -49,16 +46,65 @@ use core::marker::PhantomData; /// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`. /// # >(); /// ``` +pub trait ForLt { + /// The type parameterized by the lifetime. + type Of<'a>: 'a; +} +pub use macros::ForLt; + +/// [`trait@ForLt`] subtrait for types that are covariant over their lifetime parameter. +/// +/// Provides a safe [`cast_ref`](CovariantForLt::cast_ref) method for types that are proven to be +/// covariant. The `CovariantForLt!` macro syntax is the same as `ForLt!`. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. +/// [`CovariantForLt!`](macro@CovariantForLt) macro is provided to obtain a type that implements +/// this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt<F: CovariantForLt>() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that +/// `<CovariantForLt!(for<'a> TypeThatUse<'a>) as CovariantForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt<F: CovariantForLt>() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(TypeThatUse<'_>) // Equivalent to `CovariantForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(&u32) // Equivalent to `CovariantForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(u32) // Equivalent to `CovariantForLt!(for<'a> u32)`. +/// # >(); +/// ``` /// /// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied. /// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove /// it. /// /// ```ignore,compile_fail -/// # use kernel::types::ForLt; -/// # fn expect_lt<F: ForLt>() {} +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt<F: CovariantForLt>() {} /// # expect_lt::< -/// ForLt!(fn(&u32)) // Contravariant, will fail compilation. +/// CovariantForLt!(fn(&u32)) // Contravariant, will fail compilation. /// # >(); /// ``` /// @@ -67,26 +113,23 @@ use core::marker::PhantomData; /// the generic parameter but is in a separate item. /// /// ``` -/// # use kernel::types::ForLt; -/// fn expect_lt<F: ForLt>() {} +/// # use kernel::types::CovariantForLt; +/// fn expect_lt<F: CovariantForLt>() {} /// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")] /// fn generic_fn<T: 'static>() { /// // Syntactically proven by the macro -/// expect_lt::<ForLt!(&T)>(); +/// expect_lt::<CovariantForLt!(&T)>(); /// // Syntactically proven by the macro -/// expect_lt::<ForLt!(&KBox<T>)>(); +/// expect_lt::<CovariantForLt!(&KBox<T>)>(); /// // Cannot be syntactically proven, need to check covariance of `KBox` -/// // expect_lt::<ForLt!(&KBox<&T>)>(); +/// // expect_lt::<CovariantForLt!(&KBox<&T>)>(); /// } /// ``` /// /// # Safety /// /// `Self::Of<'a>` must be covariant over the lifetime `'a`. -pub unsafe trait ForLt { - /// The type parameterized by the lifetime. - type Of<'a>: 'a; - +pub unsafe trait CovariantForLt: ForLt { /// Cast a reference to a shorter lifetime. #[inline(always)] fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> { @@ -94,29 +137,33 @@ pub unsafe trait ForLt { unsafe { core::mem::transmute(long) } } } -pub use macros::ForLt; +pub use macros::CovariantForLt; /// This is intended to be an "unsafe-to-refer-to" type. /// -/// Must only be used by the `ForLt!` macro. +/// Must only be used by the [`ForLt!`](macro@ForLt) / [`CovariantForLt!`](macro@CovariantForLt) +/// macros. /// /// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro. /// /// `WF` is a type that the macro can use to assert some specific type is well-formed. /// /// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove -/// additional properties. +/// additional properties. [`ForLt!`](macro@ForLt) emits `N = 0`; +/// [`CovariantForLt!`](macro@CovariantForLt) emits `N = 1` after a covariance proof. #[doc(hidden)] pub struct UnsafeForLtImpl<T: ?Sized, WF, const N: usize>(PhantomData<(WF, T)>); -// This is a helper trait for implementation `ForLt` to be able to use HRTB. +// This is a helper trait for implementation of `ForLt` / `CovariantForLt` to be able to use HRTB. #[doc(hidden)] pub trait WithLt<'a> { type Of: 'a; } -// SAFETY: In `ForLt!` macro, a covariance proof is generated when naming `UnsafeForLtImpl` -// and it will fail to evaluate if the type is not covariant. -unsafe impl<T: ?Sized + for<'a> WithLt<'a>, WF> ForLt for UnsafeForLtImpl<T, WF, 0> { +impl<T: ?Sized + for<'a> WithLt<'a>, WF, const N: usize> ForLt for UnsafeForLtImpl<T, WF, N> { type Of<'a> = <T as WithLt<'a>>::Of; } + +// SAFETY: In `CovariantForLt!` macro, a covariance proof is generated in the `N` const generic +// and it will fail to evaluate if the type is not covariant. Only `N = 1` gets this impl. +unsafe impl<T: ?Sized + for<'a> WithLt<'a>, WF> CovariantForLt for UnsafeForLtImpl<T, WF, 1> {} diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 7aff0c82d0af..993760ff332b 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -24,11 +24,8 @@ use crate::{ }; use core::{ marker::PhantomData, - mem::{ - offset_of, - MaybeUninit, // - }, - ptr::NonNull, + mem::offset_of, + ptr::NonNull, // }; /// An adapter for the registration of USB drivers. @@ -63,7 +60,7 @@ unsafe impl<T: Driver> driver::RegistrationOps for Adapter<T> { // SAFETY: `udrv` is guaranteed to be a valid `DriverType`. to_result(unsafe { - bindings::usb_register_driver(udrv.get(), module.0, name.as_char_ptr()) + bindings::usb_register_driver(udrv.get(), module.as_ptr(), name.as_char_ptr()) }) } @@ -89,7 +86,10 @@ impl<T: Driver> Adapter<T> { // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::<DeviceId>() }; - let info = T::ID_TABLE.info(id.index()); + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. It + // can also come from dynamic IDs, which will ensure that `driver_data` exists in + // `T::ID_TABLE` or is 0. + let info = unsafe { id.info_unchecked_opt::<T::IdInfo>() }; let data = T::probe(intf, id, info); let dev: &device::Device<device::CoreInternal<'_>> = intf.as_ref(); @@ -130,8 +130,7 @@ impl DeviceId { match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE as u16, idVendor: vendor, idProduct: product, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -143,8 +142,7 @@ impl DeviceId { idProduct: product, bcdDevice_lo: bcd_lo, bcdDevice_hi: bcd_hi, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -155,8 +153,7 @@ impl DeviceId { bDeviceClass: class, bDeviceSubClass: subclass, bDeviceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -167,8 +164,7 @@ impl DeviceId { bInterfaceClass: class, bInterfaceSubClass: subclass, bInterfaceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -180,8 +176,7 @@ impl DeviceId { idVendor: vendor, idProduct: product, bInterfaceClass: class, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -193,8 +188,7 @@ impl DeviceId { idVendor: vendor, idProduct: product, bInterfaceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -206,8 +200,7 @@ impl DeviceId { idVendor: vendor, idProduct: product, bInterfaceNumber: number, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } @@ -227,8 +220,7 @@ impl DeviceId { bInterfaceClass: class, bInterfaceSubClass: subclass, bInterfaceProtocol: protocol, - // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`. - ..unsafe { MaybeUninit::zeroed().assume_init() } + ..pin_init::zeroed() }) } } @@ -242,10 +234,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_info` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::usb_device_id, driver_info); - - fn index(&self) -> usize { - self.0.driver_info - } } /// [`IdTable`](kernel::device_id::IdTable) type for USB. @@ -254,14 +242,8 @@ pub type IdTable<T> = &'static dyn kernel::device_id::IdTable<DeviceId, T>; /// Create a USB `IdTable` with its alias for modpost. #[macro_export] macro_rules! usb_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::usb::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("usb", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("usb", $crate::usb::DeviceId, $($tt)*); }; } @@ -277,7 +259,6 @@ macro_rules! usb_device_table { /// /// kernel::usb_device_table!( /// USB_TABLE, -/// MODULE_USB_TABLE, /// <MyDriver as usb::Driver>::IdInfo, /// [ /// (usb::DeviceId::from_id(0x1234, 0x5678), ()), @@ -293,7 +274,7 @@ macro_rules! usb_device_table { /// fn probe<'bound>( /// _interface: &'bound usb::Interface<Core<'_>>, /// _id: &usb::DeviceId, -/// _info: &'bound Self::IdInfo, +/// _info: Option<&'bound Self::IdInfo>, /// ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { /// Err(ENODEV) /// } @@ -322,7 +303,7 @@ pub trait Driver { fn probe<'bound>( interface: &'bound Interface<device::Core<'_>>, id: &DeviceId, - id_info: &'bound Self::IdInfo, + id_info: Option<&'bound Self::IdInfo>, ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound; /// USB driver disconnect. @@ -393,6 +374,7 @@ impl<Ctx: device::DeviceContext> AsRef<Device> for Interface<Ctx> { // SAFETY: Instances of `Interface` are always reference-counted. unsafe impl AlwaysRefCounted for Interface { + #[inline] fn inc_ref(&self) { // SAFETY: The invariants of `Interface` guarantee that `self.as_raw()` // returns a valid `struct usb_interface` pointer, for which we will @@ -400,6 +382,7 @@ unsafe impl AlwaysRefCounted for Interface { unsafe { bindings::usb_get_intf(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::usb_put_intf(obj.cast().as_ptr()) } @@ -444,6 +427,7 @@ kernel::impl_device_context_into_aref!(Device); // SAFETY: Instances of `Device` are always reference-counted. unsafe impl AlwaysRefCounted for Device { + #[inline] fn inc_ref(&self) { // SAFETY: The invariants of `Device` guarantee that `self.as_raw()` // returns a valid `struct usb_device` pointer, for which we will @@ -451,6 +435,7 @@ unsafe impl AlwaysRefCounted for Device { unsafe { bindings::usb_get_dev(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull<Self>) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::usb_put_dev(obj.cast().as_ptr()) } diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs index 364d4113cd10..4372cbad3ec4 100644 --- a/rust/macros/for_lt.rs +++ b/rust/macros/for_lt.rs @@ -154,8 +154,8 @@ impl<'a> Prover<'a> { // Note that if we encounter `&'other_lt T`, then we still need to make sure the type // is wellformed if `T` involves `&'lt`, so we defer to the compiler. // - // This is to block cases like `ForLt!(for<'a> &'static &'a u32)`, as the presence of - // the type implies `'a: 'static` but this is unsound. + // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the + // presence of the type implies `'a: 'static` but this is unsound. Type::Reference(ty) if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) => { @@ -176,7 +176,12 @@ impl<'a> Prover<'a> { } } -pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { +/// Shared implementation for both `ForLt!` and `CovariantForLt!`. +/// +/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime +/// instances (workaround for <https://github.com/rust-lang/rust/issues/152489>). `CovariantForLt!` +/// additionally emits covariance proof functions and sets `N = 1`. +fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream { let (ty, lifetime) = match input { HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime), HigherRankedType::Implicit { ty } => { @@ -211,14 +216,16 @@ pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { )); // Insert a proof that the type is covariant. - let cov_proof_name = format_ident!("prove_covariant_{idx}"); - proof.push(quote!( - fn #cov_proof_name<'__short, '__long: '__short>( - long: #wf_proof_name<'__long> - ) -> #wf_proof_name<'__short> { - long - } - )); + if prove_covariance { + let cov_proof_name = format_ident!("prove_covariant_{idx}"); + proof.push(quote!( + fn #cov_proof_name<'__short, '__long: '__short>( + long: #wf_proof_name<'__long> + ) -> #wf_proof_name<'__short> { + long + } + )); + } } // Make sure that the type is wellformed when substituting lifetime with `'static`. @@ -234,6 +241,8 @@ pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { }, ); + let n: usize = prove_covariance.into(); + quote!( ::kernel::types::for_lt::UnsafeForLtImpl::< dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>, @@ -241,8 +250,16 @@ pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { { #(#proof)* - 0 + #n } > ) } + +pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, false) +} + +pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, true) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 4a48fabbc268..24f96feaeb34 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -56,6 +56,7 @@ use syn::parse_macro_input; /// - [`u64`] /// - [`isize`] /// - [`usize`] +/// - [`bool`] /// /// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h) /// @@ -177,12 +178,29 @@ pub fn module(input: TokenStream) -> TokenStream { /// /// This macro should not be used when all functions are required. /// +/// Additionally, this macro automatically handles the `OwnerModule` +/// associated type: on the trait side, `type OwnerModule: ModuleMetadata;` +/// is added as a required associated type if not already defined; on the +/// impl side, `type OwnerModule = LocalModule;` is automatically inserted +/// if not explicitly defined. +/// /// # Examples /// /// ``` /// use kernel::error::VTABLE_DEFAULT_ERROR; /// use kernel::prelude::*; /// +/// # struct LocalModule; +/// # impl kernel::ModuleMetadata for LocalModule { +/// # const NAME: &'static kernel::str::CStr = c"vtable_doctest"; +/// # +/// # // SAFETY: This doctest runs on the host: there is no `THIS_MODULE`. +/// # const THIS_MODULE: kernel::ThisModule = unsafe { +/// # kernel::ThisModule::from_ptr(core::ptr::null_mut()) +/// # }; +/// # } +/// # +/// # fn main() { /// // Declares a `#[vtable]` trait /// #[vtable] /// pub trait Operations: Send + Sync + Sized { @@ -208,6 +226,7 @@ pub fn module(input: TokenStream) -> TokenStream { /// /// assert_eq!(<Foo as Operations>::HAS_FOO, true); /// assert_eq!(<Foo as Operations>::HAS_BAR, false); +/// # } /// ``` /// /// [`kernel::error::VTABLE_DEFAULT_ERROR`]: ../kernel/error/constant.VTABLE_DEFAULT_ERROR.html @@ -497,8 +516,24 @@ pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream { /// /// [`ForLt`]: trait.ForLt.html #[proc_macro] -// The macro shares the name with the trait. #[allow(non_snake_case)] pub fn ForLt(input: TokenStream) -> TokenStream { for_lt::for_lt(parse_macro_input!(input)).into() } + +/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked +/// type. +/// +/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime, +/// providing a safe [`CovariantForLt::cast_ref`] method. +/// +/// Please refer to the documentation of the [`CovariantForLt`] trait. +/// +/// [`CovariantForLt`]: trait.CovariantForLt.html +/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn CovariantForLt(input: TokenStream) -> TokenStream { + for_lt::covariant_for_lt(parse_macro_input!(input)).into() +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 06c18e207508..bc7027f8dbb2 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -192,6 +192,7 @@ fn param_ops_path(param_type: &str) -> Path { "u64" => parse_quote!(::kernel::module_param::PARAM_OPS_U64), "isize" => parse_quote!(::kernel::module_param::PARAM_OPS_ISIZE), "usize" => parse_quote!(::kernel::module_param::PARAM_OPS_USIZE), + "bool" => parse_quote!(::kernel::module_param::PARAM_OPS_BOOL), t => panic!("Unsupported parameter type {}", t), } } @@ -497,28 +498,28 @@ pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> { /// Used by the printing macros, e.g. [`info!`]. const __LOG_PREFIX: &[u8] = #name_cstr.to_bytes_with_nul(); - // SAFETY: `__this_module` is constructed by the kernel at load time and will not be - // freed until the module is unloaded. - #[cfg(MODULE)] - static THIS_MODULE: ::kernel::ThisModule = unsafe { - extern "C" { - static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; - }; - - ::kernel::ThisModule::from_ptr(__this_module.get()) - }; - - #[cfg(not(MODULE))] - static THIS_MODULE: ::kernel::ThisModule = unsafe { - ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) - }; - /// The `LocalModule` type is the type of the module created by `module!`, /// `module_pci_driver!`, `module_platform_driver!`, etc. type LocalModule = #type_; impl ::kernel::ModuleMetadata for #type_ { const NAME: &'static ::kernel::str::CStr = #name_cstr; + + #[cfg(MODULE)] + const THIS_MODULE: ::kernel::ThisModule = { + extern "C" { + static __this_module: ::kernel::types::Opaque<::kernel::bindings::module>; + } + + // SAFETY: `__this_module` is constructed by the kernel at load time + // and lives until the module is unloaded. + unsafe { ::kernel::ThisModule::from_ptr(__this_module.get()) } + }; + + #[cfg(not(MODULE))] + const THIS_MODULE: ::kernel::ThisModule = unsafe { + ::kernel::ThisModule::from_ptr(::core::ptr::null_mut()) + }; } // Double nested modules, since then nobody can access the public items inside. @@ -616,12 +617,12 @@ pub(crate) fn module(info: ModuleInfo) -> Result<TokenStream> { /// This function must only be called once. unsafe fn __init() -> ::kernel::ffi::c_int { let initer = <super::super::LocalModule as ::kernel::InPlaceModule>::init( - &super::super::THIS_MODULE + ::kernel::module::this_module::<super::super::LocalModule>() ); // SAFETY: No data race, since `__MOD` can only be accessed by this module // and there only `__init` and `__exit` access it. These functions are only // called once and `__exit` cannot be called before or during `__init`. - match unsafe { initer.__pinned_init(__MOD.as_mut_ptr()) } { + match unsafe { ::pin_init::raw_try_init(__MOD.as_mut_ptr(), initer) } { Ok(m) => 0, Err(e) => e.to_errno(), } diff --git a/rust/macros/vtable.rs b/rust/macros/vtable.rs index c6510b0c4ea1..be9a5ed8abe5 100644 --- a/rust/macros/vtable.rs +++ b/rust/macros/vtable.rs @@ -30,6 +30,22 @@ fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> { const USE_VTABLE_ATTR: (); }); + // Add `type OwnerModule: ModuleMetadata` as a required associated type if + // the trait does not already define it. + if !item + .items + .iter() + .any(|i| matches!(i, TraitItem::Type(t) if t.ident == "OwnerModule")) + { + gen_items.push(parse_quote! { + /// The module implementing this vtable trait. + /// + /// Automatically set to `crate::LocalModule` by the `#[vtable]` + /// impl macro. + type OwnerModule: ::kernel::ModuleMetadata; + }); + } + for item in &item.items { if let TraitItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; @@ -57,12 +73,18 @@ fn handle_trait(mut item: ItemTrait) -> Result<ItemTrait> { fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { let mut gen_items = Vec::new(); - let mut defined_consts = HashSet::new(); + let mut defined_items = HashSet::new(); - // Iterate over all user-defined constants to gather any possible explicit overrides. + // Iterate over all user-defined items to gather any possible explicit overrides. for item in &item.items { - if let ImplItem::Const(const_item) = item { - defined_consts.insert(const_item.ident.clone()); + match item { + ImplItem::Const(const_item) => { + defined_items.insert(const_item.ident.clone()); + } + ImplItem::Type(type_item) => { + defined_items.insert(type_item.ident.clone()); + } + _ => {} } } @@ -70,6 +92,15 @@ fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { const USE_VTABLE_ATTR: () = (); }); + // Auto-insert `type OwnerModule = crate::LocalModule` if not explicitly defined. + // `crate::LocalModule` resolves to the real module type (via `module!`) or a + // dummy fallback in non-module contexts (e.g., doctests). + if !defined_items.contains(&parse_quote!(OwnerModule)) { + gen_items.push(parse_quote! { + type OwnerModule = crate::LocalModule; + }); + } + for item in &item.items { if let ImplItem::Fn(fn_item) = item { let name = &fn_item.sig.ident; @@ -78,7 +109,7 @@ fn handle_impl(mut item: ItemImpl) -> Result<ItemImpl> { name.span(), ); // Skip if it's declared already -- this allows user override. - if defined_consts.contains(&gen_const_name) { + if defined_items.contains(&gen_const_name) { continue; } let cfg_attrs = crate::helpers::gather_cfg_attrs(&fn_item.attrs); diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs index 35ecb5f68dc3..e8d4dbb664fe 100644 --- a/rust/pin-init/examples/mutex.rs +++ b/rust/pin-init/examples/mutex.rs @@ -79,11 +79,7 @@ impl<T> CMutex<T> { wait_list <- ListHead::new(), spin_lock: SpinLock::new(), locked: Cell::new(false), - data <- unsafe { - pin_init_from_closure(|slot: *mut UnsafeCell<T>| { - val.__pinned_init(slot.cast::<T>()) - }) - }, + data <- UnsafeCell::pin_init(val), }) } @@ -91,7 +87,7 @@ impl<T> CMutex<T> { pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> { let mut sguard = self.spin_lock.acquire(); if self.locked.get() { - stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list)); + stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list)); // println!("wait list length: {}", self.wait_list.size()); while self.locked.get() { drop(sguard); @@ -99,9 +95,6 @@ impl<T> CMutex<T> { thread::park(); sguard = self.spin_lock.acquire(); } - // This does have an effect, as the ListHead inside wait_entry implements Drop! - #[expect(clippy::drop_non_drop)] - drop(wait_entry); } self.locked.set(true); unsafe { diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs index 58cd4241b78c..8dd52313c1b8 100644 --- a/rust/pin-init/examples/static_init.rs +++ b/rust/pin-init/examples/static_init.rs @@ -59,7 +59,7 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { println!("doing init"); let ptr = self.cell.get().cast::<T>(); match self.init.take() { - Some(f) => unsafe { f.__pinned_init(ptr).unwrap() }, + Some(f) => unsafe { pin_init::raw_init(ptr, f) }, None => unsafe { core::hint::unreachable_unchecked() }, } self.present.set(true); @@ -71,13 +71,11 @@ impl<T, I: PinInit<T>> ops::Deref for StaticInit<T, I> { pub struct CountInit; unsafe impl PinInit<CMutex<usize>> for CountInit { - unsafe fn __pinned_init( - self, - slot: *mut CMutex<usize>, - ) -> Result<(), core::convert::Infallible> { + unsafe fn __init(self, slot: *mut CMutex<usize>) -> Result<(), core::convert::Infallible> { let init = CMutex::new(0); std::thread::sleep(std::time::Duration::from_millis(1000)); - unsafe { init.__pinned_init(slot) } + unsafe { pin_init::raw_init(slot, init) }; + Ok(()) } } diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 28d30805d06b..fd0b5ea4a0a3 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -233,10 +233,12 @@ fn init_fields( InitializerKind::Value { ident, .. } => ident, InitializerKind::Init { ident, .. } => ident, InitializerKind::Code { block, .. } => { + let stmt = &block.stmts; res.extend(quote! { #(#attrs)* - #[allow(unused_braces)] - #block + { + #(#stmt)* + } }); continue; } @@ -334,7 +336,7 @@ fn make_field_check( }), }; quote! { - #[allow(unreachable_code, clippy::diverging_sub_expression)] + #[allow(unreachable_code)] // We use unreachable code to perform field checks. They're still checked by the compiler. // SAFETY: this code is never executed. let _ = || unsafe { diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs index 9fbbd25bcaac..ff194d27565e 100644 --- a/rust/pin-init/internal/src/pin_data.rs +++ b/rust/pin-init/internal/src/pin_data.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use proc_macro2::TokenStream; -use quote::{format_ident, quote}; +use quote::{format_ident, quote, ToTokens}; use syn::{ parse::{End, Nothing, Parse}, parse_quote, parse_quote_spanned, spanned::Spanned, visit_mut::VisitMut, - Attribute, Field, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, + Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause, }; use crate::diagnostics::{DiagCtxt, ErrorGuaranteed}; @@ -35,10 +35,18 @@ impl Parse for Args { } } +impl ToTokens for Args { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Nothing(_) => (), + Self::PinnedDrop(kw) => kw.to_tokens(tokens), + } + } +} + struct FieldInfo<'a> { field: &'a Field, pinned: bool, - cfg_attrs: Vec<&'a Attribute>, } pub(crate) fn pin_data( @@ -68,6 +76,55 @@ pub(crate) fn pin_data( } }; + // Handling cfg can gets very complicated, especially for tuple structs. Therefore, resolve all + // field cfgs first before continuing. + // + // We need to perform this after parsing so we can reliably detect field cfgs. + for (field_idx, field) in struct_.fields.iter_mut().enumerate() { + let cfg: Vec<_> = field + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .map(|a| { + a.parse_args::<TokenStream>() + .expect("parse as token stream cannot fail") + }) + .collect(); + + if cfg.is_empty() { + continue; + } + + field.attrs.retain(|a| !a.path().is_ident("cfg")); + let cfg_true_struct = quote!(#struct_); + + let punctuated = match &mut struct_.fields { + Fields::Named(fields) => &mut fields.named, + Fields::Unnamed(fields) => &mut fields.unnamed, + Fields::Unit => unreachable!(), + }; + *punctuated = std::mem::take(punctuated) + .into_pairs() + .enumerate() + .filter(|&(i, _)| i != field_idx) + .map(|(_, p)| p) + .collect(); + let cfg_false_struct = quote!(#struct_); + + // Resolve one field at a time until we've got no more field cfgs. + // + // This is linear time because macro invocations with false cfg will not be expanded. + return Ok(quote!( + #[cfg(all(#(#cfg,)*))] + #[::pin_init::pin_data(#args)] + #cfg_true_struct + + #[cfg(not(all(#(#cfg,)*)))] + #[::pin_init::pin_data(#args)] + #cfg_false_struct + )); + } + // The generics might contain the `Self` type. Since this macro will define a new type with the // same generics and bounds, this poses a problem: `Self` will refer to the new type as opposed // to this struct definition. Therefore we have to replace `Self` with the concrete name. @@ -85,18 +142,19 @@ pub(crate) fn pin_data( .map(|field| { let len = field.attrs.len(); field.attrs.retain(|a| !a.path().is_ident("pin")); - let pinned = len != field.attrs.len(); + let pinned_count = len - field.attrs.len(); + if pinned_count > 1 { + dcx.error(&field, "#[pin] attribute specified more than once"); + } - let cfg_attrs = field - .attrs - .iter() - .filter(|a| a.path().is_ident("cfg")) - .collect(); + assert!( + !field.attrs.iter().any(|a| a.path().is_ident("cfg")), + "cfgs should be all resolved at this point" + ); FieldInfo { field: &*field, - pinned, - cfg_attrs, + pinned: pinned_count != 0, } }) .collect(); @@ -182,9 +240,7 @@ fn generate_unpin_impl( let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| { let ident = f.field.ident.as_ref().unwrap(); let ty = &f.field.ty; - let cfg_attrs = &f.cfg_attrs; quote!( - #(#cfg_attrs)* #ident: #ty ) }); @@ -242,7 +298,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, // if it also implements `Drop` trait MustNotImplDrop {} - #[expect(drop_bounds)] impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {} impl #impl_generics MustNotImplDrop for #ident #ty_generics #whr @@ -250,7 +305,6 @@ fn generate_drop_impl(ident: &Ident, generics: &Generics, args: Args) -> TokenSt // We also take care to prevent users from writing a useless `PinnedDrop` implementation. // They might implement `PinnedDrop` correctly for the struct, but forget to give // `PinnedDrop` as the parameter to `#[pin_data]`. - #[expect(non_camel_case_types)] trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} @@ -279,7 +333,6 @@ fn generate_projections( .iter() .map(|field| { let Field { vis, ident, ty, .. } = &field.field; - let cfg_attrs = &field.cfg_attrs; let ident = ident .as_ref() @@ -287,11 +340,9 @@ fn generate_projections( if field.pinned { ( quote!( - #(#cfg_attrs)* #vis #ident: ::core::pin::Pin<&'__pin mut #ty>, ), quote!( - #(#cfg_attrs)* // SAFETY: this field is structurally pinned. #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) }, ), @@ -299,11 +350,9 @@ fn generate_projections( } else { ( quote!( - #(#cfg_attrs)* #vis #ident: &'__pin mut #ty, ), quote!( - #(#cfg_attrs)* #ident: &mut #this.#ident, ), ) @@ -373,7 +422,6 @@ fn generate_the_pin_data( .iter() .map(|f| { let Field { vis, ident, ty, .. } = f.field; - let cfg_attrs = &f.cfg_attrs; let field_name = ident .as_ref() @@ -390,7 +438,6 @@ fn generate_the_pin_data( /// - `(*slot).#field_name` is properly aligned. /// - `(*slot).#field_name` points to uninitialized and exclusively accessed /// memory. - #(#cfg_attrs)* // Allow `non_snake_case` since the same warning will be emitted on // the struct definition. #[allow(non_snake_case)] @@ -421,6 +468,7 @@ fn generate_the_pin_data( impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics #whr { + #[inline] fn clone(&self) -> Self { *self } } @@ -429,7 +477,6 @@ fn generate_the_pin_data( {} #[allow(dead_code)] // Some functions might never be used and private. - #[expect(clippy::missing_safety_doc)] impl #impl_generics __ThePinData #ty_generics #whr { @@ -453,6 +500,7 @@ fn generate_the_pin_data( { type PinData = __ThePinData #ty_generics; + #[inline] unsafe fn __pin_data() -> Self::PinData { __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 56dc655e323e..8e9fd18b993f 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -105,6 +105,7 @@ pub unsafe trait HasInitData { pub struct AllData<T: ?Sized>(PhantomInvariant<T>); impl<T: ?Sized> Clone for AllData<T> { + #[inline] fn clone(&self) -> Self { *self } @@ -127,6 +128,7 @@ impl<T: ?Sized> AllData<T> { unsafe impl<T: ?Sized> HasInitData for T { type InitData = AllData<T>; + #[inline] unsafe fn __init_data() -> Self::InitData { AllData(PhantomInvariant::new()) } @@ -181,7 +183,7 @@ impl<T> StackInit<T> { unsafe { this.value.assume_init_drop() }; } // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. - unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; + unsafe { init.__init(this.value.as_mut_ptr())? }; // INVARIANT: `this.value` is initialized above. this.is_init = true; // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. @@ -289,7 +291,7 @@ impl<T: ?Sized> Slot<Pinned, T> { // - when `Err` is returned, we also propagate the error without touching `ptr`; // also `self` is consumed so it cannot be touched further. // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`). - unsafe { init.__pinned_init(self.ptr)? }; + unsafe { init.__init(self.ptr)? }; // SAFETY: // - `self.ptr` is valid, properly aligned and pinned per type invariant. @@ -385,20 +387,23 @@ pub struct AlwaysFail<T: ?Sized> { impl<T: ?Sized> AlwaysFail<T> { /// Creates a new initializer that always fails. + #[inline] pub fn new() -> Self { Self { _t: PhantomData } } } impl<T: ?Sized> Default for AlwaysFail<T> { + #[inline] fn default() -> Self { Self::new() } } -// SAFETY: `__pinned_init` always fails, which is always okay. +// SAFETY: `__init` always fails, which is always okay. unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> { - unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> { + #[inline] + unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> { Err(()) } } diff --git a/rust/pin-init/src/alloc.rs b/rust/pin-init/src/alloc.rs index 5017f57442d8..471652e8663a 100644 --- a/rust/pin-init/src/alloc.rs +++ b/rust/pin-init/src/alloc.rs @@ -35,10 +35,11 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] fn pin_init(init: impl PinInit<T>) -> Result<Pin<Self>, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { - pin_init_from_closure(|slot| match init.__pinned_init(slot) { + pin_init_from_closure(|slot| match init.__init(slot) { Ok(()) => Ok(()), Err(i) => match i {}, }) @@ -52,6 +53,7 @@ pub trait InPlaceInit<T>: Sized { E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. + #[inline] fn init(init: impl Init<T>) -> Result<Self, AllocError> { // SAFETY: We delegate to `init` and only change the error type. let init = unsafe { @@ -109,7 +111,7 @@ impl<T> InPlaceInit<T> for Arc<T> { let slot = slot.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized and this is the only `Arc` to that data. Ok(unsafe { Pin::new_unchecked(this.assume_init()) }) } @@ -136,6 +138,7 @@ impl<T> InPlaceInit<T> for Arc<T> { impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { type Initialized = Box<T>; + #[inline] fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, @@ -145,11 +148,12 @@ impl<T> InPlaceWrite<T> for Box<MaybeUninit<T>> { Ok(unsafe { self.assume_init() }) } + #[inline] fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: All fields have been initialized. Ok(unsafe { self.assume_init() }.into()) } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fd40c8f244a1..7600cdbbbf98 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -70,7 +70,6 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::pin::Pin; @@ -94,7 +93,6 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![expect(clippy::disallowed_names)] //! # #![feature(allocator_api)] //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; //! # use core::{alloc::AllocError, pin::Pin}; @@ -456,7 +454,6 @@ pub use ::pin_init_internal::MaybeZeroable; /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; /// # use pin_init::*; @@ -508,7 +505,6 @@ macro_rules! stack_pin_init { /// # Examples /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -535,7 +531,6 @@ macro_rules! stack_pin_init { /// ``` /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # #![feature(allocator_api)] /// # #[path = "../examples/error.rs"] mod error; use error::Error; /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*; @@ -658,7 +653,6 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![expect(clippy::disallowed_names)] /// # use pin_init::*; /// # use core::pin::Pin; /// # #[pin_data] @@ -895,7 +889,7 @@ macro_rules! assert_pinned { /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. /// -/// The [`PinInit::__pinned_init`] function: +/// The [`PinInit::__init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -915,15 +909,33 @@ macro_rules! assert_pinned { #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { + /// Alias of [`PinInit::__init`]. + /// + /// New code should use `__init` instead. + /// + /// # Safety + /// + /// Same as `__init`. + #[inline(always)] + #[cfg(not(kernel))] + #[deprecated = "use `raw_try_init` instead"] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { self.__init(slot) } + } + /// Initializes `slot`. /// + /// It is not recommended to call this directly. Use [`raw_init`] or [`raw_try_init`]. + /// /// # Safety /// /// - `slot` is a valid pointer to uninitialized memory. /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to /// deallocate. /// - `slot` will not move until it is dropped, i.e. it will be pinned. - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + /// If `Self: Init<T, E>`, this requirement is cancelled and it may be moved. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; /// First initializes the value using `self` then calls the function `f` with the initialized /// value. @@ -943,6 +955,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// Ok(()) /// }); /// ``` + #[inline] fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> where F: FnOnce(Pin<&mut T>) -> Result<(), E>, @@ -951,10 +964,38 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { } } +/// Initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_init<T>(slot: *mut T, init: impl PinInit<T>) { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot).unwrap_or_else(|e| match e {}) } +} + +/// Fallibly initializes `slot` with an initializer. +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +/// If `init` implements `Init<T, E>`, this requirement is cancelled and it may be moved. +#[inline(always)] +pub unsafe fn raw_try_init<T, E>(slot: *mut T, init: impl PinInit<T, E>) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__init(slot) } +} + /// An initializer returned by [`PinInit::pin_chain`]. pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); -// SAFETY: The `__pinned_init` function is implemented such that it +// SAFETY: The `__init` function is implemented such that it // - returns `Ok(())` on successful initialization, // - returns `Err(err)` on error and in this case `slot` will be dropped. // - considers `slot` pinned. @@ -963,15 +1004,14 @@ where I: PinInit<T, E>, F: FnOnce(Pin<&mut T>) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__pinned_init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - let val = unsafe { &mut *slot }; - // SAFETY: `slot` is considered pinned. - let val = unsafe { Pin::new_unchecked(val) }; - // SAFETY: `slot` was initialized above. - (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) }) + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } @@ -988,19 +1028,8 @@ where /// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. /// -/// The [`Init::__init`] function: -/// - returns `Ok(())` if it initialized every field of `slot`, -/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: -/// - `slot` can be deallocated without UB occurring, -/// - `slot` does not need to be dropped, -/// - `slot` is not partially initialized. -/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. -/// -/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same -/// code as `__init`. -/// -/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to -/// move the pointee after initialization. +/// The [`PinInit::__init`] function must work without the pinning requirement; the caller is +/// allowed to move the pointee after initialization. /// #[cfg_attr( kernel, @@ -1014,15 +1043,6 @@ where #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")] #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { - /// Initializes `slot`. - /// - /// # Safety - /// - /// - `slot` is a valid pointer to uninitialized memory. - /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to - /// deallocate. - unsafe fn __init(self, slot: *mut T) -> Result<(), E>; - /// First initializes the value using `self` then calls the function `f` with the initialized /// value. /// @@ -1031,7 +1051,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// # Examples /// /// ```rust - /// # #![expect(clippy::disallowed_names)] /// use pin_init::{init, init_zeroed, Init}; /// /// struct Foo { @@ -1051,6 +1070,7 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// Ok(()) /// }); /// ``` + #[inline] fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> where F: FnOnce(&mut T) -> Result<(), E>, @@ -1062,62 +1082,55 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// An initializer returned by [`Init::chain`]. pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>); -// SAFETY: The `__init` function is implemented such that it -// - returns `Ok(())` on successful initialization, -// - returns `Err(err)` on error and in this case `slot` will be dropped. +// SAFETY: The `__init` function does not rely on the pinning requirement. unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, F: FnOnce(&mut T) -> Result<(), E>, { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: All requirements fulfilled since this function is `__init`. - unsafe { self.0.__pinned_init(slot)? }; - // SAFETY: The above call initialized `slot` and we still have unique access. - (self.1)(unsafe { &mut *slot }).inspect_err(|_| - // SAFETY: `slot` was initialized above. - unsafe { core::ptr::drop_in_place(slot) }) - } } -// SAFETY: `__pinned_init` behaves exactly the same as `__init`. +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, F: FnOnce(&mut T) -> Result<(), E>, { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. - unsafe { self.__init(slot) } + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) }; + let mut guard = slot.init(self.0)?; + (self.1)(guard.let_binding())?; + core::mem::forget(guard); + Ok(()) } } /// Implement `PinInit` and `Init` for closures. /// /// It is unsafe to create this type, since the closure needs to fulfill the same safety -/// requirement as the `__pinned_init`/`__init` functions. +/// requirement as the `__init` functions. struct InitClosure<F, T: ?Sized>(F, __internal::PhantomInvariant<T>); -// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__init` invariants. -unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> -where - F: FnOnce(*mut T) -> Result<(), E>, +// SAFETY: When constructing via `init_from_closure`, the `__init` function does not rely on the +// pinning requirement. When constructing via `pin_init_from_closure`, the opaque type prevents this +// implementation from being visible. +unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T> where + F: FnOnce(*mut T) -> Result<(), E> { - #[inline] - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - (self.0)(slot) - } } // SAFETY: While constructing the `InitClosure`, the user promised that it upholds the -// `__pinned_init` invariants. +// `__init` invariants. unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T> where F: FnOnce(*mut T) -> Result<(), E>, { #[inline] - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { (self.0)(slot) } } @@ -1166,10 +1179,11 @@ pub const unsafe fn init_from_closure<T: ?Sized, E>( /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. - unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) } + unsafe { pin_init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) } } /// Changes the to be initialized type. @@ -1178,6 +1192,7 @@ pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl Pin /// /// - `*mut U` must be castable to `*mut T` and any value of type `T` written through such a /// pointer must result in a valid `U`. +#[inline] pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> { // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety // requirements. @@ -1193,6 +1208,77 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Array initializer from element initializer. +struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>); + +// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the +// elements that have been initialized so far are dropped, thus leaving the array uninitialized and +// ready to deallocate. +unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: PinInit<T, E>, +{ + unsafe fn __init(mut self, slot: *mut [T; N]) -> Result<(), E> { + /// # Invariants + /// + /// - `ptr[..num_init]` contains initialized elements of type `T` + /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory + struct ArrayInitGuard<T> { + /// A pointer to the first element of the array. + ptr: *mut T, + /// The number of initialized elements in the array. + num_init: usize, + } + + impl<T> Drop for ArrayInitGuard<T> { + #[inline] + fn drop(&mut self) { + // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized. + unsafe { + core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( + self.ptr, + self.num_init, + )) + }; + } + } + + // INVARIANT: nothing is initialized yet. + let mut guard = ArrayInitGuard { + ptr: slot.cast::<T>(), + num_init: 0, + }; + + for i in 0..N { + // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized + // thus far. This holds true for every `self.num_init = i`. + guard.num_init = i; + + let init = (self.0)(i); + // SAFETY: + // - The subslot is derived from `slot` with a valid offset. + // - If `Err` is touched, the subslot is not touched further, the guard will drop + // previously initialized elements only. + // - `slot` is pinned so is the subslot. + unsafe { init.__init(&raw mut (*slot)[i]) }?; + } + + // Dismiss the drop guard now that all elements are initialized. + core::mem::forget(guard); + Ok(()) + } +} + +// SAFETY: `I: Init` cancels out the pinning requirement on subslots, which is the only place in the +// `__init` function that relies on `slot` being pinned. +unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F> +where + F: FnMut(usize) -> I, + I: Init<T, E>, +{ +} + /// Initializes an array by initializing each element via the provided initializer. /// /// # Examples @@ -1203,32 +1289,14 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { /// let array: Box<[usize; 1_000]> = Box::init(init_array_from_fn(|i| i)).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl Init<[T; N], E> where I: Init<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Initializes an array by initializing each element via the provided initializer. @@ -1246,32 +1314,14 @@ where /// Arc::pin_init(pin_init_array_from_fn(|i| CMutex::new(i))).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` +#[inline] pub fn pin_init_array_from_fn<I, const N: usize, T, E>( - mut make_init: impl FnMut(usize) -> I, + make_init: impl FnMut(usize) -> I, ) -> impl PinInit<[T; N], E> where I: PinInit<T, E>, { - let init = move |slot: *mut [T; N]| { - let slot = slot.cast::<T>(); - for i in 0..N { - let init = make_init(i); - // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. - let ptr = unsafe { slot.add(i) }; - // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` - // requirements. - if let Err(e) = unsafe { init.__pinned_init(ptr) } { - // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return - // `Err` below, `slot` will be considered uninitialized memory. - unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; - return Err(e); - } - } - Ok(()) - }; - // SAFETY: The initializer above initializes every element of the array. On failure it drops - // any initialized elements and returns `Err`. - unsafe { pin_init_from_closure(init) } + ArrayInit(make_init, __internal::PhantomInvariant::new()) } /// Construct an initializer in a closure and run it. @@ -1300,6 +1350,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`pin_init!`] invocation. +#[inline] pub fn pin_init_scope<T, E, F, I>(make_init: F) -> impl PinInit<T, E> where F: FnOnce() -> Result<I, E>, @@ -1307,13 +1358,13 @@ where { // SAFETY: // - If `make_init` returns `Err`, `Err` is returned and `slot` is completely uninitialized, - // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__pinned_init`. - // - The safety requirements of `init.__pinned_init` are fulfilled, since it's being called - // from an initializer. + // - If `make_init` returns `Ok`, safety requirement are fulfilled by `init.__init`. + // - The safety requirements of `init.__init` are fulfilled, since it's being called from an + // initializer. unsafe { pin_init_from_closure(move |slot: *mut T| -> Result<(), E> { let init = make_init()?; - init.__pinned_init(slot) + init.__init(slot) }) } } @@ -1343,6 +1394,7 @@ where /// This initializer will first execute `lookup_bar()`, match on it, if it returned an error, the /// initializer itself will fail with that error. If it returned `Ok`, then it will run the /// initializer returned by the [`init!`] invocation. +#[inline] pub fn init_scope<T, E, F, I>(make_init: F) -> impl Init<T, E> where F: FnOnce() -> Result<I, E>, @@ -1361,41 +1413,29 @@ where } } -// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of `slot`. -unsafe impl<T> Init<T> for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T> Init<T> for T {} -// SAFETY: the `__pinned_init` function always returns `Ok(())` and initializes every field of +// SAFETY: the `__init` function always returns `Ok(())` and initializes every field of // `slot`. Additionally, all pinning invariants of `T` are upheld. unsafe impl<T> PinInit<T> for T { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), Infallible> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self) }; Ok(()) } } -// SAFETY: when the `__init` function returns with -// - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. -// - `Err(err)`, slot was not written to. -unsafe impl<T, E> Init<T, E> for Result<T, E> { - unsafe fn __init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `slot` is valid for writes by the safety requirements of this function. - unsafe { slot.write(self?) }; - Ok(()) - } -} +// SAFETY: The `__init` function does not rely on slot being pinned after it returns. +unsafe impl<T, E> Init<T, E> for Result<T, E> {} -// SAFETY: when the `__pinned_init` function returns with +// SAFETY: when the `__init` function returns with // - `Ok(())`, `slot` was initialized and all pinned invariants of `T` are upheld. // - `Err(err)`, slot was not written to. unsafe impl<T, E> PinInit<T, E> for Result<T, E> { - unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { // SAFETY: `slot` is valid for writes by the safety requirements of this function. unsafe { slot.write(self?) }; Ok(()) @@ -1421,6 +1461,7 @@ pub trait InPlaceWrite<T> { impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { type Initialized = &'static mut T; + #[inline] fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E> { let slot = self.as_mut_ptr(); @@ -1431,6 +1472,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { unsafe { Ok(self.assume_init_mut()) } } + #[inline] fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> { let slot = self.as_mut_ptr(); @@ -1438,7 +1480,7 @@ impl<T> InPlaceWrite<T> for &'static mut MaybeUninit<T> { // // The `'static` borrow guarantees the data will not be // moved/invalidated until it gets dropped (which is never). - unsafe { init.__pinned_init(slot)? }; + unsafe { init.__init(slot)? }; // SAFETY: The above call initialized the memory. Ok(Pin::static_mut(unsafe { self.assume_init_mut() })) @@ -1510,10 +1552,13 @@ pub unsafe trait Zeroable { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// + /// As const traits are not yet stable, [`pin_init::zeroed()`] can be used instead + /// when initialization is required in a `const` context. + /// /// # Examples /// /// ``` - /// use pin_init::{Zeroable, zeroed}; + /// use pin_init::Zeroable; /// /// #[derive(Zeroable)] /// struct Point { @@ -1521,10 +1566,11 @@ pub unsafe trait Zeroable { /// y: u32, /// } /// - /// let point: Point = zeroed(); + /// let point: Point = Zeroable::zeroed(); /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` + #[inline] fn zeroed() -> Self where Self: Sized, @@ -1553,6 +1599,9 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// Whenever a type implements [`Zeroable`], this function should be preferred over /// [`core::mem::zeroed()`] or using `MaybeUninit<T>::zeroed().assume_init()`. /// +/// While const traits remain unstable, this function serves as the `const` version of +/// [`Zeroable::zeroed()`]. +/// /// # Examples /// /// ``` @@ -1568,6 +1617,7 @@ pub fn init_zeroed<T: Zeroable>() -> impl Init<T> { /// assert_eq!(point.x, 0); /// assert_eq!(point.y, 0); /// ``` +#[inline] pub const fn zeroed<T: Zeroable>() -> T { // SAFETY:By the type invariants of `Zeroable`, all zeroes is a valid bit pattern for `T`. unsafe { core::mem::zeroed() } @@ -1728,6 +1778,7 @@ pub trait Wrapper<T> { } impl<T> Wrapper<T> for UnsafeCell<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafeCell<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1735,6 +1786,7 @@ impl<T> Wrapper<T> for UnsafeCell<T> { } impl<T> Wrapper<T> for MaybeUninit<T> { + #[inline] fn pin_init<E>(value_init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `MaybeUninit<T>` has a compatible layout to `T`. unsafe { cast_pin_init(value_init) } @@ -1743,6 +1795,7 @@ impl<T> Wrapper<T> for MaybeUninit<T> { #[cfg(all(feature = "unsafe-pinned", CONFIG_RUSTC_HAS_UNSAFE_PINNED))] impl<T> Wrapper<T> for core::pin::UnsafePinned<T> { + #[inline] fn pin_init<E>(init: impl PinInit<T, E>) -> impl PinInit<Self, E> { // SAFETY: `UnsafePinned<T>` has a compatible layout to `T`. unsafe { cast_pin_init(init) } diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index b8a515de31ca..797ead5b5626 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -24,6 +24,10 @@ unsafe_op_in_unsafe_fn )] #![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#![cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] #![feature(cfi_encoding)] // Manual definition of blocklisted types. diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8da..489748ef642c 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -6,11 +6,12 @@ * Sorted alphabetically. */ -#include <uapi/asm-generic/ioctl.h> #include <uapi/drm/drm.h> #include <uapi/drm/nova_drm.h> #include <uapi/drm/panthor_drm.h> #include <uapi/linux/android/binder.h> +#include <uapi/linux/android/binder_netlink.h> +#include <uapi/linux/ioctl.h> #include <uapi/linux/mdio.h> #include <uapi/linux/mii.h> #include <uapi/linux/ethtool.h> diff --git a/rust/zerocopy-derive/README.md b/rust/zerocopy-derive/README.md index 110f4a401778..d62c79804342 100644 --- a/rust/zerocopy-derive/README.md +++ b/rust/zerocopy-derive/README.md @@ -1,14 +1,14 @@ # `zerocopy-derive` -These source files come from the Rust `zerocopy-derive` crate, version v0.8.50 -(released 2026-05-31), hosted in the <https://github.com/google/zerocopy> +These source files come from the Rust `zerocopy-derive` crate, version v0.8.54 +(released 2026-07-08), hosted in the <https://github.com/google/zerocopy> repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only -modified to add the SPDX license identifiers and to remove the generation of +modified to tweak the SPDX license identifiers and to remove the generation of non-ASCII identifiers. For copyright details, please see: - https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1 - https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD - https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE - https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT + https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT diff --git a/rust/zerocopy-derive/derive/from_bytes.rs b/rust/zerocopy-derive/derive/from_bytes.rs index d693a63b7645..66d820f6ad4c 100644 --- a/rust/zerocopy-derive/derive/from_bytes.rs +++ b/rust/zerocopy-derive/derive/from_bytes.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// use proc_macro2::{Span, TokenStream}; use syn::{ parse_quote, Data, DataEnum, DataStruct, DataUnion, Error, Expr, ExprLit, ExprUnary, Lit, UnOp, diff --git a/rust/zerocopy-derive/derive/into_bytes.rs b/rust/zerocopy-derive/derive/into_bytes.rs index ad52a6b45d28..0103a78d087f 100644 --- a/rust/zerocopy-derive/derive/into_bytes.rs +++ b/rust/zerocopy-derive/derive/into_bytes.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// use proc_macro2::{Span, TokenStream}; use quote::quote; use syn::{Data, DataEnum, DataStruct, DataUnion, Error, Type}; diff --git a/rust/zerocopy-derive/derive/known_layout.rs b/rust/zerocopy-derive/derive/known_layout.rs index fddffd167c82..d0c4cecfff15 100644 --- a/rust/zerocopy-derive/derive/known_layout.rs +++ b/rust/zerocopy-derive/derive/known_layout.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// use proc_macro2::TokenStream; use quote::quote; use syn::{parse_quote, Data, Error, Type}; @@ -87,7 +87,6 @@ fn derive_known_layout_for_repr_c_struct<'a>( }; let inner_extras = { - let leading_fields_tys = leading_fields_tys.clone(); let methods = make_methods(*trailing_field_ty); let (_, ty_generics, _) = ctx.ast.generics.split_for_impl(); diff --git a/rust/zerocopy-derive/derive/mod.rs b/rust/zerocopy-derive/derive/mod.rs index 665ba7da55a8..b3839fcf73c9 100644 --- a/rust/zerocopy-derive/derive/mod.rs +++ b/rust/zerocopy-derive/derive/mod.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// pub mod from_bytes; pub mod into_bytes; pub mod known_layout; @@ -15,8 +15,8 @@ use crate::{ util::{Ctx, DataExt, FieldBounds, ImplBlockBuilder, Trait}, }; -pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> TokenStream { - match &ctx.ast.data { +pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { + Ok(match &ctx.ast.data { Data::Struct(strct) => { ImplBlockBuilder::new(ctx, strct, Trait::Immutable, FieldBounds::ALL_SELF).build() } @@ -26,7 +26,7 @@ pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> TokenStream { Data::Union(unn) => { ImplBlockBuilder::new(ctx, unn, Trait::Immutable, FieldBounds::ALL_SELF).build() } - } + }) } pub(crate) fn derive_hash(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> { @@ -97,16 +97,20 @@ pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result<TokenStrea match &ctx.ast.data { Data::Struct(_) => {} Data::Enum(_) | Data::Union(_) => { - return Err(Error::new(Span::call_site(), "can only be applied to structs")); + return ctx + .error_or_skip(Error::new(Span::call_site(), "can only be applied to structs")); } }; if repr.get_packed().is_some() { - return Err(Error::new(Span::call_site(), "must not have #[repr(packed)] attribute")); + return ctx.error_or_skip(Error::new( + Span::call_site(), + "must not have #[repr(packed)] attribute", + )); } if !(repr.is_c() || repr.is_transparent()) { - return Err(Error::new( + return ctx.error_or_skip(Error::new( Span::call_site(), "must have #[repr(C)] or #[repr(transparent)] in order to guarantee this type's layout is splitable", )); @@ -116,7 +120,7 @@ pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result<TokenStrea let trailing_field = if let Some(((_, _, trailing_field), _)) = fields.split_last() { trailing_field } else { - return Err(Error::new(Span::call_site(), "must at least one field")); + return ctx.error_or_skip(Error::new(Span::call_site(), "must at least one field")); }; let zerocopy_crate = &ctx.zerocopy_crate; diff --git a/rust/zerocopy-derive/derive/try_from_bytes.rs b/rust/zerocopy-derive/derive/try_from_bytes.rs index a3e4a75631a5..44f083328786 100644 --- a/rust/zerocopy-derive/derive/try_from_bytes.rs +++ b/rust/zerocopy-derive/derive/try_from_bytes.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// use proc_macro2::TokenStream; use quote::quote; use syn::{ @@ -531,8 +531,8 @@ fn derive_has_field_struct_union(ctx: &Ctx, data: &dyn DataExt) -> TokenStream { data, Trait::ProjectField { variant_id: variant_id.clone(), - field: field.clone(), - field_id: field_id.clone(), + field, + field_id, invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)), }, FieldBounds::None, diff --git a/rust/zerocopy-derive/derive/unaligned.rs b/rust/zerocopy-derive/derive/unaligned.rs index d6dea0a11f1e..7c97d62e2dcb 100644 --- a/rust/zerocopy-derive/derive/unaligned.rs +++ b/rust/zerocopy-derive/derive/unaligned.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// use proc_macro2::{Span, TokenStream}; use syn::{Data, DataEnum, DataStruct, DataUnion, Error}; diff --git a/rust/zerocopy-derive/lib.rs b/rust/zerocopy-derive/lib.rs index c517ea7db1eb..d387de368367 100644 --- a/rust/zerocopy-derive/lib.rs +++ b/rust/zerocopy-derive/lib.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2019 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 @@ -129,6 +129,40 @@ derive!(ByteHash => derive_hash => crate::derive::derive_hash); derive!(ByteEq => derive_eq => crate::derive::derive_eq); derive!(SplitAt => derive_split_at => crate::derive::derive_split_at); +#[cfg_attr(not(zerocopy_unstable_linux), doc(hidden))] +#[proc_macro_derive(most_traits, attributes(zerocopy))] +pub fn most_traits(ts: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = syn::parse_macro_input!(ts as DeriveInput); + let ctx = match Ctx::try_from_derive_input(ast) { + Ok(ctx) => ctx, + Err(e) => return e.into_compile_error().into(), + } + .skip_on_error(); + + // top-level traits for which to attempt a derive + let derives: [(fn(&Ctx, Trait) -> _, _); 6] = [ + (crate::derive::known_layout::derive, Trait::KnownLayout), + (crate::derive::derive_immutable, Trait::Immutable), + (crate::derive::from_bytes::derive_from_bytes, Trait::FromBytes), + (crate::derive::into_bytes::derive_into_bytes, Trait::IntoBytes), + (crate::derive::derive_split_at, Trait::SplitAt), + (crate::derive::unaligned::derive_unaligned, Trait::Unaligned), + ]; + + let mut tokens = proc_macro2::TokenStream::new(); + for (derive, t) in derives { + tokens.extend(derive(&ctx, t)) + } + + // We wrap in `const_block` as a backstop in case any derive fails + // to wrap its output in `const_block` (and thus fails to annotate) + // with the full set of `#[allow(...)]` attributes). + let ts = const_block([Some(tokens)]); + #[cfg(test)] + crate::util::testutil::check_hygiene(ts.clone()); + ts.into() +} + /// Deprecated: prefer [`FromZeros`] instead. #[deprecated(since = "0.8.0", note = "`FromZeroes` was renamed to `FromZeros`")] #[doc(hidden)] diff --git a/rust/zerocopy-derive/repr.rs b/rust/zerocopy-derive/repr.rs index 74fd376d9fda..1525e94302d1 100644 --- a/rust/zerocopy-derive/repr.rs +++ b/rust/zerocopy-derive/repr.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2019 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy-derive/util.rs b/rust/zerocopy-derive/util.rs index 5ba5228e2a44..5c5e9d3bdcb8 100644 --- a/rust/zerocopy-derive/util.rs +++ b/rust/zerocopy-derive/util.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2019 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 @@ -98,6 +98,11 @@ impl Ctx { } } + pub(crate) fn skip_on_error(mut self) -> Self { + self.skip_on_error = true; + self + } + pub(crate) fn core_path(&self) -> TokenStream { let zerocopy_crate = &self.zerocopy_crate; quote!(#zerocopy_crate::util::macro_util::core_reexport) @@ -106,20 +111,21 @@ impl Ctx { pub(crate) fn cfg_compile_error(&self) -> TokenStream { // By checking both during the compilation of the proc macro *and* in // the generated code, we ensure that `--cfg - // zerocopy_unstable_derive_on_error` need only be passed *either* when + // zerocopy_unstable_linux` need only be passed *either* when // compiling this crate *or* when compiling the user's crate. The former // is preferable, but in some situations (such as when cross-compiling // using `cargo build --target`), it doesn't get propagated to this // crate's build by default. - if cfg!(zerocopy_unstable_derive_on_error) { + if cfg!(zerocopy_unstable_linux) { quote!() } else if let Some(span) = self.on_error_span { let core = self.core_path(); - let error_message = "`on_error` is experimental; pass '--cfg zerocopy_unstable_derive_on_error' to enable"; + let error_message = + "`on_error` is experimental; pass '--cfg zerocopy_unstable_linux' to enable"; quote::quote_spanned! {span=> #[allow(unused_attributes, unexpected_cfgs)] const _: () = { - #[cfg(not(zerocopy_unstable_derive_on_error))] + #[cfg(not(zerocopy_unstable_linux))] #core::compile_error!(#error_message); }; } @@ -612,6 +618,20 @@ impl<'a> ImplBlockBuilder<'a> { } }; + let zerocopy_bounds = + field_type_bounds + .into_iter() + .chain(padding_check_bound) + .chain(self_bounds) + .map(|bound| { + if self.ctx.skip_on_error { + parse_quote!(for<'zc> #bound) + } else { + bound.clone() + } + }) + .collect::<Vec<_>>(); + let bounds = self .ctx .ast @@ -621,9 +641,7 @@ impl<'a> ImplBlockBuilder<'a> { .map(|where_clause| where_clause.predicates.iter()) .into_iter() .flatten() - .chain(field_type_bounds.iter()) - .chain(padding_check_bound.iter()) - .chain(self_bounds.iter()); + .chain(zerocopy_bounds.iter()); // The parameters with trait bounds, but without type defaults. let mut params: Vec<_> = self diff --git a/rust/zerocopy/README.md b/rust/zerocopy/README.md index 99e6cad0e26c..3d11a6502cf0 100644 --- a/rust/zerocopy/README.md +++ b/rust/zerocopy/README.md @@ -1,14 +1,13 @@ # `zerocopy` -These source files come from the Rust `zerocopy` crate, version v0.8.50 -(released 2026-05-31), hosted in the <https://github.com/google/zerocopy> +These source files come from the Rust `zerocopy` crate, version v0.8.54 +(released 2026-07-08), hosted in the <https://github.com/google/zerocopy> repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only -modified to add the SPDX license identifiers and to remove `Display` -for `f32` and `f64`. +modified to tweak the SPDX license identifiers. For copyright details, please see: - https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1 - https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD - https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE - https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT + https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1 + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE + https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT diff --git a/rust/zerocopy/rustdoc/style.css b/rust/zerocopy/rustdoc/style.css index 414348964af2..d2d55ad2e689 100644 --- a/rust/zerocopy/rustdoc/style.css +++ b/rust/zerocopy/rustdoc/style.css @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT */ - /* Copyright 2026 The Fuchsia Authors diff --git a/rust/zerocopy/src/byte_slice.rs b/rust/zerocopy/src/byte_slice.rs index a5ded4a18b39..b7f85098dbc4 100644 --- a/rust/zerocopy/src/byte_slice.rs +++ b/rust/zerocopy/src/byte_slice.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/byteorder.rs b/rust/zerocopy/src/byteorder.rs index 8f70048f1eb0..c761d5728320 100644 --- a/rust/zerocopy/src/byteorder.rs +++ b/rust/zerocopy/src/byteorder.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2019 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 @@ -100,6 +100,7 @@ mod private { #[allow(missing_copy_implementations, missing_debug_implementations)] #[doc(hidden)] +#[derive(PartialEq)] pub enum Order { BigEndian, LittleEndian, @@ -164,6 +165,42 @@ pub type BE = BigEndian; /// A type alias for [`LittleEndian`]. pub type LE = LittleEndian; +macro_rules! impl_dbg_trait { + ($name:ident, $native:ident) => { + impl<O: ByteOrder> Debug for $name<O> { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + // This results in a format like "U16(42)". + f.debug_tuple(stringify!($name)).field(&self.get()).finish() + } + } + }; +} + +macro_rules! impl_dbg_traits { + ($name:ident, $native:ident, "floating point number") => { + #[cfg(not(no_fp_fmt_parse))] + impl_dbg_trait!($name, $native); + + #[cfg(no_fp_fmt_parse)] + impl<O: ByteOrder> Debug for $name<O> { + #[inline] + fn fmt(&self, _f: &mut Formatter<'_>) -> fmt::Result { + panic!("floating point support is turned off"); + } + } + }; + ($name:ident, $native:ident, "unsigned integer") => { + impl_dbg_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, "signed integer") => { + impl_dbg_traits!($name, $native, @all_types); + }; + ($name:ident, $native:ident, @all_types) => { + impl_dbg_trait!($name, $native); + }; +} + macro_rules! impl_fmt_trait { ($name:ident, $native:ident, $trait:ident) => { impl<O: ByteOrder> $trait for $name<O> { @@ -177,6 +214,8 @@ macro_rules! impl_fmt_trait { macro_rules! impl_fmt_traits { ($name:ident, $native:ident, "floating point number") => { + #[cfg(not(no_fp_fmt_parse))] + impl_fmt_trait!($name, $native, Display); }; ($name:ident, $native:ident, "unsigned integer") => { impl_fmt_traits!($name, $native, @all_types); @@ -687,16 +726,9 @@ example of how it can be used for parsing UDP packets. } } + impl_dbg_traits!($name, $native, $number_kind); impl_fmt_traits!($name, $native, $number_kind); impl_ops_traits!($name, $native, $number_kind); - - impl<O: ByteOrder> Debug for $name<O> { - #[inline] - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - // This results in a format like "U16(42)". - f.debug_tuple(stringify!($name)).field(&self.get()).finish() - } - } }; } diff --git a/rust/zerocopy/src/deprecated.rs b/rust/zerocopy/src/deprecated.rs index 24bafbf9adeb..59ddd35c77c6 100644 --- a/rust/zerocopy/src/deprecated.rs +++ b/rust/zerocopy/src/deprecated.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or diff --git a/rust/zerocopy/src/error.rs b/rust/zerocopy/src/error.rs index 7cb08c31d452..5eb30de934f3 100644 --- a/rust/zerocopy/src/error.rs +++ b/rust/zerocopy/src/error.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or diff --git a/rust/zerocopy/src/impls.rs b/rust/zerocopy/src/impls.rs index 22fd6c3d5d94..62e234c5202b 100644 --- a/rust/zerocopy/src/impls.rs +++ b/rust/zerocopy/src/impls.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or @@ -1359,7 +1359,7 @@ mod simd { #[cfg(not(no_zerocopy_aarch64_simd_1_59_0))] simd_arch_mod!( #[cfg(all( - target_arch = "aarch64", + target_arch = "aarch64", any( target_endian = "little", not(no_zerocopy_aarch64_simd_be_1_87_0) diff --git a/rust/zerocopy/src/layout.rs b/rust/zerocopy/src/layout.rs index 6015d0f2de52..b1fa0cd436db 100644 --- a/rust/zerocopy/src/layout.rs +++ b/rust/zerocopy/src/layout.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or @@ -71,6 +71,8 @@ impl SizeInfo { /// Attempts to create a `SizeInfo` from `Self` in which `elem_size` is a /// `NonZeroUsize`. If `elem_size` is 0, returns `None`. #[allow(unused)] + #[cfg_attr(not(zerocopy_inline_always), inline)] + #[cfg_attr(zerocopy_inline_always, inline(always))] const fn try_to_nonzero_elem_size(&self) -> Option<SizeInfo<NonZeroUsize>> { Some(match *self { SizeInfo::Sized { size } => SizeInfo::Sized { size }, diff --git a/rust/zerocopy/src/lib.rs b/rust/zerocopy/src/lib.rs index 3302d67602ab..572f0563fe2f 100644 --- a/rust/zerocopy/src/lib.rs +++ b/rust/zerocopy/src/lib.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2018 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or @@ -12,7 +12,7 @@ // After updating the following doc comment, make sure to run the following // command to update `README.md` based on its contents: // -// cargo -q run --manifest-path tools/Cargo.toml -p generate-readme > README.md +// (cd .. && cargo -q run --manifest-path tools/Cargo.toml -p generate-readme) > README.md //! ***<span style="font-size: 140%">Fast, safe, <span //! style="color:red;">compile error</span>. Pick two.</span>*** @@ -174,7 +174,7 @@ //! //! [Miri]: https://github.com/rust-lang/miri //! [Kani]: https://github.com/model-checking/kani -//! [soundness policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#soundness +//! [soundness policy]: https://github.com/google/zerocopy/blob/main/zerocopy/POLICIES.md#soundness //! //! # Relationship to Project Safe Transmute //! @@ -203,7 +203,7 @@ //! //! See our [MSRV policy]. //! -//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#msrv +//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/zerocopy/POLICIES.md#msrv //! //! # Changelog //! @@ -435,6 +435,8 @@ const _: () = { WARNING }; +#[cfg(all(any(feature = "derive", test), zerocopy_unstable_linux))] +pub use zerocopy_derive::most_traits; /// Implements [`KnownLayout`]. /// /// This derive analyzes various aspects of a type's layout that are needed for @@ -2832,7 +2834,7 @@ pub unsafe trait TryFromBytes { /// ``` /// /// [`try_mut_from_bytes`]: TryFromBytes::try_mut_from_bytes - /// + /// #[doc = codegen_header!("h5", "try_mut_from_bytes_with_elems")] /// /// See [`TryFromBytes::try_ref_from_bytes_with_elems`](#method.try_ref_from_bytes_with_elems.codegen). diff --git a/rust/zerocopy/src/macros.rs b/rust/zerocopy/src/macros.rs index b801d86a8fa6..ec67c03a44fc 100644 --- a/rust/zerocopy/src/macros.rs +++ b/rust/zerocopy/src/macros.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs index 5db08080141f..949b60a3f83e 100644 --- a/rust/zerocopy/src/pointer/inner.rs +++ b/rust/zerocopy/src/pointer/inner.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs index 1802d23563db..7ff0d43dad5e 100644 --- a/rust/zerocopy/src/pointer/invariant.rs +++ b/rust/zerocopy/src/pointer/invariant.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs index 3461f7f5ca80..d6eacc52febe 100644 --- a/rust/zerocopy/src/pointer/mod.rs +++ b/rust/zerocopy/src/pointer/mod.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2023 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs index b7c4ea56d2b2..7213f6f4a04e 100644 --- a/rust/zerocopy/src/pointer/ptr.rs +++ b/rust/zerocopy/src/pointer/ptr.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2023 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs index a534984b70d3..ef9836698203 100644 --- a/rust/zerocopy/src/pointer/transmute.rs +++ b/rust/zerocopy/src/pointer/transmute.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2025 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs index 860066d75196..e49f2a887ffa 100644 --- a/rust/zerocopy/src/ref.rs +++ b/rust/zerocopy/src/ref.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2024 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or diff --git a/rust/zerocopy/src/split_at.rs b/rust/zerocopy/src/split_at.rs index 9a67d5acbb0d..d7778425a31d 100644 --- a/rust/zerocopy/src/split_at.rs +++ b/rust/zerocopy/src/split_at.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2025 The Fuchsia Authors // // Licensed under the 2-Clause BSD License <LICENSE-BSD or diff --git a/rust/zerocopy/src/util/macro_util.rs b/rust/zerocopy/src/util/macro_util.rs index 1abb0fbeb46e..ceeb80432b0b 100644 --- a/rust/zerocopy/src/util/macro_util.rs +++ b/rust/zerocopy/src/util/macro_util.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2022 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/util/macros.rs b/rust/zerocopy/src/util/macros.rs index 43e4fd64ee15..7e63e3a54fc4 100644 --- a/rust/zerocopy/src/util/macros.rs +++ b/rust/zerocopy/src/util/macros.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2023 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 diff --git a/rust/zerocopy/src/util/mod.rs b/rust/zerocopy/src/util/mod.rs index d6d4c6c2fcd9..02fd4ed62741 100644 --- a/rust/zerocopy/src/util/mod.rs +++ b/rust/zerocopy/src/util/mod.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2023 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 @@ -150,6 +150,8 @@ pub(crate) fn validate_aligned_to<T: AsAddress, U>(t: T) -> Result<(), Alignment // Ensures that we add the minimum required padding. kani::ensures(|&p| p < align.get()), )] +#[cfg_attr(not(zerocopy_inline_always), inline)] +#[cfg_attr(zerocopy_inline_always, inline(always))] pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize { #[cfg(kani)] #[kani::proof_for_contract(padding_needed_for)] @@ -251,6 +253,8 @@ pub(crate) const fn round_down_to_next_multiple_of_alignment( n & mask } +#[cfg_attr(not(zerocopy_inline_always), inline)] +#[cfg_attr(zerocopy_inline_always, inline(always))] pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { if a.get() < b.get() { b @@ -259,6 +263,8 @@ pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { } } +#[cfg_attr(not(zerocopy_inline_always), inline)] +#[cfg_attr(zerocopy_inline_always, inline(always))] pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { if a.get() > b.get() { b diff --git a/rust/zerocopy/src/wrappers.rs b/rust/zerocopy/src/wrappers.rs index 266aec25fa58..1a8cf2b41d55 100644 --- a/rust/zerocopy/src/wrappers.rs +++ b/rust/zerocopy/src/wrappers.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT - +// // Copyright 2023 The Fuchsia Authors // // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 |