From 5fbcc6708fe32ef80122cd2a59ddca9d18b24d6e Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Thu, 6 Apr 2023 15:21:06 +0200 Subject: video/aperture: Drop primary argument With the preceding patches it's become defunct. Also I'm about to add a different boolean argument, so it's better to keep the confusion down to the absolute minimum. v2: Since the hypervfb patch got droppped (it's only a pci device for gen1 vm, not for gen2) there is one leftover user in an actual driver left to touch. v4: - fixes to commit message - fix Daniel's S-o-b address v5: - add back an S-o-b tag with Daniel's Intel address Signed-off-by: Daniel Vetter Signed-off-by: Daniel Vetter Signed-off-by: Thomas Zimmermann Cc: Thomas Zimmermann Cc: Javier Martinez Canillas Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: "K. Y. Srinivasan" Cc: Haiyang Zhang Cc: Wei Liu Cc: Dexuan Cui Cc: linux-hyperv@vger.kernel.org Reviewed-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20230406132109.32050-7-tzimmermann@suse.de --- include/linux/aperture.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/aperture.h b/include/linux/aperture.h index 442f15a57cad..7248727753be 100644 --- a/include/linux/aperture.h +++ b/include/linux/aperture.h @@ -14,7 +14,7 @@ int devm_aperture_acquire_for_platform_device(struct platform_device *pdev, resource_size_t size); int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size, - bool primary, const char *name); + const char *name); int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name); #else @@ -26,7 +26,7 @@ static inline int devm_aperture_acquire_for_platform_device(struct platform_devi } static inline int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size, - bool primary, const char *name) + const char *name) { return 0; } @@ -39,7 +39,6 @@ static inline int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, /** * aperture_remove_all_conflicting_devices - remove all existing framebuffers - * @primary: also kick vga16fb if present; only relevant for VGA devices * @name: a descriptive name of the requesting driver * * This function removes all graphics device drivers. Use this function on systems @@ -48,9 +47,9 @@ static inline int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, * Returns: * 0 on success, or a negative errno code otherwise */ -static inline int aperture_remove_all_conflicting_devices(bool primary, const char *name) +static inline int aperture_remove_all_conflicting_devices(const char *name) { - return aperture_remove_conflicting_devices(0, (resource_size_t)-1, primary, name); + return aperture_remove_conflicting_devices(0, (resource_size_t)-1, name); } #endif -- cgit v1.2.3 From 116b1c5a364bcbdc40be64d4f3ec9dbc32e264dd Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Thu, 6 Apr 2023 15:21:09 +0200 Subject: video/aperture: Provide a VGA helper for gma500 and internal use The hardware for gma500 is different from the rest, as it uses stolen framebuffer memory that is not available via PCI BAR. The regular PCI removal helper cannot detect the framebuffer, while the non-PCI helper misses possible conflicting VGA devices (i.e., a framebuffer or text console). Gma500 therefore calls both helpers to catch all cases. It's confusing as it implies that there's something about the PCI device that requires ownership management. The relationship between the PCI device and the VGA devices is non-obvious. At worst, readers might assume that calling two functions for clearing aperture ownership is a bug in the driver. Hence, move the PCI removal helper's code for VGA functionality into a separate function and call this function from gma500. Documents the purpose of each call to aperture helpers. The change contains comments and example code form the discussion at [1]. v5: * fix grammar in gma500 comment (Javier) Signed-off-by: Thomas Zimmermann Link: https://patchwork.kernel.org/project/dri-devel/patch/20230404201842.567344-1-daniel.vetter@ffwll.ch/ # 1 Reviewed-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20230406132109.32050-10-tzimmermann@suse.de --- include/linux/aperture.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/aperture.h b/include/linux/aperture.h index 7248727753be..1a9a88b11584 100644 --- a/include/linux/aperture.h +++ b/include/linux/aperture.h @@ -16,6 +16,8 @@ int devm_aperture_acquire_for_platform_device(struct platform_device *pdev, int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size, const char *name); +int __aperture_remove_legacy_vga_devices(struct pci_dev *pdev); + int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name); #else static inline int devm_aperture_acquire_for_platform_device(struct platform_device *pdev, @@ -31,6 +33,11 @@ static inline int aperture_remove_conflicting_devices(resource_size_t base, reso return 0; } +static inline int __aperture_remove_legacy_vga_devices(struct pci_dev *pdev) +{ + return 0; +} + static inline int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name) { return 0; -- cgit v1.2.3 From 26662d7347a058ca497792c4b22ac91cc415cbf6 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Thu, 20 Apr 2023 00:14:12 -0700 Subject: bpf: Add bpf_dynptr_size bpf_dynptr_size returns the number of usable bytes in a dynptr. Signed-off-by: Joanne Koong Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Acked-by: John Fastabend Link: https://lore.kernel.org/bpf/20230420071414.570108-4-joannelkoong@gmail.com --- include/linux/bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e53ceee1df37..456f33b9d205 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1197,7 +1197,7 @@ enum bpf_dynptr_type { }; int bpf_dynptr_check_size(u32 size); -u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr); +u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr); #ifdef CONFIG_BPF_JIT int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); -- cgit v1.2.3 From 7b9c13dd4d0ebbe89dfd7e1ecd09696037622101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Miros=C5=82aw?= Date: Mon, 1 May 2023 17:39:45 -0700 Subject: Input: i8042 - add missing include MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit include uses ENODEV when included with !IS_ENABLED(CONFIG_SERIO_I8042) and so need to include it. Signed-off-by: Michał Mirosław Link: https://lore.kernel.org/r/49fd4d400d1ab62095e5ed75a6637f883c0d071b.1682795105.git.mirq-linux@rere.qmqm.pl Signed-off-by: Dmitry Torokhov --- include/linux/i8042.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/i8042.h b/include/linux/i8042.h index 0261e2fb3636..95b07f8b77fe 100644 --- a/include/linux/i8042.h +++ b/include/linux/i8042.h @@ -3,6 +3,7 @@ #define _LINUX_I8042_H +#include #include /* -- cgit v1.2.3 From 407958a0e980b9e1842ab87b5a1040521e1e24e9 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 4 May 2023 21:33:10 -0700 Subject: bpf: encapsulate precision backtracking bookkeeping Add struct backtrack_state and straightforward API around it to keep track of register and stack masks used and maintained during precision backtracking process. Having this logic separately allow to keep high-level backtracking algorithm cleaner, but also it sets us up to cleanly keep track of register and stack masks per frame, allowing (with some further logic adjustments) to perform precision backpropagation across multiple frames (i.e., subprog calls). Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20230505043317.3629845-4-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3dd29a53b711..33f541366f4e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -238,6 +238,10 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +#define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ + (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ + (1 << BPF_REG_5)) + #define BPF_DYNPTR_SIZE sizeof(struct bpf_dynptr_kern) #define BPF_DYNPTR_NR_SLOTS (BPF_DYNPTR_SIZE / BPF_REG_SIZE) @@ -541,6 +545,15 @@ struct bpf_subprog_info { bool is_async_cb; }; +struct bpf_verifier_env; + +struct backtrack_state { + struct bpf_verifier_env *env; + u32 frame; + u32 reg_masks[MAX_CALL_FRAMES]; + u64 stack_masks[MAX_CALL_FRAMES]; +}; + /* single container for all structs * one verifier_env per bpf_check() call */ @@ -578,6 +591,7 @@ struct bpf_verifier_env { int *insn_stack; int cur_stack; } cfg; + struct backtrack_state bt; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ -- cgit v1.2.3 From d9439c21a9e4769bfd83a03ab39056164d44ac31 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Thu, 4 May 2023 21:33:11 -0700 Subject: bpf: improve precision backtrack logging Add helper to format register and stack masks in more human-readable format. Adjust logging a bit during backtrack propagation and especially during forcing precision fallback logic to make it clearer what's going on (with log_level=2, of course), and also start reporting affected frame depth. This is in preparation for having more than one active frame later when precision propagation between subprog calls is added. Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20230505043317.3629845-5-andrii@kernel.org Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 33f541366f4e..5b11a3b0fec0 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -18,8 +18,11 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -/* size of type_str_buf in bpf_verifier. */ -#define TYPE_STR_BUF_LEN 128 +/* size of tmp_str_buf in bpf_verifier. + * we need at least 306 bytes to fit full stack mask representation + * (in the "-8,-16,...,-512" form) + */ +#define TMP_STR_BUF_LEN 320 /* Liveness marks, used for registers and spilled-regs (in stack slots). * Read marks propagate upwards until they find a write mark; they record that @@ -620,8 +623,10 @@ struct bpf_verifier_env { /* Same as scratched_regs but for stack slots */ u64 scratched_stack_slots; u64 prev_log_pos, prev_insn_print_pos; - /* buffer used in reg_type_str() to generate reg_type string */ - char type_str_buf[TYPE_STR_BUF_LEN]; + /* buffer used to generate temporary string representations, + * e.g., in reg_type_str() to generate reg_type string + */ + char tmp_str_buf[TMP_STR_BUF_LEN]; }; __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, -- cgit v1.2.3 From 3bda08b63670c39be390fcb00e7718775508e673 Mon Sep 17 00:00:00 2001 From: Daniel Rosenberg Date: Fri, 5 May 2023 18:31:30 -0700 Subject: bpf: Allow NULL buffers in bpf_dynptr_slice(_rw) bpf_dynptr_slice(_rw) uses a user provided buffer if it can not provide a pointer to a block of contiguous memory. This buffer is unused in the case of local dynptrs, and may be unused in other cases as well. There is no need to require the buffer, as the kfunc can just return NULL if it was needed and not provided. This adds another kfunc annotation, __opt, which combines with __sz and __szk to allow the buffer associated with the size to be NULL. If the buffer is NULL, the verifier does not check that the buffer is of sufficient size. Signed-off-by: Daniel Rosenberg Link: https://lore.kernel.org/r/20230506013134.2492210-2-drosen@google.com Signed-off-by: Alexei Starovoitov --- include/linux/skbuff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 738776ab8838..8ddb4af1a501 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4033,7 +4033,7 @@ __skb_header_pointer(const struct sk_buff *skb, int offset, int len, if (likely(hlen - offset >= len)) return (void *)data + offset; - if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0)) + if (!skb || !buffer || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0)) return NULL; return buffer; -- cgit v1.2.3 From 69da5aa99ea67e86d3461fb281eadc952cc2914f Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Fri, 7 Apr 2023 07:47:33 -0400 Subject: regmap-irq: Drop map from handle_mask_sync() parameters Remove the map parameter from the struct regmap_irq_chip callback handle_mask_sync() because it can be passed via the irq_drv_data parameter instead. The gpio-104-dio-48e driver is the only consumer of this callback and is thus updated accordingly. Reviewed-by: Linus Walleij Date: Tue, 2 May 2023 15:28:11 +0900 Subject: spi: s3c64xx: change polling mode to optional Previously, Polling mode was supported as quirk for SOC without DMA. To provide more flexible support for polling mode, it changed to polling mode when the 'dmas' property is not present in the devicetree, rather than using a quirk. Signed-off-by: Jaewon Kim Date: Thu, 4 May 2023 16:30:00 +0530 Subject: perf/core: Rework forwarding of {task|cpu}-clock events Currently, PERF_TYPE_SOFTWARE is treated specially since task-clock and cpu-clock events are interfaced through it but internally gets forwarded to their own pmus. Rework this by overwriting event->attr.type in perf_swevent_init() which will cause perf_init_event() to retry with updated type and event will automatically get forwarded to right pmu. With the change, SW pmu no longer needs to be treated specially and can be included in 'pmu_idr' list. Suggested-by: Peter Zijlstra Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20230504110003.2548-2-ravi.bangoria@amd.com --- include/linux/perf_event.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index d5628a7b5eaa..bf4f346d6d70 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -295,6 +295,8 @@ struct perf_event_pmu_context; struct perf_output_handle; +#define PMU_NULL_DEV ((void *)(~0UL)) + /** * struct pmu - generic performance monitoring unit */ @@ -827,6 +829,14 @@ struct perf_event { void *security; #endif struct list_head sb_list; + + /* + * Certain events gets forwarded to another pmu internally by over- + * writing kernel copy of event->attr.type without user being aware + * of it. event->orig_type contains original 'type' requested by + * user. + */ + __u32 orig_type; #endif /* CONFIG_PERF_EVENTS */ }; -- cgit v1.2.3 From ca528cc501896a808dc79c3c0544369d23b331c8 Mon Sep 17 00:00:00 2001 From: Ricardo Neri Date: Thu, 6 Apr 2023 13:31:45 -0700 Subject: sched/topology: Remove SHARED_CHILD from ASYM_PACKING Only x86 and Power7 use ASYM_PACKING. They use it differently. Power7 has cores of equal priority, but the SMT siblings of a core have different priorities. Parent scheduling domains do not need (nor have) the ASYM_PACKING flag. SHARED_CHILD is not needed. Using SHARED_PARENT would cause the topology debug code to complain. X86 has cores of different priority, but all the SMT siblings of the core have equal priority. It needs ASYM_PACKING at the MC level, but not at the SMT level (it also needs it at upper levels if they have scheduling groups of different priority). Removing ASYM_PACKING from the SMT domain causes the topology debug code to complain. Remove SHARED_CHILD for now. We still need a topology check that satisfies both architectures. Suggested-by: Valentin Schneider Signed-off-by: Ricardo Neri Signed-off-by: Peter Zijlstra (Intel) Tested-by: Zhang Rui Link: https://lore.kernel.org/r/20230406203148.19182-10-ricardo.neri-calderon@linux.intel.com --- include/linux/sched/sd_flags.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 57bde66d95f7..fad77b5172e2 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -132,12 +132,9 @@ SD_FLAG(SD_SERIALIZE, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) /* * Place busy tasks earlier in the domain * - * SHARED_CHILD: Usually set on the SMT level. Technically could be set further - * up, but currently assumed to be set from the base domain - * upwards (see update_top_cache_domain()). * NEEDS_GROUPS: Load balancing flag. */ -SD_FLAG(SD_ASYM_PACKING, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) +SD_FLAG(SD_ASYM_PACKING, SDF_NEEDS_GROUPS) /* * Prefer to place tasks in a sibling domain -- cgit v1.2.3 From 6121cd9ef911432b14c2a17aefaf8cd2f3cfcdff Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Fri, 28 Apr 2023 14:24:51 +0200 Subject: fbdev: Move I/O read and write code into helper functions Move the existing I/O read and write code for I/O memory into the new helpers fb_cfb_read() and fb_cfb_write(). Make them the default fp_ops. No functional changes. In the near term, the new functions will be useful to the DRM subsystem, which currently provides it's own implementation. It can then use the shared code. In the longer term, it might make sense to revise the I/O helper's default status and make them opt-in by the driver. Systems that don't use them would not contain the code any longer. v2: * add detailed commit message (Javier) * rename fb_cfb_() to fb_io_() (Geert) * add fixes that got lost while moving the code (Geert) Signed-off-by: Thomas Zimmermann Tested-by: Sui Jingfeng Reviewed-by: Javier Martinez Canillas Acked-by: Helge Deller Reviewed-by: Sam Ravnborg Link: https://patchwork.freedesktop.org/patch/msgid/20230428122452.4856-19-tzimmermann@suse.de --- include/linux/fb.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 08cb47da71f8..ec978a4969a9 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -576,9 +576,19 @@ struct fb_info { extern int fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var); extern int fb_pan_display(struct fb_info *info, struct fb_var_screeninfo *var); extern int fb_blank(struct fb_info *info, int blank); + +/* + * Drawing operations where framebuffer is in I/O memory + */ + extern void cfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect); extern void cfb_copyarea(struct fb_info *info, const struct fb_copyarea *area); extern void cfb_imageblit(struct fb_info *info, const struct fb_image *image); +extern ssize_t fb_io_read(struct fb_info *info, char __user *buf, + size_t count, loff_t *ppos); +extern ssize_t fb_io_write(struct fb_info *info, const char __user *buf, + size_t count, loff_t *ppos); + /* * Drawing operations where framebuffer is in system RAM */ -- cgit v1.2.3 From 111cd11bbc54850f24191c52ff217da88a5e639b Mon Sep 17 00:00:00 2001 From: Juri Lelli Date: Mon, 8 May 2023 09:58:50 +0200 Subject: sched/cpuset: Bring back cpuset_mutex Turns out percpu_cpuset_rwsem - commit 1243dc518c9d ("cgroup/cpuset: Convert cpuset_mutex to percpu_rwsem") - wasn't such a brilliant idea, as it has been reported to cause slowdowns in workloads that need to change cpuset configuration frequently and it is also not implementing priority inheritance (which causes troubles with realtime workloads). Convert percpu_cpuset_rwsem back to regular cpuset_mutex. Also grab it only for SCHED_DEADLINE tasks (other policies don't care about stable cpusets anyway). Signed-off-by: Juri Lelli Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- include/linux/cpuset.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 980b76a1237e..f90e6325d707 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -71,8 +71,8 @@ extern void cpuset_init_smp(void); extern void cpuset_force_rebuild(void); extern void cpuset_update_active_cpus(void); extern void cpuset_wait_for_hotplug(void); -extern void cpuset_read_lock(void); -extern void cpuset_read_unlock(void); +extern void cpuset_lock(void); +extern void cpuset_unlock(void); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); @@ -189,8 +189,8 @@ static inline void cpuset_update_active_cpus(void) static inline void cpuset_wait_for_hotplug(void) { } -static inline void cpuset_read_lock(void) { } -static inline void cpuset_read_unlock(void) { } +static inline void cpuset_lock(void) { } +static inline void cpuset_unlock(void) { } static inline void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask) -- cgit v1.2.3 From 6c24849f5515e4966d94fa5279bdff4acf2e9489 Mon Sep 17 00:00:00 2001 From: Juri Lelli Date: Mon, 8 May 2023 09:58:51 +0200 Subject: sched/cpuset: Keep track of SCHED_DEADLINE task in cpusets Qais reported that iterating over all tasks when rebuilding root domains for finding out which ones are DEADLINE and need their bandwidth correctly restored on such root domains can be a costly operation (10+ ms delays on suspend-resume). To fix the problem keep track of the number of DEADLINE tasks belonging to each cpuset and then use this information (followup patch) to only perform the above iteration if DEADLINE tasks are actually present in the cpuset for which a corresponding root domain is being rebuilt. Reported-by: Qais Yousef Link: https://lore.kernel.org/lkml/20230206221428.2125324-1-qyousef@layalina.io/ Signed-off-by: Juri Lelli Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- include/linux/cpuset.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index f90e6325d707..d629094fac6e 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -71,6 +71,8 @@ extern void cpuset_init_smp(void); extern void cpuset_force_rebuild(void); extern void cpuset_update_active_cpus(void); extern void cpuset_wait_for_hotplug(void); +extern void inc_dl_tasks_cs(struct task_struct *task); +extern void dec_dl_tasks_cs(struct task_struct *task); extern void cpuset_lock(void); extern void cpuset_unlock(void); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); @@ -189,6 +191,8 @@ static inline void cpuset_update_active_cpus(void) static inline void cpuset_wait_for_hotplug(void) { } +static inline void inc_dl_tasks_cs(struct task_struct *task) { } +static inline void dec_dl_tasks_cs(struct task_struct *task) { } static inline void cpuset_lock(void) { } static inline void cpuset_unlock(void) { } -- cgit v1.2.3 From 85989106feb734437e2d598b639991b9185a43a6 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Mon, 8 May 2023 09:58:53 +0200 Subject: sched/deadline: Create DL BW alloc, free & check overflow interface While moving a set of tasks between exclusive cpusets, cpuset_can_attach() -> task_can_attach() calls dl_cpu_busy(..., p) for DL BW overflow checking and per-task DL BW allocation on the destination root_domain for the DL tasks in this set. This approach has the issue of not freeing already allocated DL BW in the following error cases: (1) The set of tasks includes multiple DL tasks and DL BW overflow checking fails for one of the subsequent DL tasks. (2) Another controller next to the cpuset controller which is attached to the same cgroup fails in its can_attach(). To address this problem rework dl_cpu_busy(): (1) Split it into dl_bw_check_overflow() & dl_bw_alloc() and add a dedicated dl_bw_free(). (2) dl_bw_alloc() & dl_bw_free() take a `u64 dl_bw` parameter instead of a `struct task_struct *p` used in dl_cpu_busy(). This allows to allocate DL BW for a set of tasks too rather than only for a single task. Signed-off-by: Dietmar Eggemann Signed-off-by: Juri Lelli Signed-off-by: Tejun Heo --- include/linux/sched.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index eed5d65b8d1f..0bee06542450 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1853,6 +1853,8 @@ current_restore_flags(unsigned long orig_flags, unsigned long flags) extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_effective_cpus); +extern int dl_bw_alloc(int cpu, u64 dl_bw); +extern void dl_bw_free(int cpu, u64 dl_bw); #ifdef CONFIG_SMP extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); -- cgit v1.2.3 From 2ef269ef1ac006acf974793d975539244d77b28f Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Mon, 8 May 2023 09:58:54 +0200 Subject: cgroup/cpuset: Free DL BW in case can_attach() fails cpuset_can_attach() can fail. Postpone DL BW allocation until all tasks have been checked. DL BW is not allocated per-task but as a sum over all DL tasks migrating. If multiple controllers are attached to the cgroup next to the cpuset controller a non-cpuset can_attach() can fail. In this case free DL BW in cpuset_cancel_attach(). Finally, update cpuset DL task count (nr_deadline_tasks) only in cpuset_attach(). Suggested-by: Waiman Long Signed-off-by: Dietmar Eggemann Signed-off-by: Juri Lelli Reviewed-by: Waiman Long Signed-off-by: Tejun Heo --- include/linux/sched.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 0bee06542450..2553918f0b61 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1852,7 +1852,7 @@ current_restore_flags(unsigned long orig_flags, unsigned long flags) } extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); -extern int task_can_attach(struct task_struct *p, const struct cpumask *cs_effective_cpus); +extern int task_can_attach(struct task_struct *p); extern int dl_bw_alloc(int cpu, u64 dl_bw); extern void dl_bw_free(int cpu, u64 dl_bw); #ifdef CONFIG_SMP -- cgit v1.2.3 From 1da82598cfc22f43fb0a3bd47774f7e886cc8b62 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 18 Mar 2023 13:51:10 -0700 Subject: srcu: Remove extraneous parentheses from srcu_read_lock() etc. This commit removes extraneous parentheses from srcu_read_lock(), srcu_read_lock_nmisafe(), srcu_read_unlock(), and srcu_read_unlock_nmisafe(). Looks like someone was once a macro. Cc: Christoph Hellwig Tested-by: Sachin Sant Tested-by: "Zhang, Qiang1" Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 41c4b26fb1c1..eb92a50a4599 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -212,7 +212,7 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) srcu_check_nmi_safety(ssp, false); retval = __srcu_read_lock(ssp); - srcu_lock_acquire(&(ssp)->dep_map); + srcu_lock_acquire(&ssp->dep_map); return retval; } @@ -229,7 +229,7 @@ static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp srcu_check_nmi_safety(ssp, true); retval = __srcu_read_lock_nmisafe(ssp); - rcu_lock_acquire(&(ssp)->dep_map); + rcu_lock_acquire(&ssp->dep_map); return retval; } @@ -284,7 +284,7 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) { WARN_ON_ONCE(idx & ~0x1); srcu_check_nmi_safety(ssp, false); - srcu_lock_release(&(ssp)->dep_map); + srcu_lock_release(&ssp->dep_map); __srcu_read_unlock(ssp, idx); } @@ -300,7 +300,7 @@ static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) { WARN_ON_ONCE(idx & ~0x1); srcu_check_nmi_safety(ssp, true); - rcu_lock_release(&(ssp)->dep_map); + rcu_lock_release(&ssp->dep_map); __srcu_read_unlock_nmisafe(ssp, idx); } -- cgit v1.2.3 From 7e3f926bf4538cb4988b3e3f8bc1cb4a603b2ef6 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Wed, 1 Feb 2023 16:09:54 +0100 Subject: rcu/kvfree: Eliminate k[v]free_rcu() single argument macro The kvfree_rcu() and kfree_rcu() APIs are hazardous in that if you forget the second argument, it works, but might sleep. This sleeping can be a correctness bug from atomic contexts, and even in non-atomic contexts it might introduce unacceptable latencies. This commit therefore removes the single-argument kvfree_rcu() and kfree_rcu() macros. Code that would have previously used these single-argument kvfree_rcu() and kfree_rcu() macros should instead use kvfree_rcu_mightsleep() or kfree_rcu_mightsleep(). [ paulmck: Apply Joel Fernandes feedback. ] Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney Signed-off-by: Joel Fernandes (Google) --- include/linux/rcupdate.h | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index dcd2cf1e8326..744869ef930a 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -957,9 +957,8 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) /** * kfree_rcu() - kfree an object after a grace period. - * @ptr: pointer to kfree for both single- and double-argument invocations. - * @rhf: the name of the struct rcu_head within the type of @ptr, - * but only for double-argument invocations. + * @ptr: pointer to kfree for double-argument invocations. + * @rhf: the name of the struct rcu_head within the type of @ptr. * * Many rcu callbacks functions just call kfree() on the base structure. * These functions are trivial, but their size adds up, and furthermore @@ -984,26 +983,18 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * The BUILD_BUG_ON check must not involve any function calls, hence the * checks are done in macros here. */ -#define kfree_rcu(ptr, rhf...) kvfree_rcu(ptr, ## rhf) +#define kfree_rcu(ptr, rhf) kvfree_rcu_arg_2(ptr, rhf) +#define kvfree_rcu(ptr, rhf) kvfree_rcu_arg_2(ptr, rhf) /** - * kvfree_rcu() - kvfree an object after a grace period. - * - * This macro consists of one or two arguments and it is - * based on whether an object is head-less or not. If it - * has a head then a semantic stays the same as it used - * to be before: - * - * kvfree_rcu(ptr, rhf); - * - * where @ptr is a pointer to kvfree(), @rhf is the name - * of the rcu_head structure within the type of @ptr. + * kfree_rcu_mightsleep() - kfree an object after a grace period. + * @ptr: pointer to kfree for single-argument invocations. * * When it comes to head-less variant, only one argument * is passed and that is just a pointer which has to be * freed after a grace period. Therefore the semantic is * - * kvfree_rcu(ptr); + * kfree_rcu_mightsleep(ptr); * * where @ptr is the pointer to be freed by kvfree(). * @@ -1012,13 +1003,9 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * annotation. Otherwise, please switch and embed the * rcu_head structure within the type of @ptr. */ -#define kvfree_rcu(...) KVFREE_GET_MACRO(__VA_ARGS__, \ - kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__) - +#define kfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr) #define kvfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr) -#define kfree_rcu_mightsleep(ptr) kvfree_rcu_mightsleep(ptr) -#define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME #define kvfree_rcu_arg_2(ptr, rhf) \ do { \ typeof (ptr) ___p = (ptr); \ -- cgit v1.2.3 From 212bc1ce618dd7c734920a68988fe4b473f09f40 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Tue, 9 May 2023 12:00:55 +0100 Subject: regmap-irq: Fix typo in documentation for .get_irq_reg() It refers to a non-existent "num_type_settings" value, which is an old name I'd used during development of config registers and later dropped because it wasn't very clear. The correct bound for the range is num_config_regs, which can be verified by checking the implementation. Signed-off-by: Aidan MacDonald Date: Tue, 9 May 2023 12:00:56 +0100 Subject: regmap-irq: Remove virtual registers No remaining users, and it's been replaced by config registers. Signed-off-by: Aidan MacDonald Date: Mon, 20 Mar 2023 18:37:51 +0100 Subject: rcu: Remove RCU_NONIDLE() Since there are now exactly _zero_ users of RCU_NONIDLE(), make it go away before someone else decides to (ab)use it. [ paulmck: Remove extraneous whitespace. ] Signed-off-by: Peter Zijlstra (Intel) Acked-by: Mark Rutland Acked-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 25 ------------------------- 1 file changed, 25 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index dcd2cf1e8326..aae31a3e28dd 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -156,31 +156,6 @@ static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; } static inline void rcu_nocb_flush_deferred_wakeup(void) { } #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ -/** - * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers - * @a: Code that RCU needs to pay attention to. - * - * RCU read-side critical sections are forbidden in the inner idle loop, - * that is, between the ct_idle_enter() and the ct_idle_exit() -- RCU - * will happily ignore any such read-side critical sections. However, - * things like powertop need tracepoints in the inner idle loop. - * - * This macro provides the way out: RCU_NONIDLE(do_something_with_RCU()) - * will tell RCU that it needs to pay attention, invoke its argument - * (in this example, calling the do_something_with_RCU() function), - * and then tell RCU to go back to ignoring this CPU. It is permissible - * to nest RCU_NONIDLE() wrappers, but not indefinitely (but the limit is - * on the order of a million or so, even on 32-bit systems). It is - * not legal to block within RCU_NONIDLE(), nor is it permissible to - * transfer control either into or out of RCU_NONIDLE()'s statement. - */ -#define RCU_NONIDLE(a) \ - do { \ - ct_irq_enter_irqson(); \ - do { a; } while (0); \ - ct_irq_exit_irqson(); \ - } while (0) - /* * Note a quasi-voluntary context switch for RCU-tasks's benefit. * This is a macro rather than an inline function to avoid #include hell. -- cgit v1.2.3 From f05cbadce7e409b38acdf21f0a05d4420afa1b11 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Thu, 11 May 2023 10:13:39 +0100 Subject: regmap-irq: Remove type registers No remaining users, these have been replaced by config registers. Signed-off-by: Aidan MacDonald Date: Thu, 11 May 2023 10:13:40 +0100 Subject: regmap-irq: Remove support for not_fixed_stride No remaining users, use a custom .get_irq_reg() callback instead. Signed-off-by: Aidan MacDonald Date: Thu, 11 May 2023 10:21:08 -0700 Subject: net: phy: Allow drivers to always call into ->suspend() A few PHY drivers are currently attempting to not suspend the PHY when Wake-on-LAN is enabled, however that code is not currently executing at all due to an early check in phy_suspend(). This prevents PHY drivers from making an appropriate decisions and put the hardware into a low power state if desired. In order to allow the PHY drivers to opt into getting their ->suspend routine to be called, add a PHY_ALWAYS_CALL_SUSPEND bit which can be set. A boolean that tracks whether the PHY or the attached MAC has Wake-on-LAN enabled is also provided for convenience. If phydev::wol_enabled then the PHY shall not prevent its own Wake-on-LAN detection logic from working and shall not prevent the Ethernet MAC from receiving packets for matching. Reviewed-by: Simon Horman Reviewed-by: Andrew Lunn Signed-off-by: Florian Fainelli Signed-off-by: David S. Miller --- include/linux/phy.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index c5a0dc829714..e0df8b3c2bdb 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -86,6 +86,7 @@ extern const int phy_10gbit_features_array[1]; #define PHY_IS_INTERNAL 0x00000001 #define PHY_RST_AFTER_CLK_EN 0x00000002 #define PHY_POLL_CABLE_TEST 0x00000004 +#define PHY_ALWAYS_CALL_SUSPEND 0x00000008 #define MDIO_DEVICE_IS_PHY 0x80000000 /** @@ -548,6 +549,8 @@ struct macsec_ops; * @downshifted_rate: Set true if link speed has been downshifted. * @is_on_sfp_module: Set true if PHY is located on an SFP module. * @mac_managed_pm: Set true if MAC driver takes of suspending/resuming PHY + * @wol_enabled: Set to true if the PHY or the attached MAC have Wake-on-LAN + * enabled. * @state: State of the PHY for management purposes * @dev_flags: Device-specific flags used by the PHY driver. * @@ -644,6 +647,7 @@ struct phy_device { unsigned downshifted_rate:1; unsigned is_on_sfp_module:1; unsigned mac_managed_pm:1; + unsigned wol_enabled:1; unsigned autoneg:1; /* The most recently read link state */ -- cgit v1.2.3 From 8baddaa9d4bac939004b5058f3ade7e2bf0a6e43 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Thu, 11 May 2023 10:21:09 -0700 Subject: net: phy: broadcom: Add support for Wake-on-LAN Add support for WAKE_UCAST, WAKE_MCAST, WAKE_BCAST, WAKE_MAGIC and WAKE_MAGICSECURE. This is only supported with the BCM54210E and compatible Ethernet PHYs. Using the in-band interrupt or an out of band GPIO interrupts are supported. Broadcom PHYs will generate a Wake-on-LAN level low interrupt on LED4 as soon as one of the supported patterns is being matched. That includes generating such an interrupt even if the PHY is operated during normal modes. If WAKE_UCAST is selected, this could lead to the LED4 interrupt firing up for every packet being received which is absolutely undesirable from a performance point of view. Because the Wake-on-LAN configuration can be set long before the system is actually put to sleep, we cannot have an interrupt service routine to clear on read the interrupt status register and ensure that new packet matches will be detected. It is desirable to enable the Wake-on-LAN interrupt as late as possible during the system suspend process such that we limit the number of interrupts to be handled by the system, but also conversely feed into the Linux's system suspend way of dealing with interrupts in and around the points of no return. Reviewed-by: Simon Horman Signed-off-by: Florian Fainelli Signed-off-by: David S. Miller --- include/linux/brcmphy.h | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'include/linux') diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 9e77165f3ef6..e9afbfb6d7a5 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -89,6 +89,7 @@ #define MII_BCM54XX_EXP_SEL 0x17 /* Expansion register select */ #define MII_BCM54XX_EXP_SEL_TOP 0x0d00 /* TOP_MISC expansion register select */ #define MII_BCM54XX_EXP_SEL_SSD 0x0e00 /* Secondary SerDes select */ +#define MII_BCM54XX_EXP_SEL_WOL 0x0e00 /* Wake-on-LAN expansion select register */ #define MII_BCM54XX_EXP_SEL_ER 0x0f00 /* Expansion register select */ #define MII_BCM54XX_EXP_SEL_ETC 0x0d00 /* Expansion register spare + 2k mem */ @@ -253,6 +254,9 @@ #define BCM54XX_TOP_MISC_IDDQ_SD (1 << 2) #define BCM54XX_TOP_MISC_IDDQ_SR (1 << 3) +#define BCM54XX_TOP_MISC_LED_CTL (MII_BCM54XX_EXP_SEL_TOP + 0x0C) +#define BCM54XX_LED4_SEL_INTR BIT(1) + /* * BCM5482: Secondary SerDes registers */ @@ -272,6 +276,57 @@ #define BCM54612E_EXP_SPARE0 (MII_BCM54XX_EXP_SEL_ETC + 0x34) #define BCM54612E_LED4_CLK125OUT_EN (1 << 1) + +/* Wake-on-LAN registers */ +#define BCM54XX_WOL_MAIN_CTL (MII_BCM54XX_EXP_SEL_WOL + 0x80) +#define BCM54XX_WOL_EN BIT(0) +#define BCM54XX_WOL_MODE_SINGLE_MPD 0 +#define BCM54XX_WOL_MODE_SINGLE_MPDSEC 1 +#define BCM54XX_WOL_MODE_DUAL 2 +#define BCM54XX_WOL_MODE_SHIFT 1 +#define BCM54XX_WOL_MODE_MASK 0x3 +#define BCM54XX_WOL_MP_MSB_FF_EN BIT(3) +#define BCM54XX_WOL_SECKEY_OPT_4B 0 +#define BCM54XX_WOL_SECKEY_OPT_6B 1 +#define BCM54XX_WOL_SECKEY_OPT_8B 2 +#define BCM54XX_WOL_SECKEY_OPT_SHIFT 4 +#define BCM54XX_WOL_SECKEY_OPT_MASK 0x3 +#define BCM54XX_WOL_L2_TYPE_CHK BIT(6) +#define BCM54XX_WOL_L4IPV4UDP_CHK BIT(7) +#define BCM54XX_WOL_L4IPV6UDP_CHK BIT(8) +#define BCM54XX_WOL_UDPPORT_CHK BIT(9) +#define BCM54XX_WOL_CRC_CHK BIT(10) +#define BCM54XX_WOL_SECKEY_MODE BIT(11) +#define BCM54XX_WOL_RST BIT(12) +#define BCM54XX_WOL_DIR_PKT_EN BIT(13) +#define BCM54XX_WOL_MASK_MODE_DA_FF 0 +#define BCM54XX_WOL_MASK_MODE_DA_MPD 1 +#define BCM54XX_WOL_MASK_MODE_DA_ONLY 2 +#define BCM54XX_WOL_MASK_MODE_MPD 3 +#define BCM54XX_WOL_MASK_MODE_SHIFT 14 +#define BCM54XX_WOL_MASK_MODE_MASK 0x3 + +#define BCM54XX_WOL_INNER_PROTO (MII_BCM54XX_EXP_SEL_WOL + 0x81) +#define BCM54XX_WOL_OUTER_PROTO (MII_BCM54XX_EXP_SEL_WOL + 0x82) +#define BCM54XX_WOL_OUTER_PROTO2 (MII_BCM54XX_EXP_SEL_WOL + 0x83) + +#define BCM54XX_WOL_MPD_DATA1(x) (MII_BCM54XX_EXP_SEL_WOL + 0x84 + (x)) +#define BCM54XX_WOL_MPD_DATA2(x) (MII_BCM54XX_EXP_SEL_WOL + 0x87 + (x)) +#define BCM54XX_WOL_SEC_KEY_8B (MII_BCM54XX_EXP_SEL_WOL + 0x8A) +#define BCM54XX_WOL_MASK(x) (MII_BCM54XX_EXP_SEL_WOL + 0x8B + (x)) +#define BCM54XX_SEC_KEY_STORE(x) (MII_BCM54XX_EXP_SEL_WOL + 0x8E) +#define BCM54XX_WOL_SHARED_CNT (MII_BCM54XX_EXP_SEL_WOL + 0x92) + +#define BCM54XX_WOL_INT_MASK (MII_BCM54XX_EXP_SEL_WOL + 0x93) +#define BCM54XX_WOL_PKT1 BIT(0) +#define BCM54XX_WOL_PKT2 BIT(1) +#define BCM54XX_WOL_DIR BIT(2) +#define BCM54XX_WOL_ALL_INTRS (BCM54XX_WOL_PKT1 | \ + BCM54XX_WOL_PKT2 | \ + BCM54XX_WOL_DIR) + +#define BCM54XX_WOL_INT_STATUS (MII_BCM54XX_EXP_SEL_WOL + 0x94) + /*****************************************************************************/ /* Fast Ethernet Transceiver definitions. */ /*****************************************************************************/ -- cgit v1.2.3 From b51f4113ebb02011f0ca86abc3134b28d2071b6a Mon Sep 17 00:00:00 2001 From: Yunsheng Lin Date: Thu, 11 May 2023 09:12:12 +0800 Subject: net: introduce and use skb_frag_fill_page_desc() Most users use __skb_frag_set_page()/skb_frag_off_set()/ skb_frag_size_set() to fill the page desc for a skb frag. Introduce skb_frag_fill_page_desc() to do that. net/bpf/test_run.c does not call skb_frag_off_set() to set the offset, "copy_from_user(page_address(page), ...)" and 'shinfo' being part of the 'data' kzalloced in bpf_test_init() suggest that it is assuming offset to be initialized as zero, so call skb_frag_fill_page_desc() with offset being zero for this case. Also, skb_frag_set_page() is not used anymore, so remove it. Signed-off-by: Yunsheng Lin Reviewed-by: Leon Romanovsky Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/skbuff.h | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 738776ab8838..30be21c7d05f 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2411,6 +2411,15 @@ static inline unsigned int skb_pagelen(const struct sk_buff *skb) return skb_headlen(skb) + __skb_pagelen(skb); } +static inline void skb_frag_fill_page_desc(skb_frag_t *frag, + struct page *page, + int off, int size) +{ + frag->bv_page = page; + frag->bv_offset = off; + skb_frag_size_set(frag, size); +} + static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, int i, struct page *page, int off, int size) @@ -2422,9 +2431,7 @@ static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, * that not all callers have unique ownership of the page but rely * on page_is_pfmemalloc doing the right thing(tm). */ - frag->bv_page = page; - frag->bv_offset = off; - skb_frag_size_set(frag, size); + skb_frag_fill_page_desc(frag, page, off, size); } /** @@ -3496,20 +3503,6 @@ static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page) frag->bv_page = page; } -/** - * skb_frag_set_page - sets the page contained in a paged fragment of an skb - * @skb: the buffer - * @f: the fragment offset - * @page: the page to set - * - * Sets the @f'th fragment of @skb to contain @page. - */ -static inline void skb_frag_set_page(struct sk_buff *skb, int f, - struct page *page) -{ - __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page); -} - bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); /** -- cgit v1.2.3 From 278fda0d52f67244044384abd7dd5b3a5b3a5604 Mon Sep 17 00:00:00 2001 From: Yunsheng Lin Date: Thu, 11 May 2023 09:12:13 +0800 Subject: net: remove __skb_frag_set_page() The remaining users calling __skb_frag_set_page() with page being NULL seems to be doing defensive programming, as shinfo->nr_frags is already decremented, so remove them. Signed-off-by: Yunsheng Lin Reviewed-by: Leon Romanovsky Reviewed-by: Michael Chan Reviewed-by: Jesse Brandeburg Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/skbuff.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 30be21c7d05f..00e8c435fa1a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3491,18 +3491,6 @@ static inline void skb_frag_page_copy(skb_frag_t *fragto, fragto->bv_page = fragfrom->bv_page; } -/** - * __skb_frag_set_page - sets the page contained in a paged fragment - * @frag: the paged fragment - * @page: the page to set - * - * Sets the fragment @frag to contain @page. - */ -static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page) -{ - frag->bv_page = page; -} - bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio); /** -- cgit v1.2.3 From a0b7955310a445fc0d45a0ac576bad8720cd6057 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 12 May 2023 17:58:37 +0100 Subject: net: phylink: constify fwnode arguments Both phylink_create() and phylink_fwnode_phy_connect() do not modify the fwnode argument that they are passed, so lets constify these. Reviewed-by: Simon Horman Signed-off-by: Russell King (Oracle) Signed-off-by: David S. Miller --- include/linux/phylink.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 71755c66c162..bb782f05ad08 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -568,16 +568,17 @@ void phylink_generic_validate(struct phylink_config *config, unsigned long *supported, struct phylink_link_state *state); -struct phylink *phylink_create(struct phylink_config *, struct fwnode_handle *, - phy_interface_t iface, - const struct phylink_mac_ops *mac_ops); +struct phylink *phylink_create(struct phylink_config *, + const struct fwnode_handle *, + phy_interface_t, + const struct phylink_mac_ops *); void phylink_destroy(struct phylink *); bool phylink_expects_phy(struct phylink *pl); int phylink_connect_phy(struct phylink *, struct phy_device *); int phylink_of_phy_connect(struct phylink *, struct device_node *, u32 flags); int phylink_fwnode_phy_connect(struct phylink *pl, - struct fwnode_handle *fwnode, + const struct fwnode_handle *fwnode, u32 flags); void phylink_disconnect_phy(struct phylink *); -- cgit v1.2.3 From 38f1755a3e59a3f88e33030f8e4ee0421de2f05a Mon Sep 17 00:00:00 2001 From: Min-Hua Chen Date: Fri, 12 May 2023 00:46:25 +0800 Subject: fs: use correct __poll_t type Fix the following sparse warnings by using __poll_t instead of unsigned type. fs/eventpoll.c:541:9: sparse: warning: restricted __poll_t degrades to integer fs/eventfd.c:67:17: sparse: warning: restricted __poll_t degrades to integer Signed-off-by: Min-Hua Chen Message-Id: <20230511164628.336586-1-minhuadotchen@gmail.com> Signed-off-by: Christian Brauner --- include/linux/eventfd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/eventfd.h b/include/linux/eventfd.h index 36a486505b08..98d31cdaca40 100644 --- a/include/linux/eventfd.h +++ b/include/linux/eventfd.h @@ -40,7 +40,7 @@ struct file *eventfd_fget(int fd); struct eventfd_ctx *eventfd_ctx_fdget(int fd); struct eventfd_ctx *eventfd_ctx_fileget(struct file *file); __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n); -__u64 eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, unsigned mask); +__u64 eventfd_signal_mask(struct eventfd_ctx *ctx, __u64 n, __poll_t mask); int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_entry_t *wait, __u64 *cnt); void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt); -- cgit v1.2.3 From 6f0621238b7e7680d5e26c00aa4cd473314d05b2 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 May 2023 23:07:27 +0200 Subject: cpu/hotplug: Add CPU state tracking and synchronization The CPU state tracking and synchronization mechanism in smpboot.c is completely independent of the hotplug code and all logic around it is implemented in architecture specific code. Except for the state reporting of the AP there is absolutely nothing architecture specific and the sychronization and decision functions can be moved into the generic hotplug core code. Provide an integrated variant and add the core synchronization and decision points. This comes in two flavours: 1) DEAD state synchronization Updated by the architecture code once the AP reaches the point where it is ready to be torn down by the control CPU, e.g. by removing power or clocks or tear down via the hypervisor. The control CPU waits for this state to be reached with a timeout. If the state is reached an architecture specific cleanup function is invoked. 2) Full state synchronization This extends #1 with AP alive synchronization. This is new functionality, which allows to replace architecture specific wait mechanims, e.g. cpumasks, completely. It also prevents that an AP which is in a limbo state can be brought up again. This can happen when an AP failed to report dead state during a previous off-line operation. The dead synchronization is what most architectures use. Only x86 makes a bringup decision based on that state at the moment. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Oleksandr Natalenko Tested-by: Helge Deller # parisc Tested-by: Guilherme G. Piccoli # Steam Deck Link: https://lore.kernel.org/r/20230512205256.476305035@linutronix.de --- include/linux/cpuhotplug.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 0f1001dca0e0..5def71f81ec5 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -517,4 +517,16 @@ void cpuhp_online_idle(enum cpuhp_state state); static inline void cpuhp_online_idle(enum cpuhp_state state) { } #endif +void cpuhp_ap_sync_alive(void); +void arch_cpuhp_sync_state_poll(void); +void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu); + +#ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD +void cpuhp_ap_report_dead(void); +void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu); +#else +static inline void cpuhp_ap_report_dead(void) { } +static inline void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { } +#endif + #endif -- cgit v1.2.3 From 5356297d12d9ee6f70d09485878904bc41bac422 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 May 2023 23:07:30 +0200 Subject: cpu/hotplug: Remove cpu_report_state() and related unused cruft No more users. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Oleksandr Natalenko Tested-by: Helge Deller # parisc Tested-by: Guilherme G. Piccoli # Steam Deck Link: https://lore.kernel.org/r/20230512205256.582584351@linutronix.de --- include/linux/cpu.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 8582a7142623..68f69e8e4f19 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -184,8 +184,6 @@ void arch_cpu_idle_enter(void); void arch_cpu_idle_exit(void); void __noreturn arch_cpu_idle_dead(void); -int cpu_report_state(int cpu); -int cpu_check_up_prepare(int cpu); void cpu_set_state_online(int cpu); void play_idle_precise(u64 duration_ns, u64 latency_ns); -- cgit v1.2.3 From bc088f9a0d5bdf12bb18980739336dfcc092e55b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 May 2023 23:07:41 +0200 Subject: cpu/hotplug: Remove unused state functions All users converted to the hotplug core mechanism. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Oleksandr Natalenko Tested-by: Helge Deller # parisc Tested-by: Guilherme G. Piccoli # Steam Deck Link: https://lore.kernel.org/r/20230512205256.972894276@linutronix.de --- include/linux/cpu.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 68f69e8e4f19..d321dbd53405 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -193,8 +193,6 @@ static inline void play_idle(unsigned long duration_us) } #ifdef CONFIG_HOTPLUG_CPU -bool cpu_wait_death(unsigned int cpu, int seconds); -bool cpu_report_death(void); void cpuhp_report_idle_dead(void); #else static inline void cpuhp_report_idle_dead(void) { } -- cgit v1.2.3 From a631be92b996c5db9b368e8b96305d22fb8c4180 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 May 2023 23:07:45 +0200 Subject: cpu/hotplug: Provide a split up CPUHP_BRINGUP mechanism The bring up logic of a to be onlined CPU consists of several parts, which are considered to be a single hotplug state: 1) Control CPU issues the wake-up 2) To be onlined CPU starts up, does the minimal initialization, reports to be alive and waits for release into the complete bring-up. 3) Control CPU waits for the alive report and releases the upcoming CPU for the complete bring-up. Allow to split this into two states: 1) Control CPU issues the wake-up After that the to be onlined CPU starts up, does the minimal initialization, reports to be alive and waits for release into the full bring-up. As this can run after the control CPU dropped the hotplug locks the code which is executed on the AP before it reports alive has to be carefully audited to not violate any of the hotplug constraints, especially not modifying any of the various cpumasks. This is really only meant to avoid waiting for the AP to react on the wake-up. Of course an architecture can move strict CPU related setup functionality, e.g. microcode loading, with care before the synchronization point to save further pointless waiting time. 2) Control CPU waits for the alive report and releases the upcoming CPU for the complete bring-up. This allows that the two states can be split up to run all to be onlined CPUs up to state #1 on the control CPU and then at a later point run state #2. This spares some of the latencies of the full serialized per CPU bringup by avoiding the per CPU wakeup/wait serialization. The assumption is that the first AP already waits when the last AP has been woken up. This obvioulsy depends on the hardware latencies and depending on the timings this might still not completely eliminate all wait scenarios. This split is just a preparatory step for enabling the parallel bringup later. The boot time bringup is still fully serialized. It has a separate config switch so that architectures which want to support parallel bringup can test the split of the CPUHP_BRINGUG step separately. To enable this the architecture must support the CPU hotplug core sync mechanism and has to be audited that there are no implicit hotplug state dependencies which require a fully serialized bringup. Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Oleksandr Natalenko Tested-by: Helge Deller # parisc Tested-by: Guilherme G. Piccoli # Steam Deck Link: https://lore.kernel.org/r/20230512205257.080801387@linutronix.de --- include/linux/cpuhotplug.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 5def71f81ec5..bc2d0a1d7608 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -133,6 +133,7 @@ enum cpuhp_state { CPUHP_MIPS_SOC_PREPARE, CPUHP_BP_PREPARE_DYN, CPUHP_BP_PREPARE_DYN_END = CPUHP_BP_PREPARE_DYN + 20, + CPUHP_BP_KICK_AP, CPUHP_BRINGUP_CPU, /* @@ -517,9 +518,12 @@ void cpuhp_online_idle(enum cpuhp_state state); static inline void cpuhp_online_idle(enum cpuhp_state state) { } #endif +struct task_struct; + void cpuhp_ap_sync_alive(void); void arch_cpuhp_sync_state_poll(void); void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu); +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle); #ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD void cpuhp_ap_report_dead(void); -- cgit v1.2.3 From 18415f33e2ac4ab382cbca8b5ff82a9036b5bd49 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 12 May 2023 23:07:50 +0200 Subject: cpu/hotplug: Allow "parallel" bringup up to CPUHP_BP_KICK_AP_STATE There is often significant latency in the early stages of CPU bringup, and time is wasted by waking each CPU (e.g. with SIPI/INIT/INIT on x86) and then waiting for it to respond before moving on to the next. Allow a platform to enable parallel setup which brings all to be onlined CPUs up to the CPUHP_BP_KICK_AP state. While this state advancement on the control CPU (BP) is single-threaded the important part is the last state CPUHP_BP_KICK_AP which wakes the to be onlined CPUs up. This allows the CPUs to run up to the first sychronization point cpuhp_ap_sync_alive() where they wait for the control CPU to release them one by one for the full onlining procedure. This parallelism depends on the CPU hotplug core sync mechanism which ensures that the parallel brought up CPUs wait for release before touching any state which would make the CPU visible to anything outside the hotplug control mechanism. To handle the SMT constraints of X86 correctly the bringup happens in two iterations when CONFIG_HOTPLUG_SMT is enabled. The control CPU brings up the primary SMT threads of each core first, which can load the microcode without the need to rendevouz with the thread siblings. Once that's completed it brings up the secondary SMT threads. Co-developed-by: David Woodhouse Signed-off-by: David Woodhouse Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley Tested-by: Oleksandr Natalenko Tested-by: Helge Deller # parisc Tested-by: Guilherme G. Piccoli # Steam Deck Link: https://lore.kernel.org/r/20230512205257.240231377@linutronix.de --- include/linux/cpuhotplug.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index bc2d0a1d7608..a5e414cd82be 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -524,6 +524,7 @@ void cpuhp_ap_sync_alive(void); void arch_cpuhp_sync_state_poll(void); void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu); int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle); +bool arch_cpuhp_init_parallel_bringup(void); #ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD void cpuhp_ap_report_dead(void); -- cgit v1.2.3 From 049449976f549605a6913d468b61356a9950a6a2 Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Thu, 4 May 2023 19:36:08 +0200 Subject: mfd: rk808: Replace 'struct i2c_client' with 'struct device' Put 'struct device' pointer into the MFD platform_data instead of the 'struct i2c_client' pointer. This simplifies the code and prepares the MFD for SPI support. Tested-by: Diederik de Haas # Rock64, Quartz64 Model A + B Tested-by: Vincent Legoll # Pine64 QuartzPro64 Signed-off-by: Sebastian Reichel Link: https://lore.kernel.org/r/20230504173618.142075-5-sebastian.reichel@collabora.com Signed-off-by: Lee Jones --- include/linux/mfd/rk808.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index 9af1f3105f80..a89ddd9ba68e 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -787,7 +787,7 @@ enum { }; struct rk808 { - struct i2c_client *i2c; + struct device *dev; struct regmap_irq_chip_data *irq_data; struct regmap *regmap; long variant; -- cgit v1.2.3 From c20e8c5b1203af3726561ee5649b147194e0618e Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Thu, 4 May 2023 19:36:09 +0200 Subject: mfd: rk808: Split into core and i2c Split rk808 into a core and an i2c part in preparation for SPI support. Acked-by: Alexandre Belloni # for RTC Tested-by: Diederik de Haas # Rock64, Quartz64 Model A + B Tested-by: Vincent Legoll # Pine64 QuartzPro64 Signed-off-by: Sebastian Reichel Link: https://lore.kernel.org/r/20230504173618.142075-6-sebastian.reichel@collabora.com Signed-off-by: Lee Jones --- include/linux/mfd/rk808.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index a89ddd9ba68e..4183427a80fe 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -794,4 +794,10 @@ struct rk808 { const struct regmap_config *regmap_cfg; const struct regmap_irq_chip *regmap_irq_chip; }; + +void rk8xx_shutdown(struct device *dev); +int rk8xx_probe(struct device *dev, int variant, unsigned int irq, struct regmap *regmap); +int rk8xx_suspend(struct device *dev); +int rk8xx_resume(struct device *dev); + #endif /* __LINUX_REGULATOR_RK808_H */ -- cgit v1.2.3 From 210f418f8ace9f056c337f7945e0ae3e242b3389 Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Thu, 4 May 2023 19:36:12 +0200 Subject: mfd: rk8xx: Add rk806 support Add support for SPI connected rk806, which is used by the RK3588 evaluation boards. The PMIC is advertised to support I2C and SPI, but the evaluation boards all use SPI. Thus only SPI support is added here. Tested-by: Diederik de Haas # Rock64, Quartz64 Model A + B Tested-by: Vincent Legoll # Pine64 QuartzPro64 Signed-off-by: Sebastian Reichel Link: https://lore.kernel.org/r/20230504173618.142075-9-sebastian.reichel@collabora.com Signed-off-by: Lee Jones --- include/linux/mfd/rk808.h | 409 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 409 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index 4183427a80fe..78e167a92483 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -289,6 +289,414 @@ enum rk805_reg { #define RK805_INT_ALARM_EN (1 << 3) #define RK805_INT_TIMER_EN (1 << 2) +/* RK806 */ +#define RK806_POWER_EN0 0x0 +#define RK806_POWER_EN1 0x1 +#define RK806_POWER_EN2 0x2 +#define RK806_POWER_EN3 0x3 +#define RK806_POWER_EN4 0x4 +#define RK806_POWER_EN5 0x5 +#define RK806_POWER_SLP_EN0 0x6 +#define RK806_POWER_SLP_EN1 0x7 +#define RK806_POWER_SLP_EN2 0x8 +#define RK806_POWER_DISCHRG_EN0 0x9 +#define RK806_POWER_DISCHRG_EN1 0xA +#define RK806_POWER_DISCHRG_EN2 0xB +#define RK806_BUCK_FB_CONFIG 0xC +#define RK806_SLP_LP_CONFIG 0xD +#define RK806_POWER_FPWM_EN0 0xE +#define RK806_POWER_FPWM_EN1 0xF +#define RK806_BUCK1_CONFIG 0x10 +#define RK806_BUCK2_CONFIG 0x11 +#define RK806_BUCK3_CONFIG 0x12 +#define RK806_BUCK4_CONFIG 0x13 +#define RK806_BUCK5_CONFIG 0x14 +#define RK806_BUCK6_CONFIG 0x15 +#define RK806_BUCK7_CONFIG 0x16 +#define RK806_BUCK8_CONFIG 0x17 +#define RK806_BUCK9_CONFIG 0x18 +#define RK806_BUCK10_CONFIG 0x19 +#define RK806_BUCK1_ON_VSEL 0x1A +#define RK806_BUCK2_ON_VSEL 0x1B +#define RK806_BUCK3_ON_VSEL 0x1C +#define RK806_BUCK4_ON_VSEL 0x1D +#define RK806_BUCK5_ON_VSEL 0x1E +#define RK806_BUCK6_ON_VSEL 0x1F +#define RK806_BUCK7_ON_VSEL 0x20 +#define RK806_BUCK8_ON_VSEL 0x21 +#define RK806_BUCK9_ON_VSEL 0x22 +#define RK806_BUCK10_ON_VSEL 0x23 +#define RK806_BUCK1_SLP_VSEL 0x24 +#define RK806_BUCK2_SLP_VSEL 0x25 +#define RK806_BUCK3_SLP_VSEL 0x26 +#define RK806_BUCK4_SLP_VSEL 0x27 +#define RK806_BUCK5_SLP_VSEL 0x28 +#define RK806_BUCK6_SLP_VSEL 0x29 +#define RK806_BUCK7_SLP_VSEL 0x2A +#define RK806_BUCK8_SLP_VSEL 0x2B +#define RK806_BUCK9_SLP_VSEL 0x2D +#define RK806_BUCK10_SLP_VSEL 0x2E +#define RK806_BUCK_DEBUG1 0x30 +#define RK806_BUCK_DEBUG2 0x31 +#define RK806_BUCK_DEBUG3 0x32 +#define RK806_BUCK_DEBUG4 0x33 +#define RK806_BUCK_DEBUG5 0x34 +#define RK806_BUCK_DEBUG6 0x35 +#define RK806_BUCK_DEBUG7 0x36 +#define RK806_BUCK_DEBUG8 0x37 +#define RK806_BUCK_DEBUG9 0x38 +#define RK806_BUCK_DEBUG10 0x39 +#define RK806_BUCK_DEBUG11 0x3A +#define RK806_BUCK_DEBUG12 0x3B +#define RK806_BUCK_DEBUG13 0x3C +#define RK806_BUCK_DEBUG14 0x3D +#define RK806_BUCK_DEBUG15 0x3E +#define RK806_BUCK_DEBUG16 0x3F +#define RK806_BUCK_DEBUG17 0x40 +#define RK806_BUCK_DEBUG18 0x41 +#define RK806_NLDO_IMAX 0x42 +#define RK806_NLDO1_ON_VSEL 0x43 +#define RK806_NLDO2_ON_VSEL 0x44 +#define RK806_NLDO3_ON_VSEL 0x45 +#define RK806_NLDO4_ON_VSEL 0x46 +#define RK806_NLDO5_ON_VSEL 0x47 +#define RK806_NLDO1_SLP_VSEL 0x48 +#define RK806_NLDO2_SLP_VSEL 0x49 +#define RK806_NLDO3_SLP_VSEL 0x4A +#define RK806_NLDO4_SLP_VSEL 0x4B +#define RK806_NLDO5_SLP_VSEL 0x4C +#define RK806_PLDO_IMAX 0x4D +#define RK806_PLDO1_ON_VSEL 0x4E +#define RK806_PLDO2_ON_VSEL 0x4F +#define RK806_PLDO3_ON_VSEL 0x50 +#define RK806_PLDO4_ON_VSEL 0x51 +#define RK806_PLDO5_ON_VSEL 0x52 +#define RK806_PLDO6_ON_VSEL 0x53 +#define RK806_PLDO1_SLP_VSEL 0x54 +#define RK806_PLDO2_SLP_VSEL 0x55 +#define RK806_PLDO3_SLP_VSEL 0x56 +#define RK806_PLDO4_SLP_VSEL 0x57 +#define RK806_PLDO5_SLP_VSEL 0x58 +#define RK806_PLDO6_SLP_VSEL 0x59 +#define RK806_CHIP_NAME 0x5A +#define RK806_CHIP_VER 0x5B +#define RK806_OTP_VER 0x5C +#define RK806_SYS_STS 0x5D +#define RK806_SYS_CFG0 0x5E +#define RK806_SYS_CFG1 0x5F +#define RK806_SYS_OPTION 0x61 +#define RK806_SLEEP_CONFIG0 0x62 +#define RK806_SLEEP_CONFIG1 0x63 +#define RK806_SLEEP_CTR_SEL0 0x64 +#define RK806_SLEEP_CTR_SEL1 0x65 +#define RK806_SLEEP_CTR_SEL2 0x66 +#define RK806_SLEEP_CTR_SEL3 0x67 +#define RK806_SLEEP_CTR_SEL4 0x68 +#define RK806_SLEEP_CTR_SEL5 0x69 +#define RK806_DVS_CTRL_SEL0 0x6A +#define RK806_DVS_CTRL_SEL1 0x6B +#define RK806_DVS_CTRL_SEL2 0x6C +#define RK806_DVS_CTRL_SEL3 0x6D +#define RK806_DVS_CTRL_SEL4 0x6E +#define RK806_DVS_CTRL_SEL5 0x6F +#define RK806_DVS_START_CTRL 0x70 +#define RK806_SLEEP_GPIO 0x71 +#define RK806_SYS_CFG3 0x72 +#define RK806_ON_SOURCE 0x74 +#define RK806_OFF_SOURCE 0x75 +#define RK806_PWRON_KEY 0x76 +#define RK806_INT_STS0 0x77 +#define RK806_INT_MSK0 0x78 +#define RK806_INT_STS1 0x79 +#define RK806_INT_MSK1 0x7A +#define RK806_GPIO_INT_CONFIG 0x7B +#define RK806_DATA_REG0 0x7C +#define RK806_DATA_REG1 0x7D +#define RK806_DATA_REG2 0x7E +#define RK806_DATA_REG3 0x7F +#define RK806_DATA_REG4 0x80 +#define RK806_DATA_REG5 0x81 +#define RK806_DATA_REG6 0x82 +#define RK806_DATA_REG7 0x83 +#define RK806_DATA_REG8 0x84 +#define RK806_DATA_REG9 0x85 +#define RK806_DATA_REG10 0x86 +#define RK806_DATA_REG11 0x87 +#define RK806_DATA_REG12 0x88 +#define RK806_DATA_REG13 0x89 +#define RK806_DATA_REG14 0x8A +#define RK806_DATA_REG15 0x8B +#define RK806_TM_REG 0x8C +#define RK806_OTP_EN_REG 0x8D +#define RK806_FUNC_OTP_EN_REG 0x8E +#define RK806_TEST_REG1 0x8F +#define RK806_TEST_REG2 0x90 +#define RK806_TEST_REG3 0x91 +#define RK806_TEST_REG4 0x92 +#define RK806_TEST_REG5 0x93 +#define RK806_BUCK_VSEL_OTP_REG0 0x94 +#define RK806_BUCK_VSEL_OTP_REG1 0x95 +#define RK806_BUCK_VSEL_OTP_REG2 0x96 +#define RK806_BUCK_VSEL_OTP_REG3 0x97 +#define RK806_BUCK_VSEL_OTP_REG4 0x98 +#define RK806_BUCK_VSEL_OTP_REG5 0x99 +#define RK806_BUCK_VSEL_OTP_REG6 0x9A +#define RK806_BUCK_VSEL_OTP_REG7 0x9B +#define RK806_BUCK_VSEL_OTP_REG8 0x9C +#define RK806_BUCK_VSEL_OTP_REG9 0x9D +#define RK806_NLDO1_VSEL_OTP_REG0 0x9E +#define RK806_NLDO1_VSEL_OTP_REG1 0x9F +#define RK806_NLDO1_VSEL_OTP_REG2 0xA0 +#define RK806_NLDO1_VSEL_OTP_REG3 0xA1 +#define RK806_NLDO1_VSEL_OTP_REG4 0xA2 +#define RK806_PLDO_VSEL_OTP_REG0 0xA3 +#define RK806_PLDO_VSEL_OTP_REG1 0xA4 +#define RK806_PLDO_VSEL_OTP_REG2 0xA5 +#define RK806_PLDO_VSEL_OTP_REG3 0xA6 +#define RK806_PLDO_VSEL_OTP_REG4 0xA7 +#define RK806_PLDO_VSEL_OTP_REG5 0xA8 +#define RK806_BUCK_EN_OTP_REG1 0xA9 +#define RK806_NLDO_EN_OTP_REG1 0xAA +#define RK806_PLDO_EN_OTP_REG1 0xAB +#define RK806_BUCK_FB_RES_OTP_REG1 0xAC +#define RK806_OTP_RESEV_REG0 0xAD +#define RK806_OTP_RESEV_REG1 0xAE +#define RK806_OTP_RESEV_REG2 0xAF +#define RK806_OTP_RESEV_REG3 0xB0 +#define RK806_OTP_RESEV_REG4 0xB1 +#define RK806_BUCK_SEQ_REG0 0xB2 +#define RK806_BUCK_SEQ_REG1 0xB3 +#define RK806_BUCK_SEQ_REG2 0xB4 +#define RK806_BUCK_SEQ_REG3 0xB5 +#define RK806_BUCK_SEQ_REG4 0xB6 +#define RK806_BUCK_SEQ_REG5 0xB7 +#define RK806_BUCK_SEQ_REG6 0xB8 +#define RK806_BUCK_SEQ_REG7 0xB9 +#define RK806_BUCK_SEQ_REG8 0xBA +#define RK806_BUCK_SEQ_REG9 0xBB +#define RK806_BUCK_SEQ_REG10 0xBC +#define RK806_BUCK_SEQ_REG11 0xBD +#define RK806_BUCK_SEQ_REG12 0xBE +#define RK806_BUCK_SEQ_REG13 0xBF +#define RK806_BUCK_SEQ_REG14 0xC0 +#define RK806_BUCK_SEQ_REG15 0xC1 +#define RK806_BUCK_SEQ_REG16 0xC2 +#define RK806_BUCK_SEQ_REG17 0xC3 +#define RK806_HK_TRIM_REG1 0xC4 +#define RK806_HK_TRIM_REG2 0xC5 +#define RK806_BUCK_REF_TRIM_REG1 0xC6 +#define RK806_BUCK_REF_TRIM_REG2 0xC7 +#define RK806_BUCK_REF_TRIM_REG3 0xC8 +#define RK806_BUCK_REF_TRIM_REG4 0xC9 +#define RK806_BUCK_REF_TRIM_REG5 0xCA +#define RK806_BUCK_OSC_TRIM_REG1 0xCB +#define RK806_BUCK_OSC_TRIM_REG2 0xCC +#define RK806_BUCK_OSC_TRIM_REG3 0xCD +#define RK806_BUCK_OSC_TRIM_REG4 0xCE +#define RK806_BUCK_OSC_TRIM_REG5 0xCF +#define RK806_BUCK_TRIM_ZCDIOS_REG1 0xD0 +#define RK806_BUCK_TRIM_ZCDIOS_REG2 0xD1 +#define RK806_NLDO_TRIM_REG1 0xD2 +#define RK806_NLDO_TRIM_REG2 0xD3 +#define RK806_NLDO_TRIM_REG3 0xD4 +#define RK806_PLDO_TRIM_REG1 0xD5 +#define RK806_PLDO_TRIM_REG2 0xD6 +#define RK806_PLDO_TRIM_REG3 0xD7 +#define RK806_TRIM_ICOMP_REG1 0xD8 +#define RK806_TRIM_ICOMP_REG2 0xD9 +#define RK806_EFUSE_CONTROL_REGH 0xDA +#define RK806_FUSE_PROG_REG 0xDB +#define RK806_MAIN_FSM_STS_REG 0xDD +#define RK806_FSM_REG 0xDE +#define RK806_TOP_RESEV_OFFR 0xEC +#define RK806_TOP_RESEV_POR 0xED +#define RK806_BUCK_VRSN_REG1 0xEE +#define RK806_BUCK_VRSN_REG2 0xEF +#define RK806_NLDO_RLOAD_SEL_REG1 0xF0 +#define RK806_PLDO_RLOAD_SEL_REG1 0xF1 +#define RK806_PLDO_RLOAD_SEL_REG2 0xF2 +#define RK806_BUCK_CMIN_MX_REG1 0xF3 +#define RK806_BUCK_CMIN_MX_REG2 0xF4 +#define RK806_BUCK_FREQ_SET_REG1 0xF5 +#define RK806_BUCK_FREQ_SET_REG2 0xF6 +#define RK806_BUCK_RS_MEABS_REG1 0xF7 +#define RK806_BUCK_RS_MEABS_REG2 0xF8 +#define RK806_BUCK_RS_ZDLEB_REG1 0xF9 +#define RK806_BUCK_RS_ZDLEB_REG2 0xFA +#define RK806_BUCK_RSERVE_REG1 0xFB +#define RK806_BUCK_RSERVE_REG2 0xFC +#define RK806_BUCK_RSERVE_REG3 0xFD +#define RK806_BUCK_RSERVE_REG4 0xFE +#define RK806_BUCK_RSERVE_REG5 0xFF + +/* INT_STS Register field definitions */ +#define RK806_INT_STS_PWRON_FALL BIT(0) +#define RK806_INT_STS_PWRON_RISE BIT(1) +#define RK806_INT_STS_PWRON BIT(2) +#define RK806_INT_STS_PWRON_LP BIT(3) +#define RK806_INT_STS_HOTDIE BIT(4) +#define RK806_INT_STS_VDC_RISE BIT(5) +#define RK806_INT_STS_VDC_FALL BIT(6) +#define RK806_INT_STS_VB_LO BIT(7) +#define RK806_INT_STS_REV0 BIT(0) +#define RK806_INT_STS_REV1 BIT(1) +#define RK806_INT_STS_REV2 BIT(2) +#define RK806_INT_STS_CRC_ERROR BIT(3) +#define RK806_INT_STS_SLP3_GPIO BIT(4) +#define RK806_INT_STS_SLP2_GPIO BIT(5) +#define RK806_INT_STS_SLP1_GPIO BIT(6) +#define RK806_INT_STS_WDT BIT(7) + +/* SPI command */ +#define RK806_CMD_READ 0 +#define RK806_CMD_WRITE BIT(7) +#define RK806_CMD_CRC_EN BIT(6) +#define RK806_CMD_CRC_DIS 0 +#define RK806_CMD_LEN_MSK 0x0f +#define RK806_REG_H 0x00 + +#define VERSION_AB 0x01 + +enum rk806_reg_id { + RK806_ID_DCDC1 = 0, + RK806_ID_DCDC2, + RK806_ID_DCDC3, + RK806_ID_DCDC4, + RK806_ID_DCDC5, + RK806_ID_DCDC6, + RK806_ID_DCDC7, + RK806_ID_DCDC8, + RK806_ID_DCDC9, + RK806_ID_DCDC10, + + RK806_ID_NLDO1, + RK806_ID_NLDO2, + RK806_ID_NLDO3, + RK806_ID_NLDO4, + RK806_ID_NLDO5, + + RK806_ID_PLDO1, + RK806_ID_PLDO2, + RK806_ID_PLDO3, + RK806_ID_PLDO4, + RK806_ID_PLDO5, + RK806_ID_PLDO6, + RK806_ID_END, +}; + +/* Define the RK806 IRQ numbers */ +enum rk806_irqs { + /* INT_STS0 registers */ + RK806_IRQ_PWRON_FALL, + RK806_IRQ_PWRON_RISE, + RK806_IRQ_PWRON, + RK806_IRQ_PWRON_LP, + RK806_IRQ_HOTDIE, + RK806_IRQ_VDC_RISE, + RK806_IRQ_VDC_FALL, + RK806_IRQ_VB_LO, + + /* INT_STS0 registers */ + RK806_IRQ_REV0, + RK806_IRQ_REV1, + RK806_IRQ_REV2, + RK806_IRQ_CRC_ERROR, + RK806_IRQ_SLP3_GPIO, + RK806_IRQ_SLP2_GPIO, + RK806_IRQ_SLP1_GPIO, + RK806_IRQ_WDT, +}; + +/* VCC1 Low Voltage Threshold */ +enum rk806_lv_sel { + VB_LO_SEL_2800, + VB_LO_SEL_2900, + VB_LO_SEL_3000, + VB_LO_SEL_3100, + VB_LO_SEL_3200, + VB_LO_SEL_3300, + VB_LO_SEL_3400, + VB_LO_SEL_3500, +}; + +/* System Shutdown Voltage Select */ +enum rk806_uv_sel { + VB_UV_SEL_2700, + VB_UV_SEL_2800, + VB_UV_SEL_2900, + VB_UV_SEL_3000, + VB_UV_SEL_3100, + VB_UV_SEL_3200, + VB_UV_SEL_3300, + VB_UV_SEL_3400, +}; + +/* Pin Function */ +enum rk806_pwrctrl_fun { + PWRCTRL_NULL_FUN, + PWRCTRL_SLP_FUN, + PWRCTRL_POWOFF_FUN, + PWRCTRL_RST_FUN, + PWRCTRL_DVS_FUN, + PWRCTRL_GPIO_FUN, +}; + +/* Pin Polarity */ +enum rk806_pin_level { + POL_LOW, + POL_HIGH, +}; + +enum rk806_vsel_ctr_sel { + CTR_BY_NO_EFFECT, + CTR_BY_PWRCTRL1, + CTR_BY_PWRCTRL2, + CTR_BY_PWRCTRL3, +}; + +enum rk806_dvs_ctr_sel { + CTR_SEL_NO_EFFECT, + CTR_SEL_DVS_START1, + CTR_SEL_DVS_START2, + CTR_SEL_DVS_START3, +}; + +enum rk806_pin_dr_sel { + RK806_PIN_INPUT, + RK806_PIN_OUTPUT, +}; + +#define RK806_INT_POL_MSK BIT(1) +#define RK806_INT_POL_H BIT(1) +#define RK806_INT_POL_L 0 + +#define RK806_SLAVE_RESTART_FUN_MSK BIT(1) +#define RK806_SLAVE_RESTART_FUN_EN BIT(1) +#define RK806_SLAVE_RESTART_FUN_OFF 0 + +#define RK806_SYS_ENB2_2M_MSK BIT(1) +#define RK806_SYS_ENB2_2M_EN BIT(1) +#define RK806_SYS_ENB2_2M_OFF 0 + +enum rk806_int_fun { + RK806_INT_ONLY, + RK806_INT_ADN_WKUP, +}; + +enum rk806_dvs_mode { + RK806_DVS_NOT_SUPPORT, + RK806_DVS_START1, + RK806_DVS_START2, + RK806_DVS_START3, + RK806_DVS_PWRCTRL1, + RK806_DVS_PWRCTRL2, + RK806_DVS_PWRCTRL3, + RK806_DVS_START_PWRCTR1, + RK806_DVS_START_PWRCTR2, + RK806_DVS_START_PWRCTR3, + RK806_DVS_END, +}; + /* RK808 IRQ Definitions */ #define RK808_IRQ_VOUT_LO 0 #define RK808_IRQ_VB_LO 1 @@ -780,6 +1188,7 @@ enum { enum { RK805_ID = 0x8050, + RK806_ID = 0x8060, RK808_ID = 0x0000, RK809_ID = 0x8090, RK817_ID = 0x8170, -- cgit v1.2.3 From 47e79cbeea4b3891ad476047f4c68543eb51c8e0 Mon Sep 17 00:00:00 2001 From: Yafang Shao Date: Mon, 15 May 2023 13:08:48 +0000 Subject: bpf: Remove bpf trampoline selector After commit e21aa341785c ("bpf: Fix fexit trampoline."), the selector is only used to indicate how many times the bpf trampoline image are updated and been displayed in the trampoline ksym name. After the trampoline is freed, the selector will start from 0 again. So the selector is a useless value to the user. We can remove it. If the user want to check whether the bpf trampoline image has been updated or not, the user can compare the address. Each time the trampoline image is updated, the address will change consequently. Jiri also pointed out another issue that perf is still using the old name "bpf_trampoline_%lu", so this change can fix the issue in perf. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Signed-off-by: Yafang Shao Signed-off-by: Daniel Borkmann Acked-by: Song Liu Cc: Jiri Olsa Link: https://lore.kernel.org/bpf/ZFvOOlrmHiY9AgXE@krava Link: https://lore.kernel.org/bpf/20230515130849.57502-3-laoar.shao@gmail.com --- include/linux/bpf.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 456f33b9d205..36e4b2d8cca2 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1125,7 +1125,6 @@ struct bpf_trampoline { int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; - u64 selector; struct module *mod; }; -- cgit v1.2.3 From 6882011e8854c6cb227770fccb57ed70a88a716f Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Sun, 23 Apr 2023 13:06:39 +0200 Subject: can: length: make header self contained Include the headers that "can/length.h" depends on. Fixes: bdd2e413192d ("can: dev: move length related code into seperate file") Link: https://lore.kernel.org/all/20230509122854.350426-1-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- include/linux/can/length.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/can/length.h b/include/linux/can/length.h index 6995092b774e..69336549d24f 100644 --- a/include/linux/can/length.h +++ b/include/linux/can/length.h @@ -6,6 +6,9 @@ #ifndef _CAN_LENGTH_H #define _CAN_LENGTH_H +#include +#include + /* * Size of a Classical CAN Standard Frame * -- cgit v1.2.3 From 8b33485128ad932f807f4535e0b440733d8b5808 Mon Sep 17 00:00:00 2001 From: Yunsheng Lin Date: Mon, 15 May 2023 13:01:07 +0800 Subject: net: skbuff: update comment about pfmemalloc propagating __skb_fill_page_desc_noacc() is not doing any pfmemalloc propagating, and yet it has a comment about that, commit 84ce071e38a6 ("net: introduce __skb_fill_page_desc_noacc") may have accidentally moved it to __skb_fill_page_desc_noacc(), so move it back to __skb_fill_page_desc() which is supposed to be doing pfmemalloc propagating. Signed-off-by: Yunsheng Lin CC: Pavel Begunkov Reviewed-by: Pavel Begunkov Link: https://lore.kernel.org/r/20230515050107.46397-1-linyunsheng@huawei.com Signed-off-by: Paolo Abeni --- include/linux/skbuff.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 00e8c435fa1a..4b8d55247198 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2426,11 +2426,6 @@ static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo, { skb_frag_t *frag = &shinfo->frags[i]; - /* - * Propagate page pfmemalloc to the skb if we can. The problem is - * that not all callers have unique ownership of the page but rely - * on page_is_pfmemalloc doing the right thing(tm). - */ skb_frag_fill_page_desc(frag, page, off, size); } @@ -2463,6 +2458,11 @@ static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, int off, int size) { __skb_fill_page_desc_noacc(skb_shinfo(skb), i, page, off, size); + + /* Propagate page pfmemalloc to the skb if we can. The problem is + * that not all callers have unique ownership of the page but rely + * on page_is_pfmemalloc doing the right thing(tm). + */ page = compound_head(page); if (page_is_pfmemalloc(page)) skb->pfmemalloc = true; -- cgit v1.2.3 From 31b2ebc0929e964f4edfbfa7129d43f7e3c17165 Mon Sep 17 00:00:00 2001 From: "Ritesh Harjani (IBM)" Date: Fri, 21 Apr 2023 15:16:12 +0530 Subject: fs/buffer.c: Add generic_buffers_fsync*() implementation Some of the higher layers like iomap takes inode_lock() when calling generic_write_sync(). Also writeback already happens from other paths without inode lock, so it's difficult to say that we really need sync_mapping_buffers() to take any inode locking here. Having said that, let's add generic_buffers_fsync/_noflush() implementation in buffer.c with no inode_lock/unlock() for now so that filesystems like ext2 and ext4's nojournal mode can use it. Ext4 when got converted to iomap for direct-io already copied it's own variant of __generic_file_fsync() without lock. This patch adds generic_buffers_fsync() & generic_buffers_fsync_noflush() implementations for use in filesystems like ext2 & ext4 respectively. Later we can review other filesystems as well to see if we can make generic_buffers_fsync/_noflush() which does not take any inode_lock() as the default path. Tested-by: Disha Goel Reviewed-by: Christoph Hellwig Signed-off-by: Ritesh Harjani (IBM) Signed-off-by: Jan Kara Message-Id: --- include/linux/buffer_head.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 1520793c72da..1bd73cefd311 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -217,6 +217,10 @@ int inode_has_buffers(struct inode *); void invalidate_inode_buffers(struct inode *); int remove_inode_buffers(struct inode *inode); int sync_mapping_buffers(struct address_space *mapping); +int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end, + bool datasync); +int generic_buffers_fsync(struct file *file, loff_t start, loff_t end, + bool datasync); void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len); static inline void clean_bdev_bh_alias(struct buffer_head *bh) -- cgit v1.2.3 From 9a38cb27668e275ed912e67388cf11f454a24cc6 Mon Sep 17 00:00:00 2001 From: Sumit Gupta Date: Thu, 11 May 2023 23:02:04 +0530 Subject: memory: tegra: Add interconnect support for DRAM scaling in Tegra234 Add Interconnect framework support to dynamically set the DRAM bandwidth from different clients. Both the MC and EMC drivers are added as ICC providers. The path for any request is: MC-Client[1-n] -> MC -> EMC -> EMEM/DRAM MC client's request for bandwidth will go to the MC driver which passes the client request info like BPMP Client ID, Client type and the Bandwidth to the BPMP-FW. The final DRAM freq to achieve the requested bandwidth is set by the BPMP-FW based on the passed parameters. Signed-off-by: Sumit Gupta Acked-by: Krzysztof Kozlowski Signed-off-by: Thierry Reding --- include/linux/tegra-icc.h | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 include/linux/tegra-icc.h (limited to 'include/linux') diff --git a/include/linux/tegra-icc.h b/include/linux/tegra-icc.h new file mode 100644 index 000000000000..4b4d4bee290c --- /dev/null +++ b/include/linux/tegra-icc.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022-2023 NVIDIA CORPORATION. All rights reserved. + */ + +#ifndef LINUX_TEGRA_ICC_H +#define LINUX_TEGRA_ICC_H + +enum tegra_icc_client_type { + TEGRA_ICC_NONE, + TEGRA_ICC_NISO, + TEGRA_ICC_ISO_DISPLAY, + TEGRA_ICC_ISO_VI, + TEGRA_ICC_ISO_AUDIO, + TEGRA_ICC_ISO_VIFAL, +}; + +/* ICC ID's for MC client's used in BPMP */ +#define TEGRA_ICC_BPMP_DEBUG 1 +#define TEGRA_ICC_BPMP_CPU_CLUSTER0 2 +#define TEGRA_ICC_BPMP_CPU_CLUSTER1 3 +#define TEGRA_ICC_BPMP_CPU_CLUSTER2 4 +#define TEGRA_ICC_BPMP_GPU 5 +#define TEGRA_ICC_BPMP_CACTMON 6 +#define TEGRA_ICC_BPMP_DISPLAY 7 +#define TEGRA_ICC_BPMP_VI 8 +#define TEGRA_ICC_BPMP_EQOS 9 +#define TEGRA_ICC_BPMP_PCIE_0 10 +#define TEGRA_ICC_BPMP_PCIE_1 11 +#define TEGRA_ICC_BPMP_PCIE_2 12 +#define TEGRA_ICC_BPMP_PCIE_3 13 +#define TEGRA_ICC_BPMP_PCIE_4 14 +#define TEGRA_ICC_BPMP_PCIE_5 15 +#define TEGRA_ICC_BPMP_PCIE_6 16 +#define TEGRA_ICC_BPMP_PCIE_7 17 +#define TEGRA_ICC_BPMP_PCIE_8 18 +#define TEGRA_ICC_BPMP_PCIE_9 19 +#define TEGRA_ICC_BPMP_PCIE_10 20 +#define TEGRA_ICC_BPMP_DLA_0 21 +#define TEGRA_ICC_BPMP_DLA_1 22 +#define TEGRA_ICC_BPMP_SDMMC_1 23 +#define TEGRA_ICC_BPMP_SDMMC_2 24 +#define TEGRA_ICC_BPMP_SDMMC_3 25 +#define TEGRA_ICC_BPMP_SDMMC_4 26 +#define TEGRA_ICC_BPMP_NVDEC 27 +#define TEGRA_ICC_BPMP_NVENC 28 +#define TEGRA_ICC_BPMP_NVJPG_0 29 +#define TEGRA_ICC_BPMP_NVJPG_1 30 +#define TEGRA_ICC_BPMP_OFAA 31 +#define TEGRA_ICC_BPMP_XUSB_HOST 32 +#define TEGRA_ICC_BPMP_XUSB_DEV 33 +#define TEGRA_ICC_BPMP_TSEC 34 +#define TEGRA_ICC_BPMP_VIC 35 +#define TEGRA_ICC_BPMP_APE 36 +#define TEGRA_ICC_BPMP_APEDMA 37 +#define TEGRA_ICC_BPMP_SE 38 +#define TEGRA_ICC_BPMP_ISP 39 +#define TEGRA_ICC_BPMP_HDA 40 +#define TEGRA_ICC_BPMP_VIFAL 41 +#define TEGRA_ICC_BPMP_VI2FAL 42 +#define TEGRA_ICC_BPMP_VI2 43 +#define TEGRA_ICC_BPMP_RCE 44 +#define TEGRA_ICC_BPMP_PVA 45 + +#endif /* LINUX_TEGRA_ICC_H */ -- cgit v1.2.3 From 514ca14ed5444b911de59ed3381dfd195d99fe4b Mon Sep 17 00:00:00 2001 From: "ndesaulniers@google.com" Date: Mon, 17 Apr 2023 15:00:05 -0700 Subject: start_kernel: Add __no_stack_protector function attribute Back during the discussion of commit a9a3ed1eff36 ("x86: Fix early boot crash on gcc-10, third try") we discussed the need for a function attribute to control the omission of stack protectors on a per-function basis; at the time Clang had support for no_stack_protector but GCC did not. This was fixed in gcc-11. Now that the function attribute is available, let's start using it. Callers of boot_init_stack_canary need to use this function attribute unless they're compiled with -fno-stack-protector, otherwise the canary stored in the stack slot of the caller will differ upon the call to boot_init_stack_canary. This will lead to a call to __stack_chk_fail() then panic. Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94722 Link: https://lore.kernel.org/all/20200316130414.GC12561@hirez.programming.kicks-ass.net/ Tested-by: Nathan Chancellor Acked-by: Michael Ellerman (powerpc) Acked-by: Miguel Ojeda Acked-by: Peter Zijlstra (Intel) Signed-off-by: Nick Desaulniers Link: https://lore.kernel.org/r/20230412-no_stackp-v2-1-116f9fe4bbe7@google.com Signed-off-by: Josh Poimboeuf Signed-off-by: ndesaulniers@google.com --- include/linux/compiler_attributes.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index e659cb6fded3..84864767a56a 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -255,6 +255,18 @@ */ #define __noreturn __attribute__((__noreturn__)) +/* + * Optional: only supported since GCC >= 11.1, clang >= 7.0. + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-no_005fstack_005fprotector-function-attribute + * clang: https://clang.llvm.org/docs/AttributeReference.html#no-stack-protector-safebuffers + */ +#if __has_attribute(__no_stack_protector__) +# define __no_stack_protector __attribute__((__no_stack_protector__)) +#else +# define __no_stack_protector +#endif + /* * Optional: not supported by gcc. * -- cgit v1.2.3 From 03d89a2de25bbc5c77e61a0cf77663978c4b6ea7 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 5 Nov 2021 17:20:54 -0600 Subject: io_uring: support for user allocated memory for rings/sqes Currently io_uring applications must call mmap(2) twice to map the rings themselves, and the sqes array. This works fine, but it does not support using huge pages to back the rings/sqes. Provide a way for the application to pass in pre-allocated memory for the rings/sqes, which can then suitably be allocated from shmfs or via mmap to get huge page support. Particularly for larger rings, this reduces the TLBs needed. If an application wishes to take advantage of that, it must pre-allocate the memory needed for the sq/cq ring, and the sqes. The former must be passed in via the io_uring_params->cq_off.user_data field, while the latter is passed in via the io_uring_params->sq_off.user_data field. Then it must set IORING_SETUP_NO_MMAP in the io_uring_params->flags field, and io_uring will then map the existing memory into the kernel for shared use. The application must not call mmap(2) to map rings as it otherwise would have, that will now fail with -EINVAL if this setup flag was used. The pages used for the rings and sqes must be contigious. The intent here is clearly that huge pages should be used, otherwise the normal setup procedure works fine as-is. The application may use one huge page for both the rings and sqes. Outside of those initialization changes, everything works like it did before. Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 1b2a20a42413..f04ce513fadb 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -211,6 +211,16 @@ struct io_ring_ctx { unsigned int compat: 1; enum task_work_notify_mode notify_method; + + /* + * If IORING_SETUP_NO_MMAP is used, then the below holds + * the gup'ed pages for the two rings, and the sqes. + */ + unsigned short n_ring_pages; + unsigned short n_sqe_pages; + struct page **ring_pages; + struct page **sqe_pages; + struct io_rings *rings; struct task_struct *submitter_task; struct percpu_ref refs; -- cgit v1.2.3 From ead62aa370a81c4fb42a44c4edeafe13e0a3a703 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Sat, 13 May 2023 11:18:40 +0200 Subject: fortify: strscpy: Fix flipped q and p docstring typo Fix typo in the strscpy() docstring where q and p were flipped. Signed-off-by: Arne Welzel Signed-off-by: Kees Cook --- include/linux/fortify-string.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index c9de1f59ee80..e29df83bff8a 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -299,8 +299,8 @@ extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(strscpy); * @q: Where to copy the string from * @size: Size of destination buffer * - * Copy the source string @p, or as much of it as fits, into the destination - * @q buffer. The behavior is undefined if the string buffers overlap. The + * Copy the source string @q, or as much of it as fits, into the destination + * @p buffer. The behavior is undefined if the string buffers overlap. The * destination @p buffer is always NUL terminated, unless it's zero-sized. * * Preferred to strlcpy() since the API doesn't require reading memory -- cgit v1.2.3 From 21a2c74b0a2a784228c9e3af63cff96d0dea7b8a Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 7 Apr 2023 12:27:10 -0700 Subject: fortify: Use const variables for __member_size tracking The sizes reported by __member_size should never change in a given function. Mark them as such. Suggested-by: Miguel Ojeda Cc: linux-hardening@vger.kernel.org Signed-off-by: Kees Cook Reviewed-by: Nick Desaulniers Link: https://lore.kernel.org/r/20230407192717.636137-4-keescook@chromium.org --- include/linux/fortify-string.h | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index e29df83bff8a..2e7a47b62ba2 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -20,7 +20,7 @@ void __write_overflow_field(size_t avail, size_t wanted) __compiletime_warning(" ({ \ char *__p = (char *)(p); \ size_t __ret = SIZE_MAX; \ - size_t __p_size = __member_size(p); \ + const size_t __p_size = __member_size(p); \ if (__p_size != SIZE_MAX && \ __builtin_constant_p(*__p)) { \ size_t __p_len = __p_size - 1; \ @@ -142,7 +142,7 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) __FORTIFY_INLINE __diagnose_as(__builtin_strncpy, 1, 2, 3) char *strncpy(char * const POS p, const char *q, __kernel_size_t size) { - size_t p_size = __member_size(p); + const size_t p_size = __member_size(p); if (__compiletime_lessthan(p_size, size)) __write_overflow(); @@ -169,7 +169,7 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size) __FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2) char *strcat(char * const POS p, const char *q) { - size_t p_size = __member_size(p); + const size_t p_size = __member_size(p); if (p_size == SIZE_MAX) return __underlying_strcat(p, q); @@ -191,8 +191,8 @@ extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(st */ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size_t maxlen) { - size_t p_size = __member_size(p); - size_t p_len = __compiletime_strlen(p); + const size_t p_size = __member_size(p); + const size_t p_len = __compiletime_strlen(p); size_t ret; /* We can take compile-time actions when maxlen is const. */ @@ -233,8 +233,8 @@ __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size __FORTIFY_INLINE __diagnose_as(__builtin_strlen, 1) __kernel_size_t __fortify_strlen(const char * const POS p) { + const size_t p_size = __member_size(p); __kernel_size_t ret; - size_t p_size = __member_size(p); /* Give up if we don't know how large p is. */ if (p_size == SIZE_MAX) @@ -267,8 +267,8 @@ extern size_t __real_strlcpy(char *, const char *, size_t) __RENAME(strlcpy); */ __FORTIFY_INLINE size_t strlcpy(char * const POS p, const char * const POS q, size_t size) { - size_t p_size = __member_size(p); - size_t q_size = __member_size(q); + const size_t p_size = __member_size(p); + const size_t q_size = __member_size(q); size_t q_len; /* Full count of source string length. */ size_t len; /* Count of characters going into destination. */ @@ -318,10 +318,10 @@ extern ssize_t __real_strscpy(char *, const char *, size_t) __RENAME(strscpy); */ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, size_t size) { - size_t len; /* Use string size rather than possible enclosing struct size. */ - size_t p_size = __member_size(p); - size_t q_size = __member_size(q); + const size_t p_size = __member_size(p); + const size_t q_size = __member_size(q); + size_t len; /* If we cannot get size of p and q default to call strscpy. */ if (p_size == SIZE_MAX && q_size == SIZE_MAX) @@ -394,9 +394,9 @@ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, s __FORTIFY_INLINE __diagnose_as(__builtin_strncat, 1, 2, 3) char *strncat(char * const POS p, const char * const POS q, __kernel_size_t count) { + const size_t p_size = __member_size(p); + const size_t q_size = __member_size(q); size_t p_len, copy_len; - size_t p_size = __member_size(p); - size_t q_size = __member_size(q); if (p_size == SIZE_MAX && q_size == SIZE_MAX) return __underlying_strncat(p, q, count); @@ -639,7 +639,7 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, extern void *__real_memscan(void *, int, __kernel_size_t) __RENAME(memscan); __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) { - size_t p_size = __struct_size(p); + const size_t p_size = __struct_size(p); if (__compiletime_lessthan(p_size, size)) __read_overflow(); @@ -651,8 +651,8 @@ __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) __FORTIFY_INLINE __diagnose_as(__builtin_memcmp, 1, 2, 3) int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t size) { - size_t p_size = __struct_size(p); - size_t q_size = __struct_size(q); + const size_t p_size = __struct_size(p); + const size_t q_size = __struct_size(q); if (__builtin_constant_p(size)) { if (__compiletime_lessthan(p_size, size)) @@ -668,7 +668,7 @@ int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t __FORTIFY_INLINE __diagnose_as(__builtin_memchr, 1, 2, 3) void *memchr(const void * const POS0 p, int c, __kernel_size_t size) { - size_t p_size = __struct_size(p); + const size_t p_size = __struct_size(p); if (__compiletime_lessthan(p_size, size)) __read_overflow(); @@ -680,7 +680,7 @@ void *memchr(const void * const POS0 p, int c, __kernel_size_t size) void *__real_memchr_inv(const void *s, int c, size_t n) __RENAME(memchr_inv); __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) { - size_t p_size = __struct_size(p); + const size_t p_size = __struct_size(p); if (__compiletime_lessthan(p_size, size)) __read_overflow(); @@ -693,7 +693,7 @@ extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kme __realloc_size(2); __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp) { - size_t p_size = __struct_size(p); + const size_t p_size = __struct_size(p); if (__compiletime_lessthan(p_size, size)) __read_overflow(); @@ -720,8 +720,8 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp __FORTIFY_INLINE __diagnose_as(__builtin_strcpy, 1, 2) char *strcpy(char * const POS p, const char * const POS q) { - size_t p_size = __member_size(p); - size_t q_size = __member_size(q); + const size_t p_size = __member_size(p); + const size_t q_size = __member_size(q); size_t size; /* If neither buffer size is known, immediately give up. */ -- cgit v1.2.3 From 605395cd7ceded5842c8ba6763ea24feee690c87 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sun, 2 Apr 2023 23:00:05 -0700 Subject: fortify: Add protection for strlcat() The definition of strcat() was defined in terms of unfortified strlcat(), but that meant there was no bounds checking done on the internal strlen() calls, and the (bounded) copy would be performed before reporting a failure. Additionally, pathological cases (i.e. unterminated destination buffer) did not make calls to fortify_panic(), which will make future unit testing more difficult. Instead, explicitly define a fortified strlcat() wrapper for strcat() to use. Signed-off-by: Kees Cook --- include/linux/fortify-string.h | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 2e7a47b62ba2..756c89bb88e0 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -371,6 +371,70 @@ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, s return __real_strscpy(p, q, len); } +/* Defined after fortified strlen() to reuse it. */ +extern size_t __real_strlcat(char *p, const char *q, size_t avail) __RENAME(strlcat); +/** + * strlcat - Append a string to an existing string + * + * @p: pointer to %NUL-terminated string to append to + * @q: pointer to %NUL-terminated string to append from + * @avail: Maximum bytes available in @p + * + * Appends %NUL-terminated string @q after the %NUL-terminated + * string at @p, but will not write beyond @avail bytes total, + * potentially truncating the copy from @q. @p will stay + * %NUL-terminated only if a %NUL already existed within + * the @avail bytes of @p. If so, the resulting number of + * bytes copied from @q will be at most "@avail - strlen(@p) - 1". + * + * Do not use this function. While FORTIFY_SOURCE tries to avoid + * read and write overflows, this is only possible when the sizes + * of @p and @q are known to the compiler. Prefer building the + * string with formatting, via scnprintf(), seq_buf, or similar. + * + * Returns total bytes that _would_ have been contained by @p + * regardless of truncation, similar to snprintf(). If return + * value is >= @avail, the string has been truncated. + * + */ +__FORTIFY_INLINE +size_t strlcat(char * const POS p, const char * const POS q, size_t avail) +{ + const size_t p_size = __member_size(p); + const size_t q_size = __member_size(q); + size_t p_len, copy_len; + size_t actual, wanted; + + /* Give up immediately if both buffer sizes are unknown. */ + if (p_size == SIZE_MAX && q_size == SIZE_MAX) + return __real_strlcat(p, q, avail); + + p_len = strnlen(p, avail); + copy_len = strlen(q); + wanted = actual = p_len + copy_len; + + /* Cannot append any more: report truncation. */ + if (avail <= p_len) + return wanted; + + /* Give up if string is already overflowed. */ + if (p_size <= p_len) + fortify_panic(__func__); + + if (actual >= avail) { + copy_len = avail - p_len - 1; + actual = p_len + copy_len; + } + + /* Give up if copy will overflow. */ + if (p_size <= actual) + fortify_panic(__func__); + __underlying_memcpy(p + p_len, q, copy_len); + p[actual] = '\0'; + + return wanted; +} + /** * strncat - Append a string to an existing string * -- cgit v1.2.3 From 55c84a5cf2c72a821719823ef2ebef01b119025b Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 4 Apr 2023 14:24:27 -0700 Subject: fortify: strcat: Move definition to use fortified strlcat() Move the definition of fortified strcat() to after strlcat() to use it for bounds checking. Signed-off-by: Kees Cook --- include/linux/fortify-string.h | 53 +++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 756c89bb88e0..da51a83b2829 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -151,33 +151,6 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size) return __underlying_strncpy(p, q, size); } -/** - * strcat - Append a string to an existing string - * - * @p: pointer to NUL-terminated string to append to - * @q: pointer to NUL-terminated source string to append from - * - * Do not use this function. While FORTIFY_SOURCE tries to avoid - * read and write overflows, this is only possible when the - * destination buffer size is known to the compiler. Prefer - * building the string with formatting, via scnprintf() or similar. - * At the very least, use strncat(). - * - * Returns @p. - * - */ -__FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2) -char *strcat(char * const POS p, const char *q) -{ - const size_t p_size = __member_size(p); - - if (p_size == SIZE_MAX) - return __underlying_strcat(p, q); - if (strlcat(p, q, p_size) >= p_size) - fortify_panic(__func__); - return p; -} - extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(strnlen); /** * strnlen - Return bounded count of characters in a NUL-terminated string @@ -435,6 +408,32 @@ size_t strlcat(char * const POS p, const char * const POS q, size_t avail) return wanted; } +/* Defined after fortified strlcat() to reuse it. */ +/** + * strcat - Append a string to an existing string + * + * @p: pointer to NUL-terminated string to append to + * @q: pointer to NUL-terminated source string to append from + * + * Do not use this function. While FORTIFY_SOURCE tries to avoid + * read and write overflows, this is only possible when the + * destination buffer size is known to the compiler. Prefer + * building the string with formatting, via scnprintf() or similar. + * At the very least, use strncat(). + * + * Returns @p. + * + */ +__FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2) +char *strcat(char * const POS p, const char *q) +{ + const size_t p_size = __member_size(p); + + if (strlcat(p, q, p_size) >= p_size) + fortify_panic(__func__); + return p; +} + /** * strncat - Append a string to an existing string * -- cgit v1.2.3 From 247c8d2f9837a3e29e3b6b7a4aa9c36c37659dd4 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 21:56:12 +0200 Subject: fs: pipe: reveal missing function protoypes A couple of functions from fs/pipe.c are used both internally and for the watch queue code, but the declaration is only visible when the latter is enabled: fs/pipe.c:1254:5: error: no previous prototype for 'pipe_resize_ring' fs/pipe.c:758:15: error: no previous prototype for 'account_pipe_buffers' fs/pipe.c:764:6: error: no previous prototype for 'too_many_pipe_buffers_soft' fs/pipe.c:771:6: error: no previous prototype for 'too_many_pipe_buffers_hard' fs/pipe.c:777:6: error: no previous prototype for 'pipe_is_unprivileged_user' Make the visible unconditionally to avoid these warnings. Fixes: c73be61cede5 ("pipe: Add general notification queue support") Signed-off-by: Arnd Bergmann Message-Id: <20230516195629.551602-1-arnd@kernel.org> Signed-off-by: Christian Brauner --- include/linux/pipe_fs_i.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index d2c3f16cf6b1..02e0086b10f6 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -261,18 +261,14 @@ void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *); extern const struct pipe_buf_operations nosteal_pipe_buf_ops; -#ifdef CONFIG_WATCH_QUEUE unsigned long account_pipe_buffers(struct user_struct *user, unsigned long old, unsigned long new); bool too_many_pipe_buffers_soft(unsigned long user_bufs); bool too_many_pipe_buffers_hard(unsigned long user_bufs); bool pipe_is_unprivileged_user(void); -#endif /* for F_SETPIPE_SZ and F_GETPIPE_SZ */ -#ifdef CONFIG_WATCH_QUEUE int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots); -#endif long pipe_fcntl(struct file *, unsigned int, unsigned long arg); struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice); -- cgit v1.2.3 From ef104443bffa004f631729dfc924f0b84abbd602 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 21:57:29 +0200 Subject: procfs: consolidate arch_report_meminfo declaration The arch_report_meminfo() function is provided by four architectures, with a __weak fallback in procfs itself. On architectures that don't have a custom version, the __weak version causes a warning because of the missing prototype. Remove the architecture specific prototypes and instead add one in linux/proc_fs.h. Signed-off-by: Arnd Bergmann Acked-by: Dave Hansen # for arch/x86 Acked-by: Helge Deller # parisc Reviewed-by: Alexander Gordeev Message-Id: <20230516195834.551901-1-arnd@kernel.org> Signed-off-by: Christian Brauner --- include/linux/proc_fs.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 0260f5ea98fe..253f2676d93a 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -158,6 +158,8 @@ int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task); #endif /* CONFIG_PROC_PID_ARCH_STATUS */ +void arch_report_meminfo(struct seq_file *m); + #else /* CONFIG_PROC_FS */ static inline void proc_root_init(void) -- cgit v1.2.3 From 67d1b0a1030fb20d54b720df6e976c06b893fb00 Mon Sep 17 00:00:00 2001 From: Tero Kristo Date: Fri, 14 Apr 2023 10:25:39 +0530 Subject: soc: ti: pruss: Add pruss_get()/put() API Add two new get and put API, pruss_get() and pruss_put() to the PRUSS platform driver to allow client drivers to request a handle to a PRUSS device. This handle will be used by client drivers to request various operations of the PRUSS platform driver through additional API that will be added in the following patches. The pruss_get() function returns the pruss handle corresponding to a PRUSS device referenced by a PRU remoteproc instance. The pruss_put() is the complimentary function to pruss_get(). Co-developed-by: Suman Anna Signed-off-by: Suman Anna Signed-off-by: Tero Kristo Co-developed-by: Grzegorz Jaszczyk Signed-off-by: Grzegorz Jaszczyk Signed-off-by: Puranjay Mohan Reviewed-by: Roger Quadros Reviewed-by: Tony Lindgren Reviewed-by: Simon Horman Acked-by: Mathieu Poirier Signed-off-by: MD Danish Anwar Link: https://lore.kernel.org/r/20230414045542.3249939-2-danishanwar@ti.com Signed-off-by: Nishanth Menon --- include/linux/pruss_driver.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pruss_driver.h b/include/linux/pruss_driver.h index ecfded30ed05..cb40c2b31045 100644 --- a/include/linux/pruss_driver.h +++ b/include/linux/pruss_driver.h @@ -9,7 +9,9 @@ #ifndef _PRUSS_DRIVER_H_ #define _PRUSS_DRIVER_H_ +#include #include +#include /* * enum pruss_mem - PRUSS memory range identifiers @@ -51,4 +53,20 @@ struct pruss { struct clk *iep_clk_mux; }; +#if IS_ENABLED(CONFIG_TI_PRUSS) + +struct pruss *pruss_get(struct rproc *rproc); +void pruss_put(struct pruss *pruss); + +#else + +static inline struct pruss *pruss_get(struct rproc *rproc) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void pruss_put(struct pruss *pruss) { } + +#endif /* CONFIG_TI_PRUSS */ + #endif /* _PRUSS_DRIVER_H_ */ -- cgit v1.2.3 From b789ca1e3380ab63b60c3356c026a7e8eb26ba01 Mon Sep 17 00:00:00 2001 From: "Andrew F. Davis" Date: Fri, 14 Apr 2023 10:25:40 +0530 Subject: soc: ti: pruss: Add pruss_{request,release}_mem_region() API Add two new API - pruss_request_mem_region() & pruss_release_mem_region(), to the PRUSS platform driver to allow client drivers to acquire and release the common memory resources present within a PRU-ICSS subsystem. This allows the client drivers to directly manipulate the respective memories, as per their design contract with the associated firmware. Co-developed-by: Suman Anna Signed-off-by: Suman Anna Signed-off-by: Andrew F. Davis Co-developed-by: Grzegorz Jaszczyk Signed-off-by: Grzegorz Jaszczyk Reviewed-by: Roger Quadros Acked-by: Mathieu Poirier Reviewed-by: Tony Lindgren Reviewed-by: Simon Horman Signed-off-by: MD Danish Anwar Link: https://lore.kernel.org/r/20230414045542.3249939-3-danishanwar@ti.com Signed-off-by: Nishanth Menon --- include/linux/pruss_driver.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pruss_driver.h b/include/linux/pruss_driver.h index cb40c2b31045..c8f2e53b911b 100644 --- a/include/linux/pruss_driver.h +++ b/include/linux/pruss_driver.h @@ -9,6 +9,7 @@ #ifndef _PRUSS_DRIVER_H_ #define _PRUSS_DRIVER_H_ +#include #include #include #include @@ -41,6 +42,8 @@ struct pruss_mem_region { * @cfg_base: base iomap for CFG region * @cfg_regmap: regmap for config region * @mem_regions: data for each of the PRUSS memory regions + * @mem_in_use: to indicate if memory resource is in use + * @lock: mutex to serialize access to resources * @core_clk_mux: clk handle for PRUSS CORE_CLK_MUX * @iep_clk_mux: clk handle for PRUSS IEP_CLK_MUX */ @@ -49,6 +52,8 @@ struct pruss { void __iomem *cfg_base; struct regmap *cfg_regmap; struct pruss_mem_region mem_regions[PRUSS_MEM_MAX]; + struct pruss_mem_region *mem_in_use[PRUSS_MEM_MAX]; + struct mutex lock; /* PRU resource lock */ struct clk *core_clk_mux; struct clk *iep_clk_mux; }; @@ -57,6 +62,10 @@ struct pruss { struct pruss *pruss_get(struct rproc *rproc); void pruss_put(struct pruss *pruss); +int pruss_request_mem_region(struct pruss *pruss, enum pruss_mem mem_id, + struct pruss_mem_region *region); +int pruss_release_mem_region(struct pruss *pruss, + struct pruss_mem_region *region); #else @@ -67,6 +76,19 @@ static inline struct pruss *pruss_get(struct rproc *rproc) static inline void pruss_put(struct pruss *pruss) { } +static inline int pruss_request_mem_region(struct pruss *pruss, + enum pruss_mem mem_id, + struct pruss_mem_region *region) +{ + return -EOPNOTSUPP; +} + +static inline int pruss_release_mem_region(struct pruss *pruss, + struct pruss_mem_region *region) +{ + return -EOPNOTSUPP; +} + #endif /* CONFIG_TI_PRUSS */ #endif /* _PRUSS_DRIVER_H_ */ -- cgit v1.2.3 From 51b5760e56ef19106a3c4487a66d186d46ccc6f4 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Fri, 14 Apr 2023 10:25:41 +0530 Subject: soc: ti: pruss: Add pruss_cfg_read()/update(), pruss_cfg_get_gpmux()/set_gpmux() APIs Add two new generic API pruss_cfg_read() and pruss_cfg_update() to the PRUSS platform driver to read and program respectively a register within the PRUSS CFG sub-module represented by a syscon driver. These APIs are internal to PRUSS driver. Add two new helper functions pruss_cfg_get_gpmux() & pruss_cfg_set_gpmux() to get and set the GP MUX mode for programming the PRUSS internal wrapper mux functionality as needed by usecases. Various useful registers and macros for certain register bit-fields and their values have also been added. Signed-off-by: Suman Anna Co-developed-by: Grzegorz Jaszczyk Signed-off-by: Grzegorz Jaszczyk Signed-off-by: Puranjay Mohan Reviewed-by: Roger Quadros Reviewed-by: Tony Lindgren Reviewed-by: Simon Horman Acked-by: Mathieu Poirier Signed-off-by: MD Danish Anwar Link: https://lore.kernel.org/r/20230414045542.3249939-4-danishanwar@ti.com Signed-off-by: Nishanth Menon --- include/linux/pruss_driver.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pruss_driver.h b/include/linux/pruss_driver.h index c8f2e53b911b..5bb8897724a9 100644 --- a/include/linux/pruss_driver.h +++ b/include/linux/pruss_driver.h @@ -14,6 +14,24 @@ #include #include +/* + * enum pruss_gp_mux_sel - PRUSS GPI/O Mux modes for the + * PRUSS_GPCFG0/1 registers + * + * NOTE: The below defines are the most common values, but there + * are some exceptions like on 66AK2G, where the RESERVED and MII2 + * values are interchanged. Also, this bit-field does not exist on + * AM335x SoCs + */ +enum pruss_gp_mux_sel { + PRUSS_GP_MUX_SEL_GP, + PRUSS_GP_MUX_SEL_ENDAT, + PRUSS_GP_MUX_SEL_RESERVED, + PRUSS_GP_MUX_SEL_SD, + PRUSS_GP_MUX_SEL_MII2, + PRUSS_GP_MUX_SEL_MAX, +}; + /* * enum pruss_mem - PRUSS memory range identifiers */ @@ -66,6 +84,8 @@ int pruss_request_mem_region(struct pruss *pruss, enum pruss_mem mem_id, struct pruss_mem_region *region); int pruss_release_mem_region(struct pruss *pruss, struct pruss_mem_region *region); +int pruss_cfg_get_gpmux(struct pruss *pruss, enum pruss_pru_id pru_id, u8 *mux); +int pruss_cfg_set_gpmux(struct pruss *pruss, enum pruss_pru_id pru_id, u8 mux); #else @@ -89,6 +109,18 @@ static inline int pruss_release_mem_region(struct pruss *pruss, return -EOPNOTSUPP; } +static inline int pruss_cfg_get_gpmux(struct pruss *pruss, + enum pruss_pru_id pru_id, u8 *mux) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int pruss_cfg_set_gpmux(struct pruss *pruss, + enum pruss_pru_id pru_id, u8 mux) +{ + return ERR_PTR(-EOPNOTSUPP); +} + #endif /* CONFIG_TI_PRUSS */ #endif /* _PRUSS_DRIVER_H_ */ -- cgit v1.2.3 From 0211cc1e4fbbc81853227147bf0982c47362c567 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Fri, 14 Apr 2023 10:25:42 +0530 Subject: soc: ti: pruss: Add helper functions to set GPI mode, MII_RT_event and XFR The PRUSS CFG module is represented as a syscon node and is currently managed by the PRUSS platform driver. Add easy accessor functions to set GPI mode, MII_RT event enable/disable and XFR (XIN XOUT) enable/disable to enable the PRUSS Ethernet usecase. These functions reuse the generic pruss_cfg_update() API function. Signed-off-by: Suman Anna Co-developed-by: Grzegorz Jaszczyk Signed-off-by: Grzegorz Jaszczyk Signed-off-by: Puranjay Mohan Reviewed-by: Roger Quadros Reviewed-by: Tony Lindgren Reviewed-by: Simon Horman Reviewed-by: Mathieu Poirier Signed-off-by: MD Danish Anwar Link: https://lore.kernel.org/r/20230414045542.3249939-5-danishanwar@ti.com Signed-off-by: Nishanth Menon --- include/linux/pruss_driver.h | 51 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pruss_driver.h b/include/linux/pruss_driver.h index 5bb8897724a9..c9a31c567e85 100644 --- a/include/linux/pruss_driver.h +++ b/include/linux/pruss_driver.h @@ -32,6 +32,33 @@ enum pruss_gp_mux_sel { PRUSS_GP_MUX_SEL_MAX, }; +/* + * enum pruss_gpi_mode - PRUSS GPI configuration modes, used + * to program the PRUSS_GPCFG0/1 registers + */ +enum pruss_gpi_mode { + PRUSS_GPI_MODE_DIRECT, + PRUSS_GPI_MODE_PARALLEL, + PRUSS_GPI_MODE_28BIT_SHIFT, + PRUSS_GPI_MODE_MII, + PRUSS_GPI_MODE_MAX, +}; + +/** + * enum pru_type - PRU core type identifier + * + * @PRU_TYPE_PRU: Programmable Real-time Unit + * @PRU_TYPE_RTU: Auxiliary Programmable Real-Time Unit + * @PRU_TYPE_TX_PRU: Transmit Programmable Real-Time Unit + * @PRU_TYPE_MAX: just keep this one at the end + */ +enum pru_type { + PRU_TYPE_PRU, + PRU_TYPE_RTU, + PRU_TYPE_TX_PRU, + PRU_TYPE_MAX, +}; + /* * enum pruss_mem - PRUSS memory range identifiers */ @@ -86,6 +113,11 @@ int pruss_release_mem_region(struct pruss *pruss, struct pruss_mem_region *region); int pruss_cfg_get_gpmux(struct pruss *pruss, enum pruss_pru_id pru_id, u8 *mux); int pruss_cfg_set_gpmux(struct pruss *pruss, enum pruss_pru_id pru_id, u8 mux); +int pruss_cfg_gpimode(struct pruss *pruss, enum pruss_pru_id pru_id, + enum pruss_gpi_mode mode); +int pruss_cfg_miirt_enable(struct pruss *pruss, bool enable); +int pruss_cfg_xfr_enable(struct pruss *pruss, enum pru_type pru_type, + bool enable); #else @@ -121,6 +153,25 @@ static inline int pruss_cfg_set_gpmux(struct pruss *pruss, return ERR_PTR(-EOPNOTSUPP); } +static inline int pruss_cfg_gpimode(struct pruss *pruss, + enum pruss_pru_id pru_id, + enum pruss_gpi_mode mode) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int pruss_cfg_miirt_enable(struct pruss *pruss, bool enable) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int pruss_cfg_xfr_enable(struct pruss *pruss, + enum pru_type pru_type, + bool enable); +{ + return ERR_PTR(-EOPNOTSUPP); +} + #endif /* CONFIG_TI_PRUSS */ #endif /* _PRUSS_DRIVER_H_ */ -- cgit v1.2.3 From e455ca40dbcf2cd50d1e59bf4b2752b300bcdad4 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:52 +0200 Subject: audit: avoid missing-prototype warnings Building with 'make W=1' reveals two function definitions without a previous prototype in the audit code: lib/compat_audit.c:32:5: error: no previous prototype for 'audit_classify_compat_syscall' [-Werror=missing-prototypes] kernel/audit.c:1813:14: error: no previous prototype for 'audit_serial' [-Werror=missing-prototypes] The first one needs a declaration from linux/audit.h but cannot include that header without causing conflicting (compat) syscall number definitions, so move the it into linux/audit_arch.h. The second one is declared conditionally based on CONFIG_AUDITSYSCALL but needed as a local function even when that option is disabled, so move the declaration out of the #ifdef block. Signed-off-by: Arnd Bergmann Signed-off-by: Paul Moore --- include/linux/audit.h | 2 -- include/linux/audit_arch.h | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/audit.h b/include/linux/audit.h index 31086a72e32a..6a3a9e122bb5 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -130,8 +130,6 @@ extern unsigned compat_dir_class[]; extern unsigned compat_chattr_class[]; extern unsigned compat_signal_class[]; -extern int audit_classify_compat_syscall(int abi, unsigned syscall); - /* audit_names->type values */ #define AUDIT_TYPE_UNKNOWN 0 /* we don't know yet */ #define AUDIT_TYPE_NORMAL 1 /* a "normal" audit record */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 8fdb1afe251a..0e34d673ef17 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -21,4 +21,6 @@ enum auditsc_class_t { AUDITSC_NVALS /* count */ }; +extern int audit_classify_compat_syscall(int abi, unsigned syscall); + #endif -- cgit v1.2.3 From 8ff1541da3908b504cb53e5384d5deae2b9c6e1a Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Fri, 12 May 2023 12:24:42 +0200 Subject: fbdev: Include instead of Replace include statements for with . Fixes the coding style: if a header is available in asm/ and linux/, it is preferable to include the header from linux/. This only affects a few source files, most of which already include . Suggested-by: Sam Ravnborg Signed-off-by: Thomas Zimmermann Reviewed-by: Arnd Bergmann Reviewed-by: Sui Jingfeng Reviewed-by: Sam Ravnborg Link: https://patchwork.freedesktop.org/patch/msgid/20230512102444.5438-6-tzimmermann@suse.de --- include/linux/fb.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index ec978a4969a9..4b4d9a5d200a 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -15,6 +15,8 @@ #include #include #include + +#include #include struct vm_area_struct; -- cgit v1.2.3 From 8f8eaa1b023580f7dce7fe8d73539b093edea65b Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Fri, 12 May 2023 12:24:43 +0200 Subject: fbdev: Move framebuffer I/O helpers into Implement framebuffer I/O helpers, such as fb_read*() and fb_write*(), in the architecture's header file or the generic one. The common case has been the use of regular I/O functions, such as __raw_readb() or memset_io(). A few architectures used plain system- memory reads and writes. Sparc used helpers for its SBus. The architectures that used special cases provide the same code in their __raw_*() I/O helpers. So the patch replaces this code with the __raw_*() functions and moves it to for all architectures. v8: * remove garbage after commit-message tags v6: * fix fb_readq()/fb_writeq() on 64-bit mips (kernel test robot) v5: * include in ; fix s390 build v4: * ia64, loongarch, sparc64: add fb_mem*() to arch headers to keep current semantics (Arnd) v3: * implement all architectures with generic helpers * support reordering and native byte order (Geert, Arnd) Signed-off-by: Thomas Zimmermann Tested-by: Sui Jingfeng Reviewed-by: Arnd Bergmann Link: https://patchwork.freedesktop.org/patch/msgid/20230512102444.5438-7-tzimmermann@suse.de --- include/linux/fb.h | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 4b4d9a5d200a..2cf8efcb9e32 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -17,7 +17,6 @@ #include #include -#include struct vm_area_struct; struct fb_info; @@ -513,58 +512,6 @@ struct fb_info { */ #define STUPID_ACCELF_TEXT_SHIT -// This will go away -#if defined(__sparc__) - -/* We map all of our framebuffers such that big-endian accesses - * are what we want, so the following is sufficient. - */ - -// This will go away -#define fb_readb sbus_readb -#define fb_readw sbus_readw -#define fb_readl sbus_readl -#define fb_readq sbus_readq -#define fb_writeb sbus_writeb -#define fb_writew sbus_writew -#define fb_writel sbus_writel -#define fb_writeq sbus_writeq -#define fb_memset sbus_memset_io -#define fb_memcpy_fromfb sbus_memcpy_fromio -#define fb_memcpy_tofb sbus_memcpy_toio - -#elif defined(__i386__) || defined(__alpha__) || defined(__x86_64__) || \ - defined(__hppa__) || defined(__sh__) || defined(__powerpc__) || \ - defined(__arm__) || defined(__aarch64__) || defined(__mips__) - -#define fb_readb __raw_readb -#define fb_readw __raw_readw -#define fb_readl __raw_readl -#define fb_readq __raw_readq -#define fb_writeb __raw_writeb -#define fb_writew __raw_writew -#define fb_writel __raw_writel -#define fb_writeq __raw_writeq -#define fb_memset memset_io -#define fb_memcpy_fromfb memcpy_fromio -#define fb_memcpy_tofb memcpy_toio - -#else - -#define fb_readb(addr) (*(volatile u8 *) (addr)) -#define fb_readw(addr) (*(volatile u16 *) (addr)) -#define fb_readl(addr) (*(volatile u32 *) (addr)) -#define fb_readq(addr) (*(volatile u64 *) (addr)) -#define fb_writeb(b,addr) (*(volatile u8 *) (addr) = (b)) -#define fb_writew(b,addr) (*(volatile u16 *) (addr) = (b)) -#define fb_writel(b,addr) (*(volatile u32 *) (addr) = (b)) -#define fb_writeq(b,addr) (*(volatile u64 *) (addr) = (b)) -#define fb_memset memset -#define fb_memcpy_fromfb memcpy -#define fb_memcpy_tofb memcpy - -#endif - #define FB_LEFT_POS(p, bpp) (fb_be_math(p) ? (32 - (bpp)) : 0) #define FB_SHIFT_HIGH(p, val, bits) (fb_be_math(p) ? (val) >> (bits) : \ (val) << (bits)) -- cgit v1.2.3 From 325bec7157b3859b45b9471447f5d130ab8a8723 Mon Sep 17 00:00:00 2001 From: Julien Panis Date: Thu, 11 May 2023 11:51:22 +0200 Subject: mfd: tps6594: Add driver for TI TPS6594 PMIC This patch adds support for TPS6594 PMIC MFD core. It provides communication through the I2C and SPI interfaces, and supports protocols with embedded CRC data fields for safety applications. Signed-off-by: Julien Panis Link: https://lore.kernel.org/r/20230511095126.105104-3-jpanis@baylibre.com Signed-off-by: Lee Jones --- include/linux/mfd/tps6594.h | 1020 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1020 insertions(+) create mode 100644 include/linux/mfd/tps6594.h (limited to 'include/linux') diff --git a/include/linux/mfd/tps6594.h b/include/linux/mfd/tps6594.h new file mode 100644 index 000000000000..3f7c5e23cd4c --- /dev/null +++ b/include/linux/mfd/tps6594.h @@ -0,0 +1,1020 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Functions to access TPS6594 Power Management IC + * + * Copyright (C) 2023 BayLibre Incorporated - https://www.baylibre.com/ + */ + +#ifndef __LINUX_MFD_TPS6594_H +#define __LINUX_MFD_TPS6594_H + +#include +#include + +struct regmap_irq_chip_data; + +/* Chip id list */ +enum pmic_id { + TPS6594, + TPS6593, + LP8764, +}; + +/* Macro to get page index from register address */ +#define TPS6594_REG_TO_PAGE(reg) ((reg) >> 8) + +/* Registers for page 0 of TPS6594 */ +#define TPS6594_REG_DEV_REV 0x01 + +#define TPS6594_REG_NVM_CODE_1 0x02 +#define TPS6594_REG_NVM_CODE_2 0x03 + +#define TPS6594_REG_BUCKX_CTRL(buck_inst) (0x04 + ((buck_inst) << 1)) +#define TPS6594_REG_BUCKX_CONF(buck_inst) (0x05 + ((buck_inst) << 1)) +#define TPS6594_REG_BUCKX_VOUT_1(buck_inst) (0x0e + ((buck_inst) << 1)) +#define TPS6594_REG_BUCKX_VOUT_2(buck_inst) (0x0f + ((buck_inst) << 1)) +#define TPS6594_REG_BUCKX_PG_WINDOW(buck_inst) (0x18 + (buck_inst)) + +#define TPS6594_REG_LDOX_CTRL(ldo_inst) (0x1d + (ldo_inst)) +#define TPS6594_REG_LDORTC_CTRL 0x22 +#define TPS6594_REG_LDOX_VOUT(ldo_inst) (0x23 + (ldo_inst)) +#define TPS6594_REG_LDOX_PG_WINDOW(ldo_inst) (0x27 + (ldo_inst)) + +#define TPS6594_REG_VCCA_VMON_CTRL 0x2b +#define TPS6594_REG_VCCA_PG_WINDOW 0x2c +#define TPS6594_REG_VMON1_PG_WINDOW 0x2d +#define TPS6594_REG_VMON1_PG_LEVEL 0x2e +#define TPS6594_REG_VMON2_PG_WINDOW 0x2f +#define TPS6594_REG_VMON2_PG_LEVEL 0x30 + +#define TPS6594_REG_GPIOX_CONF(gpio_inst) (0x31 + (gpio_inst)) +#define TPS6594_REG_NPWRON_CONF 0x3c +#define TPS6594_REG_GPIO_OUT_1 0x3d +#define TPS6594_REG_GPIO_OUT_2 0x3e +#define TPS6594_REG_GPIO_IN_1 0x3f +#define TPS6594_REG_GPIO_IN_2 0x40 +#define TPS6594_REG_GPIOX_OUT(gpio_inst) (TPS6594_REG_GPIO_OUT_1 + (gpio_inst) / 8) +#define TPS6594_REG_GPIOX_IN(gpio_inst) (TPS6594_REG_GPIO_IN_1 + (gpio_inst) / 8) + +#define TPS6594_REG_GPIO_IN_1 0x3f +#define TPS6594_REG_GPIO_IN_2 0x40 + +#define TPS6594_REG_RAIL_SEL_1 0x41 +#define TPS6594_REG_RAIL_SEL_2 0x42 +#define TPS6594_REG_RAIL_SEL_3 0x43 + +#define TPS6594_REG_FSM_TRIG_SEL_1 0x44 +#define TPS6594_REG_FSM_TRIG_SEL_2 0x45 +#define TPS6594_REG_FSM_TRIG_MASK_1 0x46 +#define TPS6594_REG_FSM_TRIG_MASK_2 0x47 +#define TPS6594_REG_FSM_TRIG_MASK_3 0x48 + +#define TPS6594_REG_MASK_BUCK1_2 0x49 +#define TPS6594_REG_MASK_BUCK3_4 0x4a +#define TPS6594_REG_MASK_BUCK5 0x4b +#define TPS6594_REG_MASK_LDO1_2 0x4c +#define TPS6594_REG_MASK_LDO3_4 0x4d +#define TPS6594_REG_MASK_VMON 0x4e +#define TPS6594_REG_MASK_GPIO1_8_FALL 0x4f +#define TPS6594_REG_MASK_GPIO1_8_RISE 0x50 +#define TPS6594_REG_MASK_GPIO9_11 0x51 +#define TPS6594_REG_MASK_STARTUP 0x52 +#define TPS6594_REG_MASK_MISC 0x53 +#define TPS6594_REG_MASK_MODERATE_ERR 0x54 +#define TPS6594_REG_MASK_FSM_ERR 0x56 +#define TPS6594_REG_MASK_COMM_ERR 0x57 +#define TPS6594_REG_MASK_READBACK_ERR 0x58 +#define TPS6594_REG_MASK_ESM 0x59 + +#define TPS6594_REG_INT_TOP 0x5a +#define TPS6594_REG_INT_BUCK 0x5b +#define TPS6594_REG_INT_BUCK1_2 0x5c +#define TPS6594_REG_INT_BUCK3_4 0x5d +#define TPS6594_REG_INT_BUCK5 0x5e +#define TPS6594_REG_INT_LDO_VMON 0x5f +#define TPS6594_REG_INT_LDO1_2 0x60 +#define TPS6594_REG_INT_LDO3_4 0x61 +#define TPS6594_REG_INT_VMON 0x62 +#define TPS6594_REG_INT_GPIO 0x63 +#define TPS6594_REG_INT_GPIO1_8 0x64 +#define TPS6594_REG_INT_STARTUP 0x65 +#define TPS6594_REG_INT_MISC 0x66 +#define TPS6594_REG_INT_MODERATE_ERR 0x67 +#define TPS6594_REG_INT_SEVERE_ERR 0x68 +#define TPS6594_REG_INT_FSM_ERR 0x69 +#define TPS6594_REG_INT_COMM_ERR 0x6a +#define TPS6594_REG_INT_READBACK_ERR 0x6b +#define TPS6594_REG_INT_ESM 0x6c + +#define TPS6594_REG_STAT_BUCK1_2 0x6d +#define TPS6594_REG_STAT_BUCK3_4 0x6e +#define TPS6594_REG_STAT_BUCK5 0x6f +#define TPS6594_REG_STAT_LDO1_2 0x70 +#define TPS6594_REG_STAT_LDO3_4 0x71 +#define TPS6594_REG_STAT_VMON 0x72 +#define TPS6594_REG_STAT_STARTUP 0x73 +#define TPS6594_REG_STAT_MISC 0x74 +#define TPS6594_REG_STAT_MODERATE_ERR 0x75 +#define TPS6594_REG_STAT_SEVERE_ERR 0x76 +#define TPS6594_REG_STAT_READBACK_ERR 0x77 + +#define TPS6594_REG_PGOOD_SEL_1 0x78 +#define TPS6594_REG_PGOOD_SEL_2 0x79 +#define TPS6594_REG_PGOOD_SEL_3 0x7a +#define TPS6594_REG_PGOOD_SEL_4 0x7b + +#define TPS6594_REG_PLL_CTRL 0x7c + +#define TPS6594_REG_CONFIG_1 0x7d +#define TPS6594_REG_CONFIG_2 0x7e + +#define TPS6594_REG_ENABLE_DRV_REG 0x80 + +#define TPS6594_REG_MISC_CTRL 0x81 + +#define TPS6594_REG_ENABLE_DRV_STAT 0x82 + +#define TPS6594_REG_RECOV_CNT_REG_1 0x83 +#define TPS6594_REG_RECOV_CNT_REG_2 0x84 + +#define TPS6594_REG_FSM_I2C_TRIGGERS 0x85 +#define TPS6594_REG_FSM_NSLEEP_TRIGGERS 0x86 + +#define TPS6594_REG_BUCK_RESET_REG 0x87 + +#define TPS6594_REG_SPREAD_SPECTRUM_1 0x88 + +#define TPS6594_REG_FREQ_SEL 0x8a + +#define TPS6594_REG_FSM_STEP_SIZE 0x8b + +#define TPS6594_REG_LDO_RV_TIMEOUT_REG_1 0x8c +#define TPS6594_REG_LDO_RV_TIMEOUT_REG_2 0x8d + +#define TPS6594_REG_USER_SPARE_REGS 0x8e + +#define TPS6594_REG_ESM_MCU_START_REG 0x8f +#define TPS6594_REG_ESM_MCU_DELAY1_REG 0x90 +#define TPS6594_REG_ESM_MCU_DELAY2_REG 0x91 +#define TPS6594_REG_ESM_MCU_MODE_CFG 0x92 +#define TPS6594_REG_ESM_MCU_HMAX_REG 0x93 +#define TPS6594_REG_ESM_MCU_HMIN_REG 0x94 +#define TPS6594_REG_ESM_MCU_LMAX_REG 0x95 +#define TPS6594_REG_ESM_MCU_LMIN_REG 0x96 +#define TPS6594_REG_ESM_MCU_ERR_CNT_REG 0x97 +#define TPS6594_REG_ESM_SOC_START_REG 0x98 +#define TPS6594_REG_ESM_SOC_DELAY1_REG 0x99 +#define TPS6594_REG_ESM_SOC_DELAY2_REG 0x9a +#define TPS6594_REG_ESM_SOC_MODE_CFG 0x9b +#define TPS6594_REG_ESM_SOC_HMAX_REG 0x9c +#define TPS6594_REG_ESM_SOC_HMIN_REG 0x9d +#define TPS6594_REG_ESM_SOC_LMAX_REG 0x9e +#define TPS6594_REG_ESM_SOC_LMIN_REG 0x9f +#define TPS6594_REG_ESM_SOC_ERR_CNT_REG 0xa0 + +#define TPS6594_REG_REGISTER_LOCK 0xa1 + +#define TPS6594_REG_MANUFACTURING_VER 0xa6 + +#define TPS6594_REG_CUSTOMER_NVM_ID_REG 0xa7 + +#define TPS6594_REG_VMON_CONF_REG 0xa8 + +#define TPS6594_REG_SOFT_REBOOT_REG 0xab + +#define TPS6594_REG_RTC_SECONDS 0xb5 +#define TPS6594_REG_RTC_MINUTES 0xb6 +#define TPS6594_REG_RTC_HOURS 0xb7 +#define TPS6594_REG_RTC_DAYS 0xb8 +#define TPS6594_REG_RTC_MONTHS 0xb9 +#define TPS6594_REG_RTC_YEARS 0xba +#define TPS6594_REG_RTC_WEEKS 0xbb + +#define TPS6594_REG_ALARM_SECONDS 0xbc +#define TPS6594_REG_ALARM_MINUTES 0xbd +#define TPS6594_REG_ALARM_HOURS 0xbe +#define TPS6594_REG_ALARM_DAYS 0xbf +#define TPS6594_REG_ALARM_MONTHS 0xc0 +#define TPS6594_REG_ALARM_YEARS 0xc1 + +#define TPS6594_REG_RTC_CTRL_1 0xc2 +#define TPS6594_REG_RTC_CTRL_2 0xc3 +#define TPS6594_REG_RTC_STATUS 0xc4 +#define TPS6594_REG_RTC_INTERRUPTS 0xc5 +#define TPS6594_REG_RTC_COMP_LSB 0xc6 +#define TPS6594_REG_RTC_COMP_MSB 0xc7 +#define TPS6594_REG_RTC_RESET_STATUS 0xc8 + +#define TPS6594_REG_SCRATCH_PAD_REG_1 0xc9 +#define TPS6594_REG_SCRATCH_PAD_REG_2 0xca +#define TPS6594_REG_SCRATCH_PAD_REG_3 0xcb +#define TPS6594_REG_SCRATCH_PAD_REG_4 0xcc + +#define TPS6594_REG_PFSM_DELAY_REG_1 0xcd +#define TPS6594_REG_PFSM_DELAY_REG_2 0xce +#define TPS6594_REG_PFSM_DELAY_REG_3 0xcf +#define TPS6594_REG_PFSM_DELAY_REG_4 0xd0 + +/* Registers for page 1 of TPS6594 */ +#define TPS6594_REG_SERIAL_IF_CONFIG 0x11a +#define TPS6594_REG_I2C1_ID 0x122 +#define TPS6594_REG_I2C2_ID 0x123 + +/* Registers for page 4 of TPS6594 */ +#define TPS6594_REG_WD_ANSWER_REG 0x401 +#define TPS6594_REG_WD_QUESTION_ANSW_CNT 0x402 +#define TPS6594_REG_WD_WIN1_CFG 0x403 +#define TPS6594_REG_WD_WIN2_CFG 0x404 +#define TPS6594_REG_WD_LONGWIN_CFG 0x405 +#define TPS6594_REG_WD_MODE_REG 0x406 +#define TPS6594_REG_WD_QA_CFG 0x407 +#define TPS6594_REG_WD_ERR_STATUS 0x408 +#define TPS6594_REG_WD_THR_CFG 0x409 +#define TPS6594_REG_DWD_FAIL_CNT_REG 0x40a + +/* BUCKX_CTRL register field definition */ +#define TPS6594_BIT_BUCK_EN BIT(0) +#define TPS6594_BIT_BUCK_FPWM BIT(1) +#define TPS6594_BIT_BUCK_FPWM_MP BIT(2) +#define TPS6594_BIT_BUCK_VSEL BIT(3) +#define TPS6594_BIT_BUCK_VMON_EN BIT(4) +#define TPS6594_BIT_BUCK_PLDN BIT(5) +#define TPS6594_BIT_BUCK_RV_SEL BIT(7) + +/* BUCKX_CONF register field definition */ +#define TPS6594_MASK_BUCK_SLEW_RATE GENMASK(2, 0) +#define TPS6594_MASK_BUCK_ILIM GENMASK(5, 3) + +/* BUCKX_PG_WINDOW register field definition */ +#define TPS6594_MASK_BUCK_OV_THR GENMASK(2, 0) +#define TPS6594_MASK_BUCK_UV_THR GENMASK(5, 3) + +/* BUCKX VSET */ +#define TPS6594_MASK_BUCKS_VSET GENMASK(7, 0) + +/* LDOX_CTRL register field definition */ +#define TPS6594_BIT_LDO_EN BIT(0) +#define TPS6594_BIT_LDO_SLOW_RAMP BIT(1) +#define TPS6594_BIT_LDO_VMON_EN BIT(4) +#define TPS6594_MASK_LDO_PLDN GENMASK(6, 5) +#define TPS6594_BIT_LDO_RV_SEL BIT(7) + +/* LDORTC_CTRL register field definition */ +#define TPS6594_BIT_LDORTC_DIS BIT(0) + +/* LDOX_VOUT register field definition */ +#define TPS6594_MASK_LDO123_VSET GENMASK(6, 1) +#define TPS6594_MASK_LDO4_VSET GENMASK(6, 0) +#define TPS6594_BIT_LDO_BYPASS BIT(7) + +/* LDOX_PG_WINDOW register field definition */ +#define TPS6594_MASK_LDO_OV_THR GENMASK(2, 0) +#define TPS6594_MASK_LDO_UV_THR GENMASK(5, 3) + +/* VCCA_VMON_CTRL register field definition */ +#define TPS6594_BIT_VMON_EN BIT(0) +#define TPS6594_BIT_VMON1_EN BIT(1) +#define TPS6594_BIT_VMON1_RV_SEL BIT(2) +#define TPS6594_BIT_VMON2_EN BIT(3) +#define TPS6594_BIT_VMON2_RV_SEL BIT(4) +#define TPS6594_BIT_VMON_DEGLITCH_SEL BIT(5) + +/* VCCA_PG_WINDOW register field definition */ +#define TPS6594_MASK_VCCA_OV_THR GENMASK(2, 0) +#define TPS6594_MASK_VCCA_UV_THR GENMASK(5, 3) +#define TPS6594_BIT_VCCA_PG_SET BIT(6) + +/* VMONX_PG_WINDOW register field definition */ +#define TPS6594_MASK_VMONX_OV_THR GENMASK(2, 0) +#define TPS6594_MASK_VMONX_UV_THR GENMASK(5, 3) +#define TPS6594_BIT_VMONX_RANGE BIT(6) + +/* GPIOX_CONF register field definition */ +#define TPS6594_BIT_GPIO_DIR BIT(0) +#define TPS6594_BIT_GPIO_OD BIT(1) +#define TPS6594_BIT_GPIO_PU_SEL BIT(2) +#define TPS6594_BIT_GPIO_PU_PD_EN BIT(3) +#define TPS6594_BIT_GPIO_DEGLITCH_EN BIT(4) +#define TPS6594_MASK_GPIO_SEL GENMASK(7, 5) + +/* NPWRON_CONF register field definition */ +#define TPS6594_BIT_NRSTOUT_OD BIT(0) +#define TPS6594_BIT_ENABLE_PU_SEL BIT(2) +#define TPS6594_BIT_ENABLE_PU_PD_EN BIT(3) +#define TPS6594_BIT_ENABLE_DEGLITCH_EN BIT(4) +#define TPS6594_BIT_ENABLE_POL BIT(5) +#define TPS6594_MASK_NPWRON_SEL GENMASK(7, 6) + +/* GPIO_OUT_X register field definition */ +#define TPS6594_BIT_GPIOX_OUT(gpio_inst) BIT((gpio_inst) % 8) + +/* GPIO_IN_X register field definition */ +#define TPS6594_BIT_GPIOX_IN(gpio_inst) BIT((gpio_inst) % 8) +#define TPS6594_BIT_NPWRON_IN BIT(3) + +/* RAIL_SEL_1 register field definition */ +#define TPS6594_MASK_BUCK1_GRP_SEL GENMASK(1, 0) +#define TPS6594_MASK_BUCK2_GRP_SEL GENMASK(3, 2) +#define TPS6594_MASK_BUCK3_GRP_SEL GENMASK(5, 4) +#define TPS6594_MASK_BUCK4_GRP_SEL GENMASK(7, 6) + +/* RAIL_SEL_2 register field definition */ +#define TPS6594_MASK_BUCK5_GRP_SEL GENMASK(1, 0) +#define TPS6594_MASK_LDO1_GRP_SEL GENMASK(3, 2) +#define TPS6594_MASK_LDO2_GRP_SEL GENMASK(5, 4) +#define TPS6594_MASK_LDO3_GRP_SEL GENMASK(7, 6) + +/* RAIL_SEL_3 register field definition */ +#define TPS6594_MASK_LDO4_GRP_SEL GENMASK(1, 0) +#define TPS6594_MASK_VCCA_GRP_SEL GENMASK(3, 2) +#define TPS6594_MASK_VMON1_GRP_SEL GENMASK(5, 4) +#define TPS6594_MASK_VMON2_GRP_SEL GENMASK(7, 6) + +/* FSM_TRIG_SEL_1 register field definition */ +#define TPS6594_MASK_MCU_RAIL_TRIG GENMASK(1, 0) +#define TPS6594_MASK_SOC_RAIL_TRIG GENMASK(3, 2) +#define TPS6594_MASK_OTHER_RAIL_TRIG GENMASK(5, 4) +#define TPS6594_MASK_SEVERE_ERR_TRIG GENMASK(7, 6) + +/* FSM_TRIG_SEL_2 register field definition */ +#define TPS6594_MASK_MODERATE_ERR_TRIG GENMASK(1, 0) + +/* FSM_TRIG_MASK_X register field definition */ +#define TPS6594_BIT_GPIOX_FSM_MASK(gpio_inst) BIT(((gpio_inst) << 1) % 8) +#define TPS6594_BIT_GPIOX_FSM_MASK_POL(gpio_inst) BIT(((gpio_inst) << 1) % 8 + 1) + +/* MASK_BUCKX register field definition */ +#define TPS6594_BIT_BUCKX_OV_MASK(buck_inst) BIT(((buck_inst) << 2) % 8) +#define TPS6594_BIT_BUCKX_UV_MASK(buck_inst) BIT(((buck_inst) << 2) % 8 + 1) +#define TPS6594_BIT_BUCKX_ILIM_MASK(buck_inst) BIT(((buck_inst) << 2) % 8 + 3) + +/* MASK_LDOX register field definition */ +#define TPS6594_BIT_LDOX_OV_MASK(ldo_inst) BIT(((ldo_inst) << 2) % 8) +#define TPS6594_BIT_LDOX_UV_MASK(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 1) +#define TPS6594_BIT_LDOX_ILIM_MASK(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 3) + +/* MASK_VMON register field definition */ +#define TPS6594_BIT_VCCA_OV_MASK BIT(0) +#define TPS6594_BIT_VCCA_UV_MASK BIT(1) +#define TPS6594_BIT_VMON1_OV_MASK BIT(2) +#define TPS6594_BIT_VMON1_UV_MASK BIT(3) +#define TPS6594_BIT_VMON2_OV_MASK BIT(5) +#define TPS6594_BIT_VMON2_UV_MASK BIT(6) + +/* MASK_GPIOX register field definition */ +#define TPS6594_BIT_GPIOX_FALL_MASK(gpio_inst) BIT((gpio_inst) < 8 ? \ + (gpio_inst) : (gpio_inst) % 8) +#define TPS6594_BIT_GPIOX_RISE_MASK(gpio_inst) BIT((gpio_inst) < 8 ? \ + (gpio_inst) : (gpio_inst) % 8 + 3) + +/* MASK_STARTUP register field definition */ +#define TPS6594_BIT_NPWRON_START_MASK BIT(0) +#define TPS6594_BIT_ENABLE_MASK BIT(1) +#define TPS6594_BIT_FSD_MASK BIT(4) +#define TPS6594_BIT_SOFT_REBOOT_MASK BIT(5) + +/* MASK_MISC register field definition */ +#define TPS6594_BIT_BIST_PASS_MASK BIT(0) +#define TPS6594_BIT_EXT_CLK_MASK BIT(1) +#define TPS6594_BIT_TWARN_MASK BIT(3) + +/* MASK_MODERATE_ERR register field definition */ +#define TPS6594_BIT_BIST_FAIL_MASK BIT(1) +#define TPS6594_BIT_REG_CRC_ERR_MASK BIT(2) +#define TPS6594_BIT_SPMI_ERR_MASK BIT(4) +#define TPS6594_BIT_NPWRON_LONG_MASK BIT(5) +#define TPS6594_BIT_NINT_READBACK_MASK BIT(6) +#define TPS6594_BIT_NRSTOUT_READBACK_MASK BIT(7) + +/* MASK_FSM_ERR register field definition */ +#define TPS6594_BIT_IMM_SHUTDOWN_MASK BIT(0) +#define TPS6594_BIT_ORD_SHUTDOWN_MASK BIT(1) +#define TPS6594_BIT_MCU_PWR_ERR_MASK BIT(2) +#define TPS6594_BIT_SOC_PWR_ERR_MASK BIT(3) + +/* MASK_COMM_ERR register field definition */ +#define TPS6594_BIT_COMM_FRM_ERR_MASK BIT(0) +#define TPS6594_BIT_COMM_CRC_ERR_MASK BIT(1) +#define TPS6594_BIT_COMM_ADR_ERR_MASK BIT(3) +#define TPS6594_BIT_I2C2_CRC_ERR_MASK BIT(5) +#define TPS6594_BIT_I2C2_ADR_ERR_MASK BIT(7) + +/* MASK_READBACK_ERR register field definition */ +#define TPS6594_BIT_EN_DRV_READBACK_MASK BIT(0) +#define TPS6594_BIT_NRSTOUT_SOC_READBACK_MASK BIT(3) + +/* MASK_ESM register field definition */ +#define TPS6594_BIT_ESM_SOC_PIN_MASK BIT(0) +#define TPS6594_BIT_ESM_SOC_FAIL_MASK BIT(1) +#define TPS6594_BIT_ESM_SOC_RST_MASK BIT(2) +#define TPS6594_BIT_ESM_MCU_PIN_MASK BIT(3) +#define TPS6594_BIT_ESM_MCU_FAIL_MASK BIT(4) +#define TPS6594_BIT_ESM_MCU_RST_MASK BIT(5) + +/* INT_TOP register field definition */ +#define TPS6594_BIT_BUCK_INT BIT(0) +#define TPS6594_BIT_LDO_VMON_INT BIT(1) +#define TPS6594_BIT_GPIO_INT BIT(2) +#define TPS6594_BIT_STARTUP_INT BIT(3) +#define TPS6594_BIT_MISC_INT BIT(4) +#define TPS6594_BIT_MODERATE_ERR_INT BIT(5) +#define TPS6594_BIT_SEVERE_ERR_INT BIT(6) +#define TPS6594_BIT_FSM_ERR_INT BIT(7) + +/* INT_BUCK register field definition */ +#define TPS6594_BIT_BUCK1_2_INT BIT(0) +#define TPS6594_BIT_BUCK3_4_INT BIT(1) +#define TPS6594_BIT_BUCK5_INT BIT(2) + +/* INT_BUCKX register field definition */ +#define TPS6594_BIT_BUCKX_OV_INT(buck_inst) BIT(((buck_inst) << 2) % 8) +#define TPS6594_BIT_BUCKX_UV_INT(buck_inst) BIT(((buck_inst) << 2) % 8 + 1) +#define TPS6594_BIT_BUCKX_SC_INT(buck_inst) BIT(((buck_inst) << 2) % 8 + 2) +#define TPS6594_BIT_BUCKX_ILIM_INT(buck_inst) BIT(((buck_inst) << 2) % 8 + 3) + +/* INT_LDO_VMON register field definition */ +#define TPS6594_BIT_LDO1_2_INT BIT(0) +#define TPS6594_BIT_LDO3_4_INT BIT(1) +#define TPS6594_BIT_VCCA_INT BIT(4) + +/* INT_LDOX register field definition */ +#define TPS6594_BIT_LDOX_OV_INT(ldo_inst) BIT(((ldo_inst) << 2) % 8) +#define TPS6594_BIT_LDOX_UV_INT(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 1) +#define TPS6594_BIT_LDOX_SC_INT(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 2) +#define TPS6594_BIT_LDOX_ILIM_INT(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 3) + +/* INT_VMON register field definition */ +#define TPS6594_BIT_VCCA_OV_INT BIT(0) +#define TPS6594_BIT_VCCA_UV_INT BIT(1) +#define TPS6594_BIT_VMON1_OV_INT BIT(2) +#define TPS6594_BIT_VMON1_UV_INT BIT(3) +#define TPS6594_BIT_VMON1_RV_INT BIT(4) +#define TPS6594_BIT_VMON2_OV_INT BIT(5) +#define TPS6594_BIT_VMON2_UV_INT BIT(6) +#define TPS6594_BIT_VMON2_RV_INT BIT(7) + +/* INT_GPIO register field definition */ +#define TPS6594_BIT_GPIO9_INT BIT(0) +#define TPS6594_BIT_GPIO10_INT BIT(1) +#define TPS6594_BIT_GPIO11_INT BIT(2) +#define TPS6594_BIT_GPIO1_8_INT BIT(3) + +/* INT_GPIOX register field definition */ +#define TPS6594_BIT_GPIOX_INT(gpio_inst) BIT(gpio_inst) + +/* INT_STARTUP register field definition */ +#define TPS6594_BIT_NPWRON_START_INT BIT(0) +#define TPS6594_BIT_ENABLE_INT BIT(1) +#define TPS6594_BIT_RTC_INT BIT(2) +#define TPS6594_BIT_FSD_INT BIT(4) +#define TPS6594_BIT_SOFT_REBOOT_INT BIT(5) + +/* INT_MISC register field definition */ +#define TPS6594_BIT_BIST_PASS_INT BIT(0) +#define TPS6594_BIT_EXT_CLK_INT BIT(1) +#define TPS6594_BIT_TWARN_INT BIT(3) + +/* INT_MODERATE_ERR register field definition */ +#define TPS6594_BIT_TSD_ORD_INT BIT(0) +#define TPS6594_BIT_BIST_FAIL_INT BIT(1) +#define TPS6594_BIT_REG_CRC_ERR_INT BIT(2) +#define TPS6594_BIT_RECOV_CNT_INT BIT(3) +#define TPS6594_BIT_SPMI_ERR_INT BIT(4) +#define TPS6594_BIT_NPWRON_LONG_INT BIT(5) +#define TPS6594_BIT_NINT_READBACK_INT BIT(6) +#define TPS6594_BIT_NRSTOUT_READBACK_INT BIT(7) + +/* INT_SEVERE_ERR register field definition */ +#define TPS6594_BIT_TSD_IMM_INT BIT(0) +#define TPS6594_BIT_VCCA_OVP_INT BIT(1) +#define TPS6594_BIT_PFSM_ERR_INT BIT(2) + +/* INT_FSM_ERR register field definition */ +#define TPS6594_BIT_IMM_SHUTDOWN_INT BIT(0) +#define TPS6594_BIT_ORD_SHUTDOWN_INT BIT(1) +#define TPS6594_BIT_MCU_PWR_ERR_INT BIT(2) +#define TPS6594_BIT_SOC_PWR_ERR_INT BIT(3) +#define TPS6594_BIT_COMM_ERR_INT BIT(4) +#define TPS6594_BIT_READBACK_ERR_INT BIT(5) +#define TPS6594_BIT_ESM_INT BIT(6) +#define TPS6594_BIT_WD_INT BIT(7) + +/* INT_COMM_ERR register field definition */ +#define TPS6594_BIT_COMM_FRM_ERR_INT BIT(0) +#define TPS6594_BIT_COMM_CRC_ERR_INT BIT(1) +#define TPS6594_BIT_COMM_ADR_ERR_INT BIT(3) +#define TPS6594_BIT_I2C2_CRC_ERR_INT BIT(5) +#define TPS6594_BIT_I2C2_ADR_ERR_INT BIT(7) + +/* INT_READBACK_ERR register field definition */ +#define TPS6594_BIT_EN_DRV_READBACK_INT BIT(0) +#define TPS6594_BIT_NRSTOUT_SOC_READBACK_INT BIT(3) + +/* INT_ESM register field definition */ +#define TPS6594_BIT_ESM_SOC_PIN_INT BIT(0) +#define TPS6594_BIT_ESM_SOC_FAIL_INT BIT(1) +#define TPS6594_BIT_ESM_SOC_RST_INT BIT(2) +#define TPS6594_BIT_ESM_MCU_PIN_INT BIT(3) +#define TPS6594_BIT_ESM_MCU_FAIL_INT BIT(4) +#define TPS6594_BIT_ESM_MCU_RST_INT BIT(5) + +/* STAT_BUCKX register field definition */ +#define TPS6594_BIT_BUCKX_OV_STAT(buck_inst) BIT(((buck_inst) << 2) % 8) +#define TPS6594_BIT_BUCKX_UV_STAT(buck_inst) BIT(((buck_inst) << 2) % 8 + 1) +#define TPS6594_BIT_BUCKX_ILIM_STAT(buck_inst) BIT(((buck_inst) << 2) % 8 + 3) + +/* STAT_LDOX register field definition */ +#define TPS6594_BIT_LDOX_OV_STAT(ldo_inst) BIT(((ldo_inst) << 2) % 8) +#define TPS6594_BIT_LDOX_UV_STAT(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 1) +#define TPS6594_BIT_LDOX_ILIM_STAT(ldo_inst) BIT(((ldo_inst) << 2) % 8 + 3) + +/* STAT_VMON register field definition */ +#define TPS6594_BIT_VCCA_OV_STAT BIT(0) +#define TPS6594_BIT_VCCA_UV_STAT BIT(1) +#define TPS6594_BIT_VMON1_OV_STAT BIT(2) +#define TPS6594_BIT_VMON1_UV_STAT BIT(3) +#define TPS6594_BIT_VMON2_OV_STAT BIT(5) +#define TPS6594_BIT_VMON2_UV_STAT BIT(6) + +/* STAT_STARTUP register field definition */ +#define TPS6594_BIT_ENABLE_STAT BIT(1) + +/* STAT_MISC register field definition */ +#define TPS6594_BIT_EXT_CLK_STAT BIT(1) +#define TPS6594_BIT_TWARN_STAT BIT(3) + +/* STAT_MODERATE_ERR register field definition */ +#define TPS6594_BIT_TSD_ORD_STAT BIT(0) + +/* STAT_SEVERE_ERR register field definition */ +#define TPS6594_BIT_TSD_IMM_STAT BIT(0) +#define TPS6594_BIT_VCCA_OVP_STAT BIT(1) + +/* STAT_READBACK_ERR register field definition */ +#define TPS6594_BIT_EN_DRV_READBACK_STAT BIT(0) +#define TPS6594_BIT_NINT_READBACK_STAT BIT(1) +#define TPS6594_BIT_NRSTOUT_READBACK_STAT BIT(2) +#define TPS6594_BIT_NRSTOUT_SOC_READBACK_STAT BIT(3) + +/* PGOOD_SEL_1 register field definition */ +#define TPS6594_MASK_PGOOD_SEL_BUCK1 GENMASK(1, 0) +#define TPS6594_MASK_PGOOD_SEL_BUCK2 GENMASK(3, 2) +#define TPS6594_MASK_PGOOD_SEL_BUCK3 GENMASK(5, 4) +#define TPS6594_MASK_PGOOD_SEL_BUCK4 GENMASK(7, 6) + +/* PGOOD_SEL_2 register field definition */ +#define TPS6594_MASK_PGOOD_SEL_BUCK5 GENMASK(1, 0) + +/* PGOOD_SEL_3 register field definition */ +#define TPS6594_MASK_PGOOD_SEL_LDO1 GENMASK(1, 0) +#define TPS6594_MASK_PGOOD_SEL_LDO2 GENMASK(3, 2) +#define TPS6594_MASK_PGOOD_SEL_LDO3 GENMASK(5, 4) +#define TPS6594_MASK_PGOOD_SEL_LDO4 GENMASK(7, 6) + +/* PGOOD_SEL_4 register field definition */ +#define TPS6594_BIT_PGOOD_SEL_VCCA BIT(0) +#define TPS6594_BIT_PGOOD_SEL_VMON1 BIT(1) +#define TPS6594_BIT_PGOOD_SEL_VMON2 BIT(2) +#define TPS6594_BIT_PGOOD_SEL_TDIE_WARN BIT(3) +#define TPS6594_BIT_PGOOD_SEL_NRSTOUT BIT(4) +#define TPS6594_BIT_PGOOD_SEL_NRSTOUT_SOC BIT(5) +#define TPS6594_BIT_PGOOD_POL BIT(6) +#define TPS6594_BIT_PGOOD_WINDOW BIT(7) + +/* PLL_CTRL register field definition */ +#define TPS6594_MASK_EXT_CLK_FREQ GENMASK(1, 0) + +/* CONFIG_1 register field definition */ +#define TPS6594_BIT_TWARN_LEVEL BIT(0) +#define TPS6594_BIT_TSD_ORD_LEVEL BIT(1) +#define TPS6594_BIT_I2C1_HS BIT(3) +#define TPS6594_BIT_I2C2_HS BIT(4) +#define TPS6594_BIT_EN_ILIM_FSM_CTRL BIT(5) +#define TPS6594_BIT_NSLEEP1_MASK BIT(6) +#define TPS6594_BIT_NSLEEP2_MASK BIT(7) + +/* CONFIG_2 register field definition */ +#define TPS6594_BIT_BB_CHARGER_EN BIT(0) +#define TPS6594_BIT_BB_ICHR BIT(1) +#define TPS6594_MASK_BB_VEOC GENMASK(3, 2) +#define TPS6594_BB_EOC_RDY BIT(7) + +/* ENABLE_DRV_REG register field definition */ +#define TPS6594_BIT_ENABLE_DRV BIT(0) + +/* MISC_CTRL register field definition */ +#define TPS6594_BIT_NRSTOUT BIT(0) +#define TPS6594_BIT_NRSTOUT_SOC BIT(1) +#define TPS6594_BIT_LPM_EN BIT(2) +#define TPS6594_BIT_CLKMON_EN BIT(3) +#define TPS6594_BIT_AMUXOUT_EN BIT(4) +#define TPS6594_BIT_SEL_EXT_CLK BIT(5) +#define TPS6594_MASK_SYNCCLKOUT_FREQ_SEL GENMASK(7, 6) + +/* ENABLE_DRV_STAT register field definition */ +#define TPS6594_BIT_EN_DRV_IN BIT(0) +#define TPS6594_BIT_NRSTOUT_IN BIT(1) +#define TPS6594_BIT_NRSTOUT_SOC_IN BIT(2) +#define TPS6594_BIT_FORCE_EN_DRV_LOW BIT(3) +#define TPS6594_BIT_SPMI_LPM_EN BIT(4) + +/* RECOV_CNT_REG_1 register field definition */ +#define TPS6594_MASK_RECOV_CNT GENMASK(3, 0) + +/* RECOV_CNT_REG_2 register field definition */ +#define TPS6594_MASK_RECOV_CNT_THR GENMASK(3, 0) +#define TPS6594_BIT_RECOV_CNT_CLR BIT(4) + +/* FSM_I2C_TRIGGERS register field definition */ +#define TPS6594_BIT_TRIGGER_I2C(bit) BIT(bit) + +/* FSM_NSLEEP_TRIGGERS register field definition */ +#define TPS6594_BIT_NSLEEP1B BIT(0) +#define TPS6594_BIT_NSLEEP2B BIT(1) + +/* BUCK_RESET_REG register field definition */ +#define TPS6594_BIT_BUCKX_RESET(buck_inst) BIT(buck_inst) + +/* SPREAD_SPECTRUM_1 register field definition */ +#define TPS6594_MASK_SS_DEPTH GENMASK(1, 0) +#define TPS6594_BIT_SS_EN BIT(2) + +/* FREQ_SEL register field definition */ +#define TPS6594_BIT_BUCKX_FREQ_SEL(buck_inst) BIT(buck_inst) + +/* FSM_STEP_SIZE register field definition */ +#define TPS6594_MASK_PFSM_DELAY_STEP GENMASK(4, 0) + +/* LDO_RV_TIMEOUT_REG_1 register field definition */ +#define TPS6594_MASK_LDO1_RV_TIMEOUT GENMASK(3, 0) +#define TPS6594_MASK_LDO2_RV_TIMEOUT GENMASK(7, 4) + +/* LDO_RV_TIMEOUT_REG_2 register field definition */ +#define TPS6594_MASK_LDO3_RV_TIMEOUT GENMASK(3, 0) +#define TPS6594_MASK_LDO4_RV_TIMEOUT GENMASK(7, 4) + +/* USER_SPARE_REGS register field definition */ +#define TPS6594_BIT_USER_SPARE(bit) BIT(bit) + +/* ESM_MCU_START_REG register field definition */ +#define TPS6594_BIT_ESM_MCU_START BIT(0) + +/* ESM_MCU_MODE_CFG register field definition */ +#define TPS6594_MASK_ESM_MCU_ERR_CNT_TH GENMASK(3, 0) +#define TPS6594_BIT_ESM_MCU_ENDRV BIT(5) +#define TPS6594_BIT_ESM_MCU_EN BIT(6) +#define TPS6594_BIT_ESM_MCU_MODE BIT(7) + +/* ESM_MCU_ERR_CNT_REG register field definition */ +#define TPS6594_MASK_ESM_MCU_ERR_CNT GENMASK(4, 0) + +/* ESM_SOC_START_REG register field definition */ +#define TPS6594_BIT_ESM_SOC_START BIT(0) + +/* ESM_SOC_MODE_CFG register field definition */ +#define TPS6594_MASK_ESM_SOC_ERR_CNT_TH GENMASK(3, 0) +#define TPS6594_BIT_ESM_SOC_ENDRV BIT(5) +#define TPS6594_BIT_ESM_SOC_EN BIT(6) +#define TPS6594_BIT_ESM_SOC_MODE BIT(7) + +/* ESM_SOC_ERR_CNT_REG register field definition */ +#define TPS6594_MASK_ESM_SOC_ERR_CNT GENMASK(4, 0) + +/* REGISTER_LOCK register field definition */ +#define TPS6594_BIT_REGISTER_LOCK_STATUS BIT(0) + +/* VMON_CONF register field definition */ +#define TPS6594_MASK_VMON1_SLEW_RATE GENMASK(2, 0) +#define TPS6594_MASK_VMON2_SLEW_RATE GENMASK(5, 3) + +/* SOFT_REBOOT_REG register field definition */ +#define TPS6594_BIT_SOFT_REBOOT BIT(0) + +/* RTC_SECONDS & ALARM_SECONDS register field definition */ +#define TPS6594_MASK_SECOND_0 GENMASK(3, 0) +#define TPS6594_MASK_SECOND_1 GENMASK(6, 4) + +/* RTC_MINUTES & ALARM_MINUTES register field definition */ +#define TPS6594_MASK_MINUTE_0 GENMASK(3, 0) +#define TPS6594_MASK_MINUTE_1 GENMASK(6, 4) + +/* RTC_HOURS & ALARM_HOURS register field definition */ +#define TPS6594_MASK_HOUR_0 GENMASK(3, 0) +#define TPS6594_MASK_HOUR_1 GENMASK(5, 4) +#define TPS6594_BIT_PM_NAM BIT(7) + +/* RTC_DAYS & ALARM_DAYS register field definition */ +#define TPS6594_MASK_DAY_0 GENMASK(3, 0) +#define TPS6594_MASK_DAY_1 GENMASK(5, 4) + +/* RTC_MONTHS & ALARM_MONTHS register field definition */ +#define TPS6594_MASK_MONTH_0 GENMASK(3, 0) +#define TPS6594_BIT_MONTH_1 BIT(4) + +/* RTC_YEARS & ALARM_YEARS register field definition */ +#define TPS6594_MASK_YEAR_0 GENMASK(3, 0) +#define TPS6594_MASK_YEAR_1 GENMASK(7, 4) + +/* RTC_WEEKS register field definition */ +#define TPS6594_MASK_WEEK GENMASK(2, 0) + +/* RTC_CTRL_1 register field definition */ +#define TPS6594_BIT_STOP_RTC BIT(0) +#define TPS6594_BIT_ROUND_30S BIT(1) +#define TPS6594_BIT_AUTO_COMP BIT(2) +#define TPS6594_BIT_MODE_12_24 BIT(3) +#define TPS6594_BIT_SET_32_COUNTER BIT(5) +#define TPS6594_BIT_GET_TIME BIT(6) +#define TPS6594_BIT_RTC_V_OPT BIT(7) + +/* RTC_CTRL_2 register field definition */ +#define TPS6594_BIT_XTAL_EN BIT(0) +#define TPS6594_MASK_XTAL_SEL GENMASK(2, 1) +#define TPS6594_BIT_LP_STANDBY_SEL BIT(3) +#define TPS6594_BIT_FAST_BIST BIT(4) +#define TPS6594_MASK_STARTUP_DEST GENMASK(6, 5) +#define TPS6594_BIT_FIRST_STARTUP_DONE BIT(7) + +/* RTC_STATUS register field definition */ +#define TPS6594_BIT_RUN BIT(1) +#define TPS6594_BIT_TIMER BIT(5) +#define TPS6594_BIT_ALARM BIT(6) +#define TPS6594_BIT_POWER_UP BIT(7) + +/* RTC_INTERRUPTS register field definition */ +#define TPS6594_MASK_EVERY GENMASK(1, 0) +#define TPS6594_BIT_IT_TIMER BIT(2) +#define TPS6594_BIT_IT_ALARM BIT(3) + +/* RTC_RESET_STATUS register field definition */ +#define TPS6594_BIT_RESET_STATUS_RTC BIT(0) + +/* SERIAL_IF_CONFIG register field definition */ +#define TPS6594_BIT_I2C_SPI_SEL BIT(0) +#define TPS6594_BIT_I2C1_SPI_CRC_EN BIT(1) +#define TPS6594_BIT_I2C2_CRC_EN BIT(2) +#define TPS6594_MASK_T_CRC GENMASK(7, 3) + +/* WD_QUESTION_ANSW_CNT register field definition */ +#define TPS6594_MASK_WD_QUESTION GENMASK(3, 0) +#define TPS6594_MASK_WD_ANSW_CNT GENMASK(5, 4) + +/* WD_MODE_REG register field definition */ +#define TPS6594_BIT_WD_RETURN_LONGWIN BIT(0) +#define TPS6594_BIT_WD_MODE_SELECT BIT(1) +#define TPS6594_BIT_WD_PWRHOLD BIT(2) + +/* WD_QA_CFG register field definition */ +#define TPS6594_MASK_WD_QUESTION_SEED GENMASK(3, 0) +#define TPS6594_MASK_WD_QA_LFSR GENMASK(5, 4) +#define TPS6594_MASK_WD_QA_FDBK GENMASK(7, 6) + +/* WD_ERR_STATUS register field definition */ +#define TPS6594_BIT_WD_LONGWIN_TIMEOUT_INT BIT(0) +#define TPS6594_BIT_WD_TIMEOUT BIT(1) +#define TPS6594_BIT_WD_TRIG_EARLY BIT(2) +#define TPS6594_BIT_WD_ANSW_EARLY BIT(3) +#define TPS6594_BIT_WD_SEQ_ERR BIT(4) +#define TPS6594_BIT_WD_ANSW_ERR BIT(5) +#define TPS6594_BIT_WD_FAIL_INT BIT(6) +#define TPS6594_BIT_WD_RST_INT BIT(7) + +/* WD_THR_CFG register field definition */ +#define TPS6594_MASK_WD_RST_TH GENMASK(2, 0) +#define TPS6594_MASK_WD_FAIL_TH GENMASK(5, 3) +#define TPS6594_BIT_WD_EN BIT(6) +#define TPS6594_BIT_WD_RST_EN BIT(7) + +/* WD_FAIL_CNT_REG register field definition */ +#define TPS6594_MASK_WD_FAIL_CNT GENMASK(3, 0) +#define TPS6594_BIT_WD_FIRST_OK BIT(5) +#define TPS6594_BIT_WD_BAD_EVENT BIT(6) + +/* CRC8 polynomial for I2C & SPI protocols */ +#define TPS6594_CRC8_POLYNOMIAL 0x07 + +/* IRQs */ +enum tps6594_irqs { + /* INT_BUCK1_2 register */ + TPS6594_IRQ_BUCK1_OV, + TPS6594_IRQ_BUCK1_UV, + TPS6594_IRQ_BUCK1_SC, + TPS6594_IRQ_BUCK1_ILIM, + TPS6594_IRQ_BUCK2_OV, + TPS6594_IRQ_BUCK2_UV, + TPS6594_IRQ_BUCK2_SC, + TPS6594_IRQ_BUCK2_ILIM, + /* INT_BUCK3_4 register */ + TPS6594_IRQ_BUCK3_OV, + TPS6594_IRQ_BUCK3_UV, + TPS6594_IRQ_BUCK3_SC, + TPS6594_IRQ_BUCK3_ILIM, + TPS6594_IRQ_BUCK4_OV, + TPS6594_IRQ_BUCK4_UV, + TPS6594_IRQ_BUCK4_SC, + TPS6594_IRQ_BUCK4_ILIM, + /* INT_BUCK5 register */ + TPS6594_IRQ_BUCK5_OV, + TPS6594_IRQ_BUCK5_UV, + TPS6594_IRQ_BUCK5_SC, + TPS6594_IRQ_BUCK5_ILIM, + /* INT_LDO1_2 register */ + TPS6594_IRQ_LDO1_OV, + TPS6594_IRQ_LDO1_UV, + TPS6594_IRQ_LDO1_SC, + TPS6594_IRQ_LDO1_ILIM, + TPS6594_IRQ_LDO2_OV, + TPS6594_IRQ_LDO2_UV, + TPS6594_IRQ_LDO2_SC, + TPS6594_IRQ_LDO2_ILIM, + /* INT_LDO3_4 register */ + TPS6594_IRQ_LDO3_OV, + TPS6594_IRQ_LDO3_UV, + TPS6594_IRQ_LDO3_SC, + TPS6594_IRQ_LDO3_ILIM, + TPS6594_IRQ_LDO4_OV, + TPS6594_IRQ_LDO4_UV, + TPS6594_IRQ_LDO4_SC, + TPS6594_IRQ_LDO4_ILIM, + /* INT_VMON register */ + TPS6594_IRQ_VCCA_OV, + TPS6594_IRQ_VCCA_UV, + TPS6594_IRQ_VMON1_OV, + TPS6594_IRQ_VMON1_UV, + TPS6594_IRQ_VMON1_RV, + TPS6594_IRQ_VMON2_OV, + TPS6594_IRQ_VMON2_UV, + TPS6594_IRQ_VMON2_RV, + /* INT_GPIO register */ + TPS6594_IRQ_GPIO9, + TPS6594_IRQ_GPIO10, + TPS6594_IRQ_GPIO11, + /* INT_GPIO1_8 register */ + TPS6594_IRQ_GPIO1, + TPS6594_IRQ_GPIO2, + TPS6594_IRQ_GPIO3, + TPS6594_IRQ_GPIO4, + TPS6594_IRQ_GPIO5, + TPS6594_IRQ_GPIO6, + TPS6594_IRQ_GPIO7, + TPS6594_IRQ_GPIO8, + /* INT_STARTUP register */ + TPS6594_IRQ_NPWRON_START, + TPS6594_IRQ_ENABLE, + TPS6594_IRQ_FSD, + TPS6594_IRQ_SOFT_REBOOT, + /* INT_MISC register */ + TPS6594_IRQ_BIST_PASS, + TPS6594_IRQ_EXT_CLK, + TPS6594_IRQ_TWARN, + /* INT_MODERATE_ERR register */ + TPS6594_IRQ_TSD_ORD, + TPS6594_IRQ_BIST_FAIL, + TPS6594_IRQ_REG_CRC_ERR, + TPS6594_IRQ_RECOV_CNT, + TPS6594_IRQ_SPMI_ERR, + TPS6594_IRQ_NPWRON_LONG, + TPS6594_IRQ_NINT_READBACK, + TPS6594_IRQ_NRSTOUT_READBACK, + /* INT_SEVERE_ERR register */ + TPS6594_IRQ_TSD_IMM, + TPS6594_IRQ_VCCA_OVP, + TPS6594_IRQ_PFSM_ERR, + /* INT_FSM_ERR register */ + TPS6594_IRQ_IMM_SHUTDOWN, + TPS6594_IRQ_ORD_SHUTDOWN, + TPS6594_IRQ_MCU_PWR_ERR, + TPS6594_IRQ_SOC_PWR_ERR, + /* INT_COMM_ERR register */ + TPS6594_IRQ_COMM_FRM_ERR, + TPS6594_IRQ_COMM_CRC_ERR, + TPS6594_IRQ_COMM_ADR_ERR, + TPS6594_IRQ_I2C2_CRC_ERR, + TPS6594_IRQ_I2C2_ADR_ERR, + /* INT_READBACK_ERR register */ + TPS6594_IRQ_EN_DRV_READBACK, + TPS6594_IRQ_NRSTOUT_SOC_READBACK, + /* INT_ESM register */ + TPS6594_IRQ_ESM_SOC_PIN, + TPS6594_IRQ_ESM_SOC_FAIL, + TPS6594_IRQ_ESM_SOC_RST, + /* RTC_STATUS register */ + TPS6594_IRQ_TIMER, + TPS6594_IRQ_ALARM, + TPS6594_IRQ_POWER_UP, +}; + +#define TPS6594_IRQ_NAME_BUCK1_OV "buck1_ov" +#define TPS6594_IRQ_NAME_BUCK1_UV "buck1_uv" +#define TPS6594_IRQ_NAME_BUCK1_SC "buck1_sc" +#define TPS6594_IRQ_NAME_BUCK1_ILIM "buck1_ilim" +#define TPS6594_IRQ_NAME_BUCK2_OV "buck2_ov" +#define TPS6594_IRQ_NAME_BUCK2_UV "buck2_uv" +#define TPS6594_IRQ_NAME_BUCK2_SC "buck2_sc" +#define TPS6594_IRQ_NAME_BUCK2_ILIM "buck2_ilim" +#define TPS6594_IRQ_NAME_BUCK3_OV "buck3_ov" +#define TPS6594_IRQ_NAME_BUCK3_UV "buck3_uv" +#define TPS6594_IRQ_NAME_BUCK3_SC "buck3_sc" +#define TPS6594_IRQ_NAME_BUCK3_ILIM "buck3_ilim" +#define TPS6594_IRQ_NAME_BUCK4_OV "buck4_ov" +#define TPS6594_IRQ_NAME_BUCK4_UV "buck4_uv" +#define TPS6594_IRQ_NAME_BUCK4_SC "buck4_sc" +#define TPS6594_IRQ_NAME_BUCK4_ILIM "buck4_ilim" +#define TPS6594_IRQ_NAME_BUCK5_OV "buck5_ov" +#define TPS6594_IRQ_NAME_BUCK5_UV "buck5_uv" +#define TPS6594_IRQ_NAME_BUCK5_SC "buck5_sc" +#define TPS6594_IRQ_NAME_BUCK5_ILIM "buck5_ilim" +#define TPS6594_IRQ_NAME_LDO1_OV "ldo1_ov" +#define TPS6594_IRQ_NAME_LDO1_UV "ldo1_uv" +#define TPS6594_IRQ_NAME_LDO1_SC "ldo1_sc" +#define TPS6594_IRQ_NAME_LDO1_ILIM "ldo1_ilim" +#define TPS6594_IRQ_NAME_LDO2_OV "ldo2_ov" +#define TPS6594_IRQ_NAME_LDO2_UV "ldo2_uv" +#define TPS6594_IRQ_NAME_LDO2_SC "ldo2_sc" +#define TPS6594_IRQ_NAME_LDO2_ILIM "ldo2_ilim" +#define TPS6594_IRQ_NAME_LDO3_OV "ldo3_ov" +#define TPS6594_IRQ_NAME_LDO3_UV "ldo3_uv" +#define TPS6594_IRQ_NAME_LDO3_SC "ldo3_sc" +#define TPS6594_IRQ_NAME_LDO3_ILIM "ldo3_ilim" +#define TPS6594_IRQ_NAME_LDO4_OV "ldo4_ov" +#define TPS6594_IRQ_NAME_LDO4_UV "ldo4_uv" +#define TPS6594_IRQ_NAME_LDO4_SC "ldo4_sc" +#define TPS6594_IRQ_NAME_LDO4_ILIM "ldo4_ilim" +#define TPS6594_IRQ_NAME_VCCA_OV "vcca_ov" +#define TPS6594_IRQ_NAME_VCCA_UV "vcca_uv" +#define TPS6594_IRQ_NAME_VMON1_OV "vmon1_ov" +#define TPS6594_IRQ_NAME_VMON1_UV "vmon1_uv" +#define TPS6594_IRQ_NAME_VMON1_RV "vmon1_rv" +#define TPS6594_IRQ_NAME_VMON2_OV "vmon2_ov" +#define TPS6594_IRQ_NAME_VMON2_UV "vmon2_uv" +#define TPS6594_IRQ_NAME_VMON2_RV "vmon2_rv" +#define TPS6594_IRQ_NAME_GPIO9 "gpio9" +#define TPS6594_IRQ_NAME_GPIO10 "gpio10" +#define TPS6594_IRQ_NAME_GPIO11 "gpio11" +#define TPS6594_IRQ_NAME_GPIO1 "gpio1" +#define TPS6594_IRQ_NAME_GPIO2 "gpio2" +#define TPS6594_IRQ_NAME_GPIO3 "gpio3" +#define TPS6594_IRQ_NAME_GPIO4 "gpio4" +#define TPS6594_IRQ_NAME_GPIO5 "gpio5" +#define TPS6594_IRQ_NAME_GPIO6 "gpio6" +#define TPS6594_IRQ_NAME_GPIO7 "gpio7" +#define TPS6594_IRQ_NAME_GPIO8 "gpio8" +#define TPS6594_IRQ_NAME_NPWRON_START "npwron_start" +#define TPS6594_IRQ_NAME_ENABLE "enable" +#define TPS6594_IRQ_NAME_FSD "fsd" +#define TPS6594_IRQ_NAME_SOFT_REBOOT "soft_reboot" +#define TPS6594_IRQ_NAME_BIST_PASS "bist_pass" +#define TPS6594_IRQ_NAME_EXT_CLK "ext_clk" +#define TPS6594_IRQ_NAME_TWARN "twarn" +#define TPS6594_IRQ_NAME_TSD_ORD "tsd_ord" +#define TPS6594_IRQ_NAME_BIST_FAIL "bist_fail" +#define TPS6594_IRQ_NAME_REG_CRC_ERR "reg_crc_err" +#define TPS6594_IRQ_NAME_RECOV_CNT "recov_cnt" +#define TPS6594_IRQ_NAME_SPMI_ERR "spmi_err" +#define TPS6594_IRQ_NAME_NPWRON_LONG "npwron_long" +#define TPS6594_IRQ_NAME_NINT_READBACK "nint_readback" +#define TPS6594_IRQ_NAME_NRSTOUT_READBACK "nrstout_readback" +#define TPS6594_IRQ_NAME_TSD_IMM "tsd_imm" +#define TPS6594_IRQ_NAME_VCCA_OVP "vcca_ovp" +#define TPS6594_IRQ_NAME_PFSM_ERR "pfsm_err" +#define TPS6594_IRQ_NAME_IMM_SHUTDOWN "imm_shutdown" +#define TPS6594_IRQ_NAME_ORD_SHUTDOWN "ord_shutdown" +#define TPS6594_IRQ_NAME_MCU_PWR_ERR "mcu_pwr_err" +#define TPS6594_IRQ_NAME_SOC_PWR_ERR "soc_pwr_err" +#define TPS6594_IRQ_NAME_COMM_FRM_ERR "comm_frm_err" +#define TPS6594_IRQ_NAME_COMM_CRC_ERR "comm_crc_err" +#define TPS6594_IRQ_NAME_COMM_ADR_ERR "comm_adr_err" +#define TPS6594_IRQ_NAME_EN_DRV_READBACK "en_drv_readback" +#define TPS6594_IRQ_NAME_NRSTOUT_SOC_READBACK "nrstout_soc_readback" +#define TPS6594_IRQ_NAME_ESM_SOC_PIN "esm_soc_pin" +#define TPS6594_IRQ_NAME_ESM_SOC_FAIL "esm_soc_fail" +#define TPS6594_IRQ_NAME_ESM_SOC_RST "esm_soc_rst" +#define TPS6594_IRQ_NAME_TIMER "timer" +#define TPS6594_IRQ_NAME_ALARM "alarm" +#define TPS6594_IRQ_NAME_POWERUP "powerup" + +/** + * struct tps6594 - device private data structure + * + * @dev: MFD parent device + * @chip_id: chip ID + * @reg: I2C slave address or SPI chip select number + * @use_crc: if true, use CRC for I2C and SPI interface protocols + * @regmap: regmap for accessing the device registers + * @irq: irq generated by the device + * @irq_data: regmap irq data used for the irq chip + */ +struct tps6594 { + struct device *dev; + unsigned long chip_id; + unsigned short reg; + bool use_crc; + struct regmap *regmap; + int irq; + struct regmap_irq_chip_data *irq_data; +}; + +bool tps6594_is_volatile_reg(struct device *dev, unsigned int reg); +int tps6594_device_init(struct tps6594 *tps, bool enable_crc); + +#endif /* __LINUX_MFD_TPS6594_H */ -- cgit v1.2.3 From e3d9387f002612093dbeaa272f7930ce5108033f Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Thu, 20 Apr 2023 19:17:13 +0200 Subject: security, lsm: Introduce security_mptcp_add_subflow() MPTCP can create subflows in kernel context, and later indirectly expose them to user-space, via the owning MPTCP socket. As discussed in the reported link, the above causes unexpected failures for server, MPTCP-enabled applications. Let's introduce a new LSM hook to allow the security module to relabel the subflow according to the owning user-space process, via the MPTCP socket owning the subflow. Note that the new hook requires both the MPTCP socket and the new subflow. This could allow future extensions, e.g. explicitly validating the MPTCP <-> subflow linkage. Link: https://lore.kernel.org/mptcp/CAHC9VhTNh-YwiyTds=P1e3rixEDqbRTFj22bpya=+qJqfcaMfg@mail.gmail.com/ Signed-off-by: Paolo Abeni Acked-by: Matthieu Baerts Signed-off-by: Matthieu Baerts Signed-off-by: Paul Moore --- include/linux/lsm_hook_defs.h | 1 + include/linux/security.h | 6 ++++++ 2 files changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 6bb55e61e8e8..7308a1a7599b 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -343,6 +343,7 @@ LSM_HOOK(void, LSM_RET_VOID, sctp_sk_clone, struct sctp_association *asoc, struct sock *sk, struct sock *newsk) LSM_HOOK(int, 0, sctp_assoc_established, struct sctp_association *asoc, struct sk_buff *skb) +LSM_HOOK(int, 0, mptcp_add_subflow, struct sock *sk, struct sock *ssk) #endif /* CONFIG_SECURITY_NETWORK */ #ifdef CONFIG_SECURITY_INFINIBAND diff --git a/include/linux/security.h b/include/linux/security.h index e2734e9e44d5..32828502f09e 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -1465,6 +1465,7 @@ void security_sctp_sk_clone(struct sctp_association *asoc, struct sock *sk, struct sock *newsk); int security_sctp_assoc_established(struct sctp_association *asoc, struct sk_buff *skb); +int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk); #else /* CONFIG_SECURITY_NETWORK */ static inline int security_unix_stream_connect(struct sock *sock, @@ -1692,6 +1693,11 @@ static inline int security_sctp_assoc_established(struct sctp_association *asoc, { return 0; } + +static inline int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk) +{ + return 0; +} #endif /* CONFIG_SECURITY_NETWORK */ #ifdef CONFIG_SECURITY_INFINIBAND -- cgit v1.2.3 From fc522f3bdf43efa75b54775978b6b6c19d0d997d Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Thu, 11 May 2023 11:52:42 -0700 Subject: Input: libps2 - remove special handling of ACK for command byte When getting unexpected data while waiting for an acknowledgement it does not matter what command phase is currently executed, and ps2_handle_ack() should indicate that no further processing is needed for the received data byte. Remove PS2_FLAG_ACK_CMD and associated handling. Note that while it is possible to make ps2_handle_ack (and ps2_handle_repsonse) return void, it will be done when the code will be converted to common PS/2 interrupt handler later. Reviewed-by: Raul E Rangel Link: https://lore.kernel.org/r/20230511185252.386941-3-dmitry.torokhov@gmail.com Signed-off-by: Dmitry Torokhov --- include/linux/libps2.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/libps2.h b/include/linux/libps2.h index 53f7e4d0f4b7..193dd53ad18b 100644 --- a/include/linux/libps2.h +++ b/include/linux/libps2.h @@ -28,7 +28,6 @@ #define PS2_FLAG_CMD1 BIT(2) /* Waiting for the first byte of command response */ #define PS2_FLAG_WAITID BIT(3) /* Command executing is GET ID */ #define PS2_FLAG_NAK BIT(4) /* Last transmission was NAKed */ -#define PS2_FLAG_ACK_CMD BIT(5) /* Waiting to ACK the command (first) byte */ struct ps2dev { struct serio *serio; -- cgit v1.2.3 From c4c7eac8ee78d896635ce05d7a1c3f813fcbe24c Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Mon, 15 May 2023 16:14:29 -0700 Subject: Input: libps2 - introduce common interrupt handler Instead of exposing inner workings of libps2 to drivers such as atkbd and psmouse, have them define pre-receive and receive callbacks, and provide a common handler that can be used with underlying serio port. While at this add kerneldoc to the module. Link: https://lore.kernel.org/r/ZGK81cxqjr/KS1kA@google.com Signed-off-by: Dmitry Torokhov --- include/linux/libps2.h | 61 +++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/libps2.h b/include/linux/libps2.h index 193dd53ad18b..9ca9ce4e6e64 100644 --- a/include/linux/libps2.h +++ b/include/linux/libps2.h @@ -8,43 +8,59 @@ */ #include +#include #include #include #include -#define PS2_CMD_SETSCALE11 0x00e6 -#define PS2_CMD_SETRES 0x10e8 -#define PS2_CMD_GETID 0x02f2 -#define PS2_CMD_RESET_BAT 0x02ff +struct ps2dev; -#define PS2_RET_BAT 0xaa -#define PS2_RET_ID 0x00 -#define PS2_RET_ACK 0xfa -#define PS2_RET_NAK 0xfe -#define PS2_RET_ERR 0xfc +/** + * enum ps2_disposition - indicates how received byte should be handled + * @PS2_PROCESS: pass to the main protocol handler, process normally + * @PS2_IGNORE: skip the byte + * @PS2_ERROR: do not process the byte, abort command in progress + */ +enum ps2_disposition { + PS2_PROCESS, + PS2_IGNORE, + PS2_ERROR, +}; -#define PS2_FLAG_ACK BIT(0) /* Waiting for ACK/NAK */ -#define PS2_FLAG_CMD BIT(1) /* Waiting for a command to finish */ -#define PS2_FLAG_CMD1 BIT(2) /* Waiting for the first byte of command response */ -#define PS2_FLAG_WAITID BIT(3) /* Command executing is GET ID */ -#define PS2_FLAG_NAK BIT(4) /* Last transmission was NAKed */ +typedef enum ps2_disposition (*ps2_pre_receive_handler_t)(struct ps2dev *, u8, + unsigned int); +typedef void (*ps2_receive_handler_t)(struct ps2dev *, u8); +/** + * struct ps2dev - represents a device using PS/2 protocol + * @serio: a serio port used by the PS/2 device + * @cmd_mutex: a mutex ensuring that only one command is executing at a time + * @wait: a waitqueue used to signal completion from the serio interrupt handler + * @flags: various internal flags indicating stages of PS/2 command execution + * @cmdbuf: buffer holding command response + * @cmdcnt: outstanding number of bytes of the command response + * @nak: a byte transmitted by the device when it refuses command + * @pre_receive_handler: checks communication errors and returns disposition + * (&enum ps2_disposition) of the received data byte + * @receive_handler: main handler of particular PS/2 protocol, such as keyboard + * or mouse protocol + */ struct ps2dev { struct serio *serio; - - /* Ensures that only one command is executing at a time */ struct mutex cmd_mutex; - - /* Used to signal completion from interrupt handler */ wait_queue_head_t wait; - unsigned long flags; u8 cmdbuf[8]; u8 cmdcnt; u8 nak; + + ps2_pre_receive_handler_t pre_receive_handler; + ps2_receive_handler_t receive_handler; }; -void ps2_init(struct ps2dev *ps2dev, struct serio *serio); +void ps2_init(struct ps2dev *ps2dev, struct serio *serio, + ps2_pre_receive_handler_t pre_receive_handler, + ps2_receive_handler_t receive_handler); int ps2_sendbyte(struct ps2dev *ps2dev, u8 byte, unsigned int timeout); void ps2_drain(struct ps2dev *ps2dev, size_t maxbytes, unsigned int timeout); void ps2_begin_command(struct ps2dev *ps2dev); @@ -52,9 +68,8 @@ void ps2_end_command(struct ps2dev *ps2dev); int __ps2_command(struct ps2dev *ps2dev, u8 *param, unsigned int command); int ps2_command(struct ps2dev *ps2dev, u8 *param, unsigned int command); int ps2_sliced_command(struct ps2dev *ps2dev, u8 command); -bool ps2_handle_ack(struct ps2dev *ps2dev, u8 data); -bool ps2_handle_response(struct ps2dev *ps2dev, u8 data); -void ps2_cmd_aborted(struct ps2dev *ps2dev); bool ps2_is_keyboard_id(u8 id); +irqreturn_t ps2_interrupt(struct serio *serio, u8 data, unsigned int flags); + #endif /* _LIBPS2_H */ -- cgit v1.2.3 From 8a3e82d38674066f4cbed3588b78b0d9b8b15ed7 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 21:35:42 +0200 Subject: x86/hibernate: Declare global functions in suspend.h Three functions that are defined in x86 specific code to override generic __weak implementations cause a warning because of a missing prototype: arch/x86/power/cpu.c:298:5: error: no previous prototype for 'hibernate_resume_nonboot_cpu_disable' [-Werror=missing-prototypes] arch/x86/power/hibernate.c:129:5: error: no previous prototype for 'arch_hibernation_header_restore' [-Werror=missing-prototypes] arch/x86/power/hibernate.c:91:5: error: no previous prototype for 'arch_hibernation_header_save' [-Werror=missing-prototypes] Move the declarations into a global header so it can be included by any file defining one of these. Signed-off-by: Arnd Bergmann Signed-off-by: Dave Hansen Reviewed-by: Alexander Lobakin Link: https://lore.kernel.org/all/20230516193549.544673-14-arnd%40kernel.org --- include/linux/suspend.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index d0d4598a7b3f..f16653f7be32 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -452,6 +452,10 @@ extern struct pbe *restore_pblist; int pfn_is_nosave(unsigned long pfn); int hibernate_quiet_exec(int (*func)(void *data), void *data); +int hibernate_resume_nonboot_cpu_disable(void); +int arch_hibernation_header_save(void *addr, unsigned int max_size); +int arch_hibernation_header_restore(void *addr); + #else /* CONFIG_HIBERNATION */ static inline void register_nosave_region(unsigned long b, unsigned long e) {} static inline int swsusp_page_is_forbidden(struct page *p) { return 0; } -- cgit v1.2.3 From 4d312ac057da57b4a844ec8af14236e74b652efe Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 21:35:47 +0200 Subject: x86/mm: Add early_memremap_pgprot_adjust() prototype early_memremap_pgprot_adjust() is a __weak function with a local prototype, but x86 has a custom implementation that does not see the prototype, causing a W=1 warning: arch/x86/mm/ioremap.c:785:17: error: no previous prototype for 'early_memremap_pgprot_adjust' [-Werror=missing-prototypes] Move the declaration into the global linux/io.h header to avoid this. Signed-off-by: Arnd Bergmann Signed-off-by: Dave Hansen Reviewed-by: Alexander Lobakin Link: https://lore.kernel.org/all/20230516193549.544673-19-arnd%40kernel.org --- include/linux/io.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/io.h b/include/linux/io.h index 308f4f0cfb93..7304f2a69960 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -68,6 +68,11 @@ void *devm_memremap(struct device *dev, resource_size_t offset, size_t size, unsigned long flags); void devm_memunmap(struct device *dev, void *addr); +/* architectures can override this */ +pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr, + unsigned long size, pgprot_t prot); + + #ifdef CONFIG_PCI /* * The PCI specifications (Rev 3.0, 3.2.5 "Transaction Ordering and -- cgit v1.2.3 From 454a348714954f7b626c027a90c3967278e3f93b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 21:35:49 +0200 Subject: x86/platform: Avoid missing-prototype warnings for OLPC There are two functions in the olpc platform that have no prototype: arch/x86/platform/olpc/olpc_dt.c:237:13: error: no previous prototype for 'olpc_dt_fixup' [-Werror=missing-prototypes] arch/x86/platform/olpc/olpc-xo1-pm.c:73:26: error: no previous prototype for 'xo1_do_sleep' [-Werror=missing-prototypes] The first one should just be marked 'static' as there are no other callers, while the second one is called from assembler and is just a false-positive warning that can be silenced by adding a prototype. Signed-off-by: Arnd Bergmann Signed-off-by: Dave Hansen Reviewed-by: Alexander Lobakin Link: https://lore.kernel.org/all/20230516193549.544673-21-arnd%40kernel.org --- include/linux/olpc-ec.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/olpc-ec.h b/include/linux/olpc-ec.h index c4602364e909..3c2891d85c41 100644 --- a/include/linux/olpc-ec.h +++ b/include/linux/olpc-ec.h @@ -56,6 +56,8 @@ extern int olpc_ec_sci_query(u16 *sci_value); extern bool olpc_ec_wakeup_available(void); +asmlinkage int xo1_do_sleep(u8 sleep_state); + #else static inline int olpc_ec_cmd(u8 cmd, u8 *inbuf, size_t inlen, u8 *outbuf, -- cgit v1.2.3 From fdcab6cddef24a26b86d798814b3c25057e53c21 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 18 May 2023 07:31:00 +0200 Subject: blk-mq: remove RQF_ELVPRIV RQF_ELVPRIV is set for all non-flush requests that have RQF_ELV set. Expand this condition in the two users of the flag and remove it. Signed-off-by: Christoph Hellwig Reviewed-by: Ming Lei Reviewed-by: Bart Van Assche Link: https://lore.kernel.org/r/20230518053101.760632-3-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 06caacd77ed6..5529e7d28ae6 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -42,8 +42,6 @@ typedef __u32 __bitwise req_flags_t; #define RQF_FAILED ((__force req_flags_t)(1 << 10)) /* don't warn about errors */ #define RQF_QUIET ((__force req_flags_t)(1 << 11)) -/* elevator private data attached */ -#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12)) /* account into disk and partition IO statistics */ #define RQF_IO_STAT ((__force req_flags_t)(1 << 13)) /* runtime pm request */ -- cgit v1.2.3 From dd6216bb16e83e349d5d987227328031b0b0d30d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 18 May 2023 07:31:01 +0200 Subject: blk-mq: make sure elevator callbacks aren't called for passthrough request In case of q->elevator, passthrough request can still be marked as RQF_ELV, so some elevator callbacks will be called for them. Fix this by splitting RQF_SCHED_TAGS, which is set for all requests that are issued on a queue that uses an I/O scheduler, and RQF_USE_SCHED for non-flush, non-passthrough requests on such a queue. Roughly based on two different patches from Ming Lei . Signed-off-by: Christoph Hellwig Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20230518053101.760632-4-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 5529e7d28ae6..e4a211957db6 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -38,6 +38,10 @@ typedef __u32 __bitwise req_flags_t; #define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6)) /* don't call prep for this one */ #define RQF_DONTPREP ((__force req_flags_t)(1 << 7)) +/* use hctx->sched_tags */ +#define RQF_SCHED_TAGS ((__force req_flags_t)(1 << 8)) +/* use an I/O scheduler for this request */ +#define RQF_USE_SCHED ((__force req_flags_t)(1 << 9)) /* vaguely specified driver internal error. Ignored by the block layer */ #define RQF_FAILED ((__force req_flags_t)(1 << 10)) /* don't warn about errors */ @@ -57,9 +61,7 @@ typedef __u32 __bitwise req_flags_t; #define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19)) /* ->timeout has been called, don't expire again */ #define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21)) -/* queue has elevator attached */ -#define RQF_ELV ((__force req_flags_t)(1 << 22)) -#define RQF_RESV ((__force req_flags_t)(1 << 23)) +#define RQF_RESV ((__force req_flags_t)(1 << 23)) /* flags that prevent us from merging requests: */ #define RQF_NOMERGE_FLAGS \ @@ -842,7 +844,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *ib); */ static inline bool blk_mq_need_time_stamp(struct request *rq) { - return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV)); + return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED)); } static inline bool blk_mq_is_reserved_rq(struct request *rq) @@ -858,7 +860,7 @@ static inline bool blk_mq_add_to_batch(struct request *req, struct io_comp_batch *iob, int ioerror, void (*complete)(struct io_comp_batch *)) { - if (!iob || (req->rq_flags & RQF_ELV) || ioerror || + if (!iob || (req->rq_flags & RQF_USE_SCHED) || ioerror || (req->end_io && !blk_rq_is_passthrough(req))) return false; -- cgit v1.2.3 From 3ddbe2a7e0d4a155a805f69c906c9beed30d4cc4 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 17 May 2023 10:42:21 -0700 Subject: block: Fix the type of the second bdev_op_is_zoned_write() argument Change the type of the second argument of bdev_op_is_zoned_write() from blk_opf_t into enum req_op because this function expects an operation without flags as second argument. Reviewed-by: Johannes Thumshirn Reviewed-by: Pankaj Raghav Reviewed-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Hannes Reinecke Cc: Ming Lei Fixes: 8cafdb5ab94c ("block: adapt blk_mq_plug() to not plug for writes that require a zone lock") Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20230517174230.897144-4-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b441e633f4dd..db24cf98ccfb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1282,7 +1282,7 @@ static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) } static inline bool bdev_op_is_zoned_write(struct block_device *bdev, - blk_opf_t op) + enum req_op op) { if (!bdev_is_zoned(bdev)) return false; -- cgit v1.2.3 From a370798201b537f78288e4ef5e0f7fc70889e7ee Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 17 May 2023 10:42:22 -0700 Subject: block: Introduce op_needs_zoned_write_locking() Introduce a helper function for checking whether write serialization is required if the operation will be sent to a zoned device. A second caller for op_needs_zoned_write_locking() will be introduced in the next patch in this series. Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Cc: Damien Le Moal Cc: Ming Lei Signed-off-by: Bart Van Assche Reviewed-by: Damien Le Moal Link: https://lore.kernel.org/r/20230517174230.897144-5-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index db24cf98ccfb..3952c52d6cd1 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1281,13 +1281,16 @@ static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec) return disk_zone_no(bdev->bd_disk, sec); } +/* Whether write serialization is required for @op on zoned devices. */ +static inline bool op_needs_zoned_write_locking(enum req_op op) +{ + return op == REQ_OP_WRITE || op == REQ_OP_WRITE_ZEROES; +} + static inline bool bdev_op_is_zoned_write(struct block_device *bdev, enum req_op op) { - if (!bdev_is_zoned(bdev)) - return false; - - return op == REQ_OP_WRITE || op == REQ_OP_WRITE_ZEROES; + return bdev_is_zoned(bdev) && op_needs_zoned_write_locking(op); } static inline sector_t bdev_zone_sectors(struct block_device *bdev) -- cgit v1.2.3 From 19821fee3ed42e5b294e95814892d0ad6a9890c9 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 17 May 2023 10:42:23 -0700 Subject: block: Introduce blk_rq_is_seq_zoned_write() Introduce the function blk_rq_is_seq_zoned_write(). This function will be used in later patches to preserve the order of zoned writes that require write serialization. This patch includes an optimization: instead of using rq->q->disk->part0->bd_queue to check whether or not the queue is associated with a zoned block device, use rq->q->disk->queue. Cc: Christoph Hellwig Cc: Damien Le Moal Cc: Ming Lei Signed-off-by: Bart Van Assche Reviewed-by: Damien Le Moal Link: https://lore.kernel.org/r/20230517174230.897144-6-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index e4a211957db6..49d14b1acfa5 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -1164,6 +1164,18 @@ static inline unsigned int blk_rq_zone_is_seq(struct request *rq) return disk_zone_is_seq(rq->q->disk, blk_rq_pos(rq)); } +/** + * blk_rq_is_seq_zoned_write() - Check if @rq requires write serialization. + * @rq: Request to examine. + * + * Note: REQ_OP_ZONE_APPEND requests do not require serialization. + */ +static inline bool blk_rq_is_seq_zoned_write(struct request *rq) +{ + return op_needs_zoned_write_locking(req_op(rq)) && + blk_rq_zone_is_seq(rq); +} + bool blk_req_needs_zone_write_lock(struct request *rq); bool blk_req_zone_write_trylock(struct request *rq); void __blk_req_zone_write_lock(struct request *rq); @@ -1194,6 +1206,11 @@ static inline bool blk_req_can_dispatch_to_zone(struct request *rq) return !blk_req_zone_is_write_locked(rq); } #else /* CONFIG_BLK_DEV_ZONED */ +static inline bool blk_rq_is_seq_zoned_write(struct request *rq) +{ + return false; +} + static inline bool blk_req_needs_zone_write_lock(struct request *rq) { return false; -- cgit v1.2.3 From dc18582211b34bce8250ddf3cac2a2230e192120 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 17 May 2023 11:38:12 +0100 Subject: net: sfp: add support for setting signalling rate Add support to the SFP layer to allow phylink to set the signalling rate for a SFP module. The rate given will be in units of kilo-baud (1000 baud). Reviewed-by: Simon Horman Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- include/linux/sfp.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sfp.h b/include/linux/sfp.h index ef06a195b3c2..2f66e03e9dbd 100644 --- a/include/linux/sfp.h +++ b/include/linux/sfp.h @@ -556,6 +556,7 @@ int sfp_get_module_eeprom_by_page(struct sfp_bus *bus, struct netlink_ext_ack *extack); void sfp_upstream_start(struct sfp_bus *bus); void sfp_upstream_stop(struct sfp_bus *bus); +void sfp_upstream_set_signal_rate(struct sfp_bus *bus, unsigned int rate_kbd); void sfp_bus_put(struct sfp_bus *bus); struct sfp_bus *sfp_bus_find_fwnode(const struct fwnode_handle *fwnode); int sfp_bus_add_upstream(struct sfp_bus *bus, void *upstream, @@ -615,6 +616,11 @@ static inline void sfp_upstream_stop(struct sfp_bus *bus) { } +static inline void sfp_upstream_set_signal_rate(struct sfp_bus *bus, + unsigned int rate_kbd) +{ +} + static inline void sfp_bus_put(struct sfp_bus *bus) { } -- cgit v1.2.3 From fc082b39d0a29891ab4b54c88a40f42385103f71 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 17 May 2023 11:38:17 +0100 Subject: net: sfp: add support for rate selection Add support for parsing the rate select thresholds and switching of the RS0 and RS1 signals to the transceiver. This is complicated by various revisions of SFF-8472 and interaction of SFF-8431, SFF-8079 and INF-8074. Reviewed-by: Simon Horman Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Signed-off-by: Jakub Kicinski --- include/linux/sfp.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sfp.h b/include/linux/sfp.h index 2f66e03e9dbd..9346cd44814d 100644 --- a/include/linux/sfp.h +++ b/include/linux/sfp.h @@ -342,6 +342,12 @@ enum { SFP_ENCODING = 11, SFP_BR_NOMINAL = 12, SFP_RATE_ID = 13, + SFF_RID_8079 = 0x01, + SFF_RID_8431_RX_ONLY = 0x02, + SFF_RID_8431_TX_ONLY = 0x04, + SFF_RID_8431 = 0x06, + SFF_RID_10G8G = 0x0e, + SFP_LINK_LEN_SM_KM = 14, SFP_LINK_LEN_SM_100M = 15, SFP_LINK_LEN_50UM_OM2_10M = 16, @@ -465,6 +471,7 @@ enum { SFP_STATUS = 110, SFP_STATUS_TX_DISABLE = BIT(7), SFP_STATUS_TX_DISABLE_FORCE = BIT(6), + SFP_STATUS_RS0_SELECT = BIT(3), SFP_STATUS_TX_FAULT = BIT(2), SFP_STATUS_RX_LOS = BIT(1), SFP_ALARM0 = 112, @@ -496,6 +503,7 @@ enum { SFP_WARN1_RXPWR_LOW = BIT(6), SFP_EXT_STATUS = 118, + SFP_EXT_STATUS_RS1_SELECT = BIT(3), SFP_EXT_STATUS_PWRLVL_SELECT = BIT(0), SFP_VSL = 120, -- cgit v1.2.3 From eb1cfd09f788e39948a82be8063e54e40dd018d9 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 9 May 2023 15:58:46 -0400 Subject: lockdep: Add lock_set_cmp_fn() annotation This implements a new interface to lockdep, lock_set_cmp_fn(), for defining a custom ordering when taking multiple locks of the same class. This is an alternative to subclasses, but can not fully replace them since subclasses allow lock hierarchies with other clasees inter-twined, while this relies on pure class nesting. Specifically, if A is our nesting class then: A/0 <- B <- A/1 Would be a valid lock order with subclasses (each subclass really is a full class from the validation PoV) but not with this annotation, which requires all nesting to be consecutive. Example output: | ============================================ | WARNING: possible recursive locking detected | 6.2.0-rc8-00003-g7d81e591ca6a-dirty #15 Not tainted | -------------------------------------------- | kworker/14:3/938 is trying to acquire lock: | ffff8880143218c8 (&b->lock l=0 0:2803368){++++}-{3:3}, at: bch_btree_node_get.part.0+0x81/0x2b0 | | but task is already holding lock: | ffff8880143de8c8 (&b->lock l=1 1048575:9223372036854775807){++++}-{3:3}, at: __bch_btree_map_nodes+0xea/0x1e0 | and the lock comparison function returns 1: | | other info that might help us debug this: | Possible unsafe locking scenario: | | CPU0 | ---- | lock(&b->lock l=1 1048575:9223372036854775807); | lock(&b->lock l=0 0:2803368); | | *** DEADLOCK *** | | May be due to missing lock nesting notation | | 3 locks held by kworker/14:3/938: | #0: ffff888005ea9d38 ((wq_completion)bcache){+.+.}-{0:0}, at: process_one_work+0x1ec/0x530 | #1: ffff8880098c3e70 ((work_completion)(&cl->work)#3){+.+.}-{0:0}, at: process_one_work+0x1ec/0x530 | #2: ffff8880143de8c8 (&b->lock l=1 1048575:9223372036854775807){++++}-{3:3}, at: __bch_btree_map_nodes+0xea/0x1e0 [peterz: extended changelog] Signed-off-by: Kent Overstreet Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20230509195847.1745548-1-kent.overstreet@linux.dev --- include/linux/lockdep.h | 8 ++++++++ include/linux/lockdep_types.h | 8 ++++++++ 2 files changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index b32256e9e944..3bac1501fb58 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -434,6 +434,14 @@ extern int lockdep_is_held(const void *); #endif /* !LOCKDEP */ +#ifdef CONFIG_PROVE_LOCKING +void lockdep_set_lock_cmp_fn(struct lockdep_map *, lock_cmp_fn, lock_print_fn); + +#define lock_set_cmp_fn(lock, ...) lockdep_set_lock_cmp_fn(&(lock)->dep_map, __VA_ARGS__) +#else +#define lock_set_cmp_fn(lock, ...) do { } while (0) +#endif + enum xhlock_context_t { XHLOCK_HARD, XHLOCK_SOFT, diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index d22430840b53..8bf79c4e4873 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -84,6 +84,11 @@ struct lock_trace; #define LOCKSTAT_POINTS 4 +struct lockdep_map; +typedef int (*lock_cmp_fn)(const struct lockdep_map *a, + const struct lockdep_map *b); +typedef void (*lock_print_fn)(const struct lockdep_map *map); + /* * The lock-class itself. The order of the structure members matters. * reinit_class() zeroes the key member and all subsequent members. @@ -109,6 +114,9 @@ struct lock_class { struct list_head locks_after, locks_before; const struct lockdep_subclass_key *key; + lock_cmp_fn cmp_fn; + lock_print_fn print_fn; + unsigned int subclass; unsigned int dep_gen_id; -- cgit v1.2.3 From 4d744ce9d5d7cf0e3ab68d0cf160194da6504eb8 Mon Sep 17 00:00:00 2001 From: James Seo Date: Tue, 9 May 2023 10:55:46 -0700 Subject: err.h: Add missing kerneldocs for error pointer functions Add kerneldocs for ERR_PTR(), PTR_ERR(), PTR_ERR_OR_ZERO(), IS_ERR(), and IS_ERR_OR_NULL(). Doing so will help convert hundreds of mentions of them in existing documentation into automatic cross-references. Also add kerneldocs for IS_ERR_VALUE(). Doing so adds no automatic cross-references, but this macro has a slightly different use case than the functionally similar IS_ERR(), and documenting it may be helpful to readers who encounter it in existing code. ERR_CAST() already has kerneldocs and has not been touched. Signed-off-by: James Seo Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20230509175543.2065835-3-james@equiv.tech --- include/linux/err.h | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) (limited to 'include/linux') diff --git a/include/linux/err.h b/include/linux/err.h index a139c64aef2a..b5d9bb2a2349 100644 --- a/include/linux/err.h +++ b/include/linux/err.h @@ -19,23 +19,54 @@ #ifndef __ASSEMBLY__ +/** + * IS_ERR_VALUE - Detect an error pointer. + * @x: The pointer to check. + * + * Like IS_ERR(), but does not generate a compiler warning if result is unused. + */ #define IS_ERR_VALUE(x) unlikely((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO) +/** + * ERR_PTR - Create an error pointer. + * @error: A negative error code. + * + * Encodes @error into a pointer value. Users should consider the result + * opaque and not assume anything about how the error is encoded. + * + * Return: A pointer with @error encoded within its value. + */ static inline void * __must_check ERR_PTR(long error) { return (void *) error; } +/** + * PTR_ERR - Extract the error code from an error pointer. + * @ptr: An error pointer. + * Return: The error code within @ptr. + */ static inline long __must_check PTR_ERR(__force const void *ptr) { return (long) ptr; } +/** + * IS_ERR - Detect an error pointer. + * @ptr: The pointer to check. + * Return: true if @ptr is an error pointer, false otherwise. + */ static inline bool __must_check IS_ERR(__force const void *ptr) { return IS_ERR_VALUE((unsigned long)ptr); } +/** + * IS_ERR_OR_NULL - Detect an error pointer or a null pointer. + * @ptr: The pointer to check. + * + * Like IS_ERR(), but also returns true for a null pointer. + */ static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) { return unlikely(!ptr) || IS_ERR_VALUE((unsigned long)ptr); @@ -54,6 +85,23 @@ static inline void * __must_check ERR_CAST(__force const void *ptr) return (void *) ptr; } +/** + * PTR_ERR_OR_ZERO - Extract the error code from a pointer if it has one. + * @ptr: A potential error pointer. + * + * Convenience function that can be used inside a function that returns + * an error code to propagate errors received as error pointers. + * For example, ``return PTR_ERR_OR_ZERO(ptr);`` replaces: + * + * .. code-block:: c + * + * if (IS_ERR(ptr)) + * return PTR_ERR(ptr); + * else + * return 0; + * + * Return: The error code within @ptr if it is an error pointer; 0 otherwise. + */ static inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) { if (IS_ERR(ptr)) -- cgit v1.2.3 From be4c427809b0a746aff54dbb8ef663f0184291d0 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 19 May 2023 06:40:47 +0200 Subject: blk-mq: use the I/O scheduler for writes from the flush state machine Send write requests issued by the flush state machine through the normal I/O submission path including the I/O scheduler (if present) so that I/O scheduler policies are applied to writes with the FUA flag set. Separate the I/O scheduler members from the flush members in struct request since now a request may pass through both an I/O scheduler and the flush machinery. Note that the actual flush requests, which have no bio attached to the request still bypass the I/O schedulers. Signed-off-by: Bart Van Assche [hch: rebased] Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://lore.kernel.org/r/20230519044050.107790-5-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 49d14b1acfa5..935201c89743 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -169,25 +169,20 @@ struct request { void *completion_data; }; - /* * Three pointers are available for the IO schedulers, if they need - * more they have to dynamically allocate it. Flush requests are - * never put on the IO scheduler. So let the flush fields share - * space with the elevator data. + * more they have to dynamically allocate it. */ - union { - struct { - struct io_cq *icq; - void *priv[2]; - } elv; - - struct { - unsigned int seq; - struct list_head list; - rq_end_io_fn *saved_end_io; - } flush; - }; + struct { + struct io_cq *icq; + void *priv[2]; + } elv; + + struct { + unsigned int seq; + struct list_head list; + rq_end_io_fn *saved_end_io; + } flush; union { struct __call_single_data csd; -- cgit v1.2.3 From 9a67aa52a42b31ad44220cc218df3b75a5cd5d05 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 19 May 2023 06:40:50 +0200 Subject: blk-mq: don't use the requeue list to queue flush commands Currently both requeues of commands that were already sent to the driver and flush commands submitted from the flush state machine share the same requeue_list struct request_queue, despite requeues doing head insertions and flushes not. Switch to using two separate lists instead. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://lore.kernel.org/r/20230519044050.107790-8-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 4 +--- include/linux/blkdev.h | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 935201c89743..d778cb6b2112 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -28,8 +28,6 @@ typedef __u32 __bitwise req_flags_t; /* drive already may have started this one */ #define RQF_STARTED ((__force req_flags_t)(1 << 1)) -/* may not be passed by ioscheduler */ -#define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3)) /* request for flush sequence */ #define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4)) /* merge of different types, fail separately */ @@ -65,7 +63,7 @@ typedef __u32 __bitwise req_flags_t; /* flags that prevent us from merging requests: */ #define RQF_NOMERGE_FLAGS \ - (RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD) + (RQF_STARTED | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD) enum mq_rq_state { MQ_RQ_IDLE = 0, diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 3952c52d6cd1..fe99948688df 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -487,6 +487,7 @@ struct request_queue { * for flush operations */ struct blk_flush_queue *fq; + struct list_head flush_list; struct list_head requeue_list; spinlock_t requeue_lock; -- cgit v1.2.3 From e924e80ee6a39bc28d2ef8f51e19d336a98e3be0 Mon Sep 17 00:00:00 2001 From: Aditi Ghag Date: Fri, 19 May 2023 22:51:54 +0000 Subject: bpf: Add kfunc filter function to 'struct btf_kfunc_id_set' This commit adds the ability to filter kfuncs to certain BPF program types. This is required to limit bpf_sock_destroy kfunc implemented in follow-up commits to programs with attach type 'BPF_TRACE_ITER'. The commit adds a callback filter to 'struct btf_kfunc_id_set'. The filter has access to the `bpf_prog` construct including its properties such as `expected_attached_type`. Signed-off-by: Aditi Ghag Link: https://lore.kernel.org/r/20230519225157.760788-7-aditi.ghag@isovalent.com Signed-off-by: Martin KaFai Lau --- include/linux/btf.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 508199e38415..cac9f304e27a 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -98,10 +98,14 @@ struct btf_type; union bpf_attr; struct btf_show; struct btf_id_set; +struct bpf_prog; + +typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id); struct btf_kfunc_id_set { struct module *owner; struct btf_id_set8 *set; + btf_kfunc_filter_t filter; }; struct btf_id_dtor_kfunc { @@ -479,7 +483,6 @@ static inline void *btf_id_set8_contains(const struct btf_id_set8 *set, u32 id) return bsearch(&id, set->pairs, set->cnt, sizeof(set->pairs[0]), btf_id_cmp_func); } -struct bpf_prog; struct bpf_verifier_log; #ifdef CONFIG_BPF_SYSCALL @@ -487,10 +490,10 @@ const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); const char *btf_name_by_offset(const struct btf *btf, u32 offset); struct btf *btf_parse_vmlinux(void); struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog); -u32 *btf_kfunc_id_set_contains(const struct btf *btf, - enum bpf_prog_type prog_type, - u32 kfunc_btf_id); -u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id); +u32 *btf_kfunc_id_set_contains(const struct btf *btf, u32 kfunc_btf_id, + const struct bpf_prog *prog); +u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id, + const struct bpf_prog *prog); int register_btf_kfunc_id_set(enum bpf_prog_type prog_type, const struct btf_kfunc_id_set *s); int register_btf_fmodret_id_set(const struct btf_kfunc_id_set *kset); @@ -517,8 +520,9 @@ static inline const char *btf_name_by_offset(const struct btf *btf, return NULL; } static inline u32 *btf_kfunc_id_set_contains(const struct btf *btf, - enum bpf_prog_type prog_type, - u32 kfunc_btf_id) + u32 kfunc_btf_id, + struct bpf_prog *prog) + { return NULL; } -- cgit v1.2.3 From 4b159f5048b90844679dad08afb3240c1957aba1 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 19 May 2023 14:03:59 +0100 Subject: net: phy: add helpers for comparing phy IDs There are several places which open code comparing PHY IDs. Provide a couple of helpers to assist with this, using a slightly simpler test than the original: - phy_id_compare() compares two arbitary PHY IDs and a mask of the significant bits in the ID. - phydev_id_compare() compares the bound phydev with the specified PHY ID, using the bound driver's mask. Signed-off-by: Russell King Reviewed-by: Simon Horman Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- include/linux/phy.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index d8cd7115c773..2da87a36200d 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1112,6 +1112,34 @@ struct phy_driver { #define PHY_ID_MATCH_MODEL(id) .phy_id = (id), .phy_id_mask = GENMASK(31, 4) #define PHY_ID_MATCH_VENDOR(id) .phy_id = (id), .phy_id_mask = GENMASK(31, 10) +/** + * phy_id_compare - compare @id1 with @id2 taking account of @mask + * @id1: first PHY ID + * @id2: second PHY ID + * @mask: the PHY ID mask, set bits are significant in matching + * + * Return true if the bits from @id1 and @id2 specified by @mask match. + * This uses an equivalent test to (@id & @mask) == (@phy_id & @mask). + */ +static inline bool phy_id_compare(u32 id1, u32 id2, u32 mask) +{ + return !((id1 ^ id2) & mask); +} + +/** + * phydev_id_compare - compare @id with the PHY's Clause 22 ID + * @phydev: the PHY device + * @id: the PHY ID to be matched + * + * Compare the @phydev clause 22 ID with the provided @id and return true or + * false depending whether it matches, using the bound driver mask. The + * @phydev must be bound to a driver. + */ +static inline bool phydev_id_compare(struct phy_device *phydev, u32 id) +{ + return phy_id_compare(id, phydev->phy_id, phydev->drv->phy_id_mask); +} + /* A Structure for boards to register fixups with the PHY Lib */ struct phy_fixup { struct list_head list; -- cgit v1.2.3 From 72b44f6577f15f37fe964c8dcc42a7c5736e604c Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Tue, 16 May 2023 11:22:05 +1000 Subject: nubus: Don't list slot resources by default Some Nubus card ROMs contain many slot resources. A single Radius video card produced well over a thousand entries under /proc/bus/nubus/. Populating /proc/bus/nubus/ on a slow machine with several such cards installed takes long enough that the user may think that the system is wedged. All those procfs entries also consume significant RAM though they are not normally needed (except by developers). Omit these resources from /proc/bus/nubus/ by default and add a kernel parameter to enable them when needed. On the test machine, this saved 300 kB and 10 seconds. Cc: Brad Boyer Reviewed-by: Brad Boyer Tested-by: Stan Johnson Signed-off-by: Finn Thain Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/71ed7fb234a5f7381a50253b0d841a656d53e64c.1684200125.git.fthain@linux-m68k.org Signed-off-by: Geert Uytterhoeven --- include/linux/nubus.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/nubus.h b/include/linux/nubus.h index 392fc6c53e96..bdcd85e622d8 100644 --- a/include/linux/nubus.h +++ b/include/linux/nubus.h @@ -93,6 +93,7 @@ extern struct bus_type nubus_bus_type; /* Generic NuBus interface functions, modelled after the PCI interface */ #ifdef CONFIG_PROC_FS +extern bool nubus_populate_procfs; void nubus_proc_init(void); struct proc_dir_entry *nubus_proc_add_board(struct nubus_board *board); struct proc_dir_entry *nubus_proc_add_rsrc_dir(struct proc_dir_entry *procdir, -- cgit v1.2.3 From 4a20ce0ff68eb6fc4b1e8f25139c93b312f21229 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 4 May 2023 22:10:55 +0100 Subject: iommu: Add a capability for flush queue support Passing a special type to domain_alloc to indirectly query whether flush queues are a worthwhile optimisation with the given driver is a bit clunky, and looking increasingly anachronistic. Let's put that into an explicit capability instead. Signed-off-by: Robin Murphy Reviewed-by: Lu Baolu Tested-by: Jerry Snitselaar # amd, intel, smmu-v3 Reviewed-by: Jerry Snitselaar Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/f0086a93dbccb92622e1ace775846d81c1c4b174.1683233867.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e8c9a7da1060..1b7180d6edae 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -127,6 +127,11 @@ enum iommu_cap { * this device. */ IOMMU_CAP_ENFORCE_CACHE_COHERENCY, + /* + * IOMMU driver does not issue TLB maintenance during .unmap, so can + * usefully support the non-strict DMA flush queue. + */ + IOMMU_CAP_DEFERRED_FLUSH, }; /* These are the possible reserved region types */ -- cgit v1.2.3 From a4fdd976227240b06ced89b5df88a1a1f388f092 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 4 May 2023 22:10:56 +0100 Subject: iommu: Use flush queue capability It remains really handy to have distinct DMA domain types within core code for the sake of default domain policy selection, but we can now hide that detail from drivers by using the new capability instead. Signed-off-by: Robin Murphy Tested-by: Jerry Snitselaar # amd, intel, smmu-v3 Reviewed-by: Jerry Snitselaar Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/1c552d99e8ba452bdac48209fa74c0bdd52fd9d9.1683233867.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 1b7180d6edae..d31642596675 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -65,6 +65,7 @@ struct iommu_domain_geometry { #define __IOMMU_DOMAIN_SVA (1U << 4) /* Shared process address space */ +#define IOMMU_DOMAIN_ALLOC_FLAGS ~__IOMMU_DOMAIN_DMA_FQ /* * This are the possible domain-types * -- cgit v1.2.3 From b52878275ce54b5d3a654ed24dfb169c1c501998 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 2 May 2023 15:48:14 +0300 Subject: exportfs: change connectable argument to bit flags Convert the bool connectable arguemnt into a bit flags argument and define the EXPORT_FS_CONNECTABLE flag as a requested property of the file handle. We are going to add a flag for requesting non-decodeable file handles. Acked-by: Jeff Layton Acked-by: Chuck Lever Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20230502124817.3070545-2-amir73il@gmail.com> --- include/linux/exportfs.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 9edb29101ec8..fe4967ba61b2 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -135,6 +135,8 @@ struct fid { }; }; +#define EXPORT_FH_CONNECTABLE 0x1 /* Encode file handle with parent */ + /** * struct export_operations - for nfsd to communicate with file systems * @encode_fh: encode a file handle fragment from a dentry @@ -150,7 +152,7 @@ struct fid { * encode_fh: * @encode_fh should store in the file handle fragment @fh (using at most * @max_len bytes) information that can be used by @decode_fh to recover the - * file referred to by the &struct dentry @de. If the @connectable flag is + * file referred to by the &struct dentry @de. If @flag has CONNECTABLE bit * set, the encode_fh() should store sufficient information so that a good * attempt can be made to find not only the file but also it's place in the * filesystem. This typically means storing a reference to de->d_parent in @@ -227,7 +229,7 @@ struct export_operations { extern int exportfs_encode_inode_fh(struct inode *inode, struct fid *fid, int *max_len, struct inode *parent); extern int exportfs_encode_fh(struct dentry *dentry, struct fid *fid, - int *max_len, int connectable); + int *max_len, int flags); extern struct dentry *exportfs_decode_fh_raw(struct vfsmount *mnt, struct fid *fid, int fh_len, int fileid_type, -- cgit v1.2.3 From 304e9c83e80d5cbe20ab64ffa1fac9fc51d30bc9 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Tue, 2 May 2023 15:48:15 +0300 Subject: exportfs: add explicit flag to request non-decodeable file handles So far, all callers of exportfs_encode_inode_fh(), except for fsnotify's show_mark_fhandle(), check that filesystem can decode file handles, but we would like to add more callers that do not require a file handle that can be decoded. Introduce a flag to explicitly request a file handle that may not to be decoded later and a wrapper exportfs_encode_fid() that sets this flag and convert show_mark_fhandle() to use the new wrapper. This will be used to allow adding fanotify support to filesystems that do not support NFS export. Acked-by: Jeff Layton Acked-by: Chuck Lever Signed-off-by: Amir Goldstein Signed-off-by: Jan Kara Message-Id: <20230502124817.3070545-3-amir73il@gmail.com> --- include/linux/exportfs.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index fe4967ba61b2..11fbd0ee1370 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -136,6 +136,7 @@ struct fid { }; #define EXPORT_FH_CONNECTABLE 0x1 /* Encode file handle with parent */ +#define EXPORT_FH_FID 0x2 /* File handle may be non-decodeable */ /** * struct export_operations - for nfsd to communicate with file systems @@ -227,9 +228,18 @@ struct export_operations { }; extern int exportfs_encode_inode_fh(struct inode *inode, struct fid *fid, - int *max_len, struct inode *parent); + int *max_len, struct inode *parent, + int flags); extern int exportfs_encode_fh(struct dentry *dentry, struct fid *fid, int *max_len, int flags); + +static inline int exportfs_encode_fid(struct inode *inode, struct fid *fid, + int *max_len) +{ + return exportfs_encode_inode_fh(inode, fid, max_len, NULL, + EXPORT_FH_FID); +} + extern struct dentry *exportfs_decode_fh_raw(struct vfsmount *mnt, struct fid *fid, int fh_len, int fileid_type, -- cgit v1.2.3 From f8ddb0fb3289dfb6f064b1f0573fd4f032189e9e Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Tue, 23 May 2023 09:53:29 +0200 Subject: ALSA: usb-audio: Define USB MIDI 2.0 specs Define new structs and constants from USB MIDI 2.0 specification, to be used in the upcoming MIDI 2.0 support in USB-audio driver. A new class-specific endpoint descriptor and group terminal block descriptors are defined. Acked-by: Greg Kroah-Hartman Acked-by: Jaroslav Kysela Link: https://lore.kernel.org/r/20230523075358.9672-9-tiwai@suse.de Signed-off-by: Takashi Iwai --- include/linux/usb/midi-v2.h | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 include/linux/usb/midi-v2.h (limited to 'include/linux') diff --git a/include/linux/usb/midi-v2.h b/include/linux/usb/midi-v2.h new file mode 100644 index 000000000000..ebbffcae0417 --- /dev/null +++ b/include/linux/usb/midi-v2.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * -- USB MIDI 2.0 definitions. + */ + +#ifndef __LINUX_USB_MIDI_V2_H +#define __LINUX_USB_MIDI_V2_H + +#include +#include + +/* A.1 MS Class-Specific Interface Descriptor Types */ +#define USB_DT_CS_GR_TRM_BLOCK 0x26 + +/* A.1 MS Class-Specific Interface Descriptor Subtypes */ +/* same as MIDI 1.0 */ + +/* A.2 MS Class-Specific Endpoint Descriptor Subtypes */ +#define USB_MS_GENERAL_2_0 0x02 + +/* A.3 MS Class-Specific Group Terminal Block Descriptor Subtypes */ +#define USB_MS_GR_TRM_BLOCK_UNDEFINED 0x00 +#define USB_MS_GR_TRM_BLOCK_HEADER 0x01 +#define USB_MS_GR_TRM_BLOCK 0x02 + +/* A.4 MS Interface Header MIDIStreaming Class Revision */ +#define USB_MS_REV_MIDI_1_0 0x0100 +#define USB_MS_REV_MIDI_2_0 0x0200 + +/* A.5 MS MIDI IN and OUT Jack Types */ +/* same as MIDI 1.0 */ + +/* A.6 Group Terminal Block Types */ +#define USB_MS_GR_TRM_BLOCK_TYPE_BIDIRECTIONAL 0x00 +#define USB_MS_GR_TRM_BLOCK_TYPE_INPUT_ONLY 0x01 +#define USB_MS_GR_TRM_BLOCK_TYPE_OUTPUT_ONLY 0x02 + +/* A.7 Group Terminal Default MIDI Protocol */ +#define USB_MS_MIDI_PROTO_UNKNOWN 0x00 /* Unknown (Use MIDI-CI) */ +#define USB_MS_MIDI_PROTO_1_0_64 0x01 /* MIDI 1.0, UMP up to 64bits */ +#define USB_MS_MIDI_PROTO_1_0_64_JRTS 0x02 /* MIDI 1.0, UMP up to 64bits, Jitter Reduction Timestamps */ +#define USB_MS_MIDI_PROTO_1_0_128 0x03 /* MIDI 1.0, UMP up to 128bits */ +#define USB_MS_MIDI_PROTO_1_0_128_JRTS 0x04 /* MIDI 1.0, UMP up to 128bits, Jitter Reduction Timestamps */ +#define USB_MS_MIDI_PROTO_2_0 0x11 /* MIDI 2.0 */ +#define USB_MS_MIDI_PROTO_2_0_JRTS 0x12 /* MIDI 2.0, Jitter Reduction Timestamps */ + +/* 5.2.2.1 Class-Specific MS Interface Header Descriptor */ +/* Same as MIDI 1.0, use struct usb_ms_header_descriptor */ + +/* 5.3.2 Class-Specific MIDI Streaming Data Endpoint Descriptor */ +struct usb_ms20_endpoint_descriptor { + __u8 bLength; /* 4+n */ + __u8 bDescriptorType; /* USB_DT_CS_ENDPOINT */ + __u8 bDescriptorSubtype; /* USB_MS_GENERAL_2_0 */ + __u8 bNumGrpTrmBlock; /* Number of Group Terminal Blocks: n */ + __u8 baAssoGrpTrmBlkID[]; /* ID of the Group Terminal Blocks [n] */ +} __packed; + +#define USB_DT_MS20_ENDPOINT_SIZE(n) (4 + (n)) + +/* As above, but more useful for defining your own descriptors: */ +#define DECLARE_USB_MS20_ENDPOINT_DESCRIPTOR(n) \ +struct usb_ms20_endpoint_descriptor_##n { \ + __u8 bLength; \ + __u8 bDescriptorType; \ + __u8 bDescriptorSubtype; \ + __u8 bNumGrpTrmBlock; \ + __u8 baAssoGrpTrmBlkID[n]; \ +} __packed + +/* 5.4.1 Class-Specific Group Terminal Block Header Descriptor */ +struct usb_ms20_gr_trm_block_header_descriptor { + __u8 bLength; /* 5 */ + __u8 bDescriptorType; /* USB_DT_CS_GR_TRM_BLOCK */ + __u8 bDescriptorSubtype; /* USB_MS_GR_TRM_BLOCK_HEADER */ + __u16 wTotalLength; /* Total number of bytes */ +} __packed; + +/* 5.4.2.1 Group Terminal Block Descriptor */ +struct usb_ms20_gr_trm_block_descriptor { + __u8 bLength; /* 13 */ + __u8 bDescriptorType; /* USB_DT_CS_GR_TRM_BLOCK */ + __u8 bDescriptorSubtype; /* USB_MS_GR_TRM_BLOCK */ + __u8 bGrpTrmBlkID; /* ID of this Group Terminal Block */ + __u8 bGrpTrmBlkType; /* Group Terminal Block Type */ + __u8 nGroupTrm; /* The first member Group Terminal in this block */ + __u8 nNumGroupTrm; /* Number of member Group Terminals spanned */ + __u8 iBlockItem; /* String ID of Block item */ + __u8 bMIDIProtocol; /* Default MIDI protocol */ + __u16 wMaxInputBandwidth; /* Max input bandwidth capability in 4kB/s */ + __u16 wMaxOutputBandwidth; /* Max output bandwidth capability in 4kB/s */ +} __packed; + +#endif /* __LINUX_USB_MIDI_V2_H */ -- cgit v1.2.3 From 712fd23a90eed6a73ea5135a500e59d30356d4f1 Mon Sep 17 00:00:00 2001 From: Li Nan Date: Mon, 22 May 2023 16:53:55 +0800 Subject: block: remove redundant req_op in blk_rq_is_passthrough op &= REQ_OP_MASK in blk_op_is_passthrough() is exactly what req_op() do. Therefore, it is redundant to call req_op() for blk_op_is_passthrough(). Signed-off-by: Li Nan Reviewed-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Link: https://lore.kernel.org/r/20230522085355.1740772-1-linan666@huaweicloud.com Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index d778cb6b2112..59b52ec155b1 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -201,7 +201,7 @@ static inline enum req_op req_op(const struct request *req) static inline bool blk_rq_is_passthrough(struct request *rq) { - return blk_op_is_passthrough(req_op(rq)); + return blk_op_is_passthrough(rq->cmd_flags); } static inline unsigned short req_get_ioprio(struct request *req) -- cgit v1.2.3 From a13bd91be22318768d55470cbc0b0f4488ef9edf Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Fri, 14 Apr 2023 16:40:08 +0800 Subject: block/rq_qos: protect rq_qos apis with a new lock commit 50e34d78815e ("block: disable the elevator int del_gendisk") move rq_qos_exit() from disk_release() to del_gendisk(), this will introduce some problems: 1) If rq_qos_add() is triggered by enabling iocost/iolatency through cgroupfs, then it can concurrent with del_gendisk(), it's not safe to write 'q->rq_qos' concurrently. 2) Activate cgroup policy that is relied on rq_qos will call rq_qos_add() and blkcg_activate_policy(), and if rq_qos_exit() is called in the middle, null-ptr-dereference will be triggered in blkcg_activate_policy(). 3) blkg_conf_open_bdev() can call blkdev_get_no_open() first to find the disk, then if rq_qos_exit() from del_gendisk() is done before rq_qos_add(), then memory will be leaked. This patch add a new disk level mutex 'rq_qos_mutex': 1) The lock will protect rq_qos_exit() directly. 2) For wbt that doesn't relied on blk-cgroup, rq_qos_add() can only be called from disk initialization for now because wbt can't be destructed until rq_qos_exit(), so it's safe not to protect wbt for now. Hoever, in case that rq_qos dynamically destruction is supported in the furture, this patch also protect rq_qos_add() from wbt_init() directly, this is enough because blk-sysfs already synchronize writers with disk removal. 3) For iocost and iolatency, in order to synchronize disk removal and cgroup configuration, the lock is held after blkdev_get_no_open() from blkg_conf_open_bdev(), and is released in blkg_conf_exit(). In order to fix the above memory leak, disk_live() is checked after holding the new lock. Fixes: 50e34d78815e ("block: disable the elevator int del_gendisk") Signed-off-by: Yu Kuai Acked-by: Tejun Heo Link: https://lore.kernel.org/r/20230414084008.2085155-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index fe99948688df..b2ac587e3402 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -392,6 +392,7 @@ struct request_queue { struct blk_queue_stats *stats; struct rq_qos *rq_qos; + struct mutex rq_qos_mutex; const struct blk_mq_ops *mq_ops; -- cgit v1.2.3 From 431cb97b763133fba8b1c68c1ed089315f25e4dd Mon Sep 17 00:00:00 2001 From: Sebastian Reichel Date: Thu, 4 May 2023 19:36:14 +0200 Subject: regulator: expose regulator_find_closest_bigger Expose and document the table lookup logic used by regulator_set_ramp_delay_regmap, so that it can be reused for devices that cannot be configured via regulator_set_ramp_delay_regmap. Tested-by: Diederik de Haas # Rock64, Quartz64 Model A + B Tested-by: Vincent Legoll # Pine64 QuartzPro64 Signed-off-by: Sebastian Reichel Link: https://lore.kernel.org/r/20230504173618.142075-11-sebastian.reichel@collabora.com Signed-off-by: Mark Brown --- include/linux/regulator/driver.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index d3b4a3d4514a..c6ef7d68eb9a 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -758,6 +758,8 @@ int regulator_set_current_limit_regmap(struct regulator_dev *rdev, int min_uA, int max_uA); int regulator_get_current_limit_regmap(struct regulator_dev *rdev); void *regulator_get_init_drvdata(struct regulator_init_data *reg_init_data); +int regulator_find_closest_bigger(unsigned int target, const unsigned int *table, + unsigned int num_sel, unsigned int *sel); int regulator_set_ramp_delay_regmap(struct regulator_dev *rdev, int ramp_delay); int regulator_sync_voltage_rdev(struct regulator_dev *rdev); -- cgit v1.2.3 From cb8edce28073a906401c9e421eca7c99f3396da1 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 15 May 2023 16:48:06 -0700 Subject: bpf: Support O_PATH FDs in BPF_OBJ_PIN and BPF_OBJ_GET commands Current UAPI of BPF_OBJ_PIN and BPF_OBJ_GET commands of bpf() syscall forces users to specify pinning location as a string-based absolute or relative (to current working directory) path. This has various implications related to security (e.g., symlink-based attacks), forces BPF FS to be exposed in the file system, which can cause races with other applications. One of the feedbacks we got from folks working with containers heavily was that inability to use purely FD-based location specification was an unfortunate limitation and hindrance for BPF_OBJ_PIN and BPF_OBJ_GET commands. This patch closes this oversight, adding path_fd field to BPF_OBJ_PIN and BPF_OBJ_GET UAPI, following conventions established by *at() syscalls for dirfd + pathname combinations. This now allows interesting possibilities like working with detached BPF FS mount (e.g., to perform multiple pinnings without running a risk of someone interfering with them), and generally making pinning/getting more secure and not prone to any races and/or security attacks. This is demonstrated by a selftest added in subsequent patch that takes advantage of new mount APIs (fsopen, fsconfig, fsmount) to demonstrate creating detached BPF FS mount, pinning, and then getting BPF map out of it, all while never exposing this private instance of BPF FS to outside worlds. Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Reviewed-by: Christian Brauner Link: https://lore.kernel.org/bpf/20230523170013.728457-4-andrii@kernel.org --- include/linux/bpf.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 36e4b2d8cca2..f58895830ada 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2077,8 +2077,8 @@ struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); struct bpf_link *bpf_link_get_from_fd(u32 ufd); struct bpf_link *bpf_link_get_curr_or_next(u32 *id); -int bpf_obj_pin_user(u32 ufd, const char __user *pathname); -int bpf_obj_get_user(const char __user *pathname, int flags); +int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); +int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); #define BPF_ITER_FUNC_PREFIX "bpf_iter_" #define DEFINE_BPF_ITER_FUNC(target, args...) \ -- cgit v1.2.3 From b841b901c452d92610f739a36e54978453528876 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 13:11:10 +0100 Subject: net: Declare MSG_SPLICE_PAGES internal sendmsg() flag Declare MSG_SPLICE_PAGES, an internal sendmsg() flag, that hints to a network protocol that it should splice pages from the source iterator rather than copying the data if it can. This flag is added to a list that is cleared by sendmsg syscalls on entry. This is intended as a replacement for the ->sendpage() op, allowing a way to splice in several multipage folios in one go. Signed-off-by: David Howells Reviewed-by: Willem de Bruijn cc: Jens Axboe cc: Matthew Wilcox Signed-off-by: Jakub Kicinski --- include/linux/socket.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index 13c3a237b9c9..bd1cc3238851 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -327,6 +327,7 @@ struct ucred { */ #define MSG_ZEROCOPY 0x4000000 /* Use user data in kernel path */ +#define MSG_SPLICE_PAGES 0x8000000 /* Splice the pages from the iterator in sendmsg() */ #define MSG_FASTOPEN 0x20000000 /* Send data in TCP SYN */ #define MSG_CMSG_CLOEXEC 0x40000000 /* Set close_on_exec for file descriptor received through @@ -337,6 +338,8 @@ struct ucred { #define MSG_CMSG_COMPAT 0 /* We never have 32 bit fixups */ #endif +/* Flags to be cleared on entry by sendmsg and sendmmsg syscalls */ +#define MSG_INTERNAL_SENDMSG_FLAGS (MSG_SPLICE_PAGES) /* Setsockoptions(2) level. Thanks to BSD these must match IPPROTO_xxx */ #define SOL_IP 0 -- cgit v1.2.3 From 96449f90240713bd9bd653d6b15266a1044cfa7b Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 13:11:11 +0100 Subject: net: Pass max frags into skb_append_pagefrags() Pass the maximum number of fragments into skb_append_pagefrags() rather than using MAX_SKB_FRAGS so that it can be used from code that wants to specify sysctl_max_skb_frags. Signed-off-by: David Howells cc: David Ahern cc: Jens Axboe cc: Matthew Wilcox Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 8cff3d817131..15011408c47c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1383,7 +1383,7 @@ static inline int skb_pad(struct sk_buff *skb, int pad) #define dev_kfree_skb(a) consume_skb(a) int skb_append_pagefrags(struct sk_buff *skb, struct page *page, - int offset, size_t size); + int offset, size_t size, size_t max_frags); struct skb_seq_state { __u32 lower_offset; -- cgit v1.2.3 From 2e910b95329c2dc7feffbec00907f9e02d1a850a Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 13:11:12 +0100 Subject: net: Add a function to splice pages into an skbuff for MSG_SPLICE_PAGES Add a function to handle MSG_SPLICE_PAGES being passed internally to sendmsg(). Pages are spliced into the given socket buffer if possible and copied in if not (e.g. they're slab pages or have a zero refcount). Signed-off-by: David Howells cc: David Ahern cc: Al Viro cc: Jens Axboe cc: Matthew Wilcox Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 15011408c47c..1b2ebf6113e0 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -5097,5 +5097,8 @@ static inline void skb_mark_for_recycle(struct sk_buff *skb) #endif } +ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, + ssize_t maxsize, gfp_t gfp); + #endif /* __KERNEL__ */ #endif /* _LINUX_SKBUFF_H */ -- cgit v1.2.3 From 7c0bf4dad6bf44eef4a573985dd053de77688df1 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 23 May 2023 14:22:13 +0200 Subject: parport: Move magic number "15" to a define Put the size of a parport name behind a define so we can use it in other files. This is a preparation patch to be able to use this size in parport/procfs.c. Signed-off-by: Joel Granados Reviewed-by: Luis Chamberlain Signed-off-by: Luis Chamberlain --- include/linux/parport.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/parport.h b/include/linux/parport.h index a0bc9e0267b7..243c82d7f852 100644 --- a/include/linux/parport.h +++ b/include/linux/parport.h @@ -180,6 +180,8 @@ struct ieee1284_info { struct semaphore irq; }; +#define PARPORT_NAME_MAX_LEN 15 + /* A parallel port */ struct parport { unsigned long base; /* base address */ -- cgit v1.2.3 From 19c4e618a1bc3d0cad1f04c857be8076cb05bbb2 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 23 May 2023 14:22:18 +0200 Subject: sysctl: stop exporting register_sysctl_table We make register_sysctl_table static because the only function calling it is in fs/proc/proc_sysctl.c (__register_sysctl_base). We remove it from the sysctl.h header and modify the documentation in both the header and proc_sysctl.c files to mention "register_sysctl" instead of "register_sysctl_table". This plus the commits that remove register_sysctl_table from parport save 217 bytes: ./scripts/bloat-o-meter .bsysctl/vmlinux.old .bsysctl/vmlinux.new add/remove: 0/1 grow/shrink: 5/1 up/down: 458/-675 (-217) Function old new delta __register_sysctl_base 8 286 +278 parport_proc_register 268 379 +111 parport_device_proc_register 195 247 +52 kzalloc.constprop 598 608 +10 parport_default_proc_register 62 69 +7 register_sysctl_table 291 - -291 parport_sysctl_template 1288 904 -384 Total: Before=8603076, After=8602859, chg -0.00% Signed-off-by: Joel Granados Reviewed-by: Luis Chamberlain Signed-off-by: Luis Chamberlain --- include/linux/sysctl.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 3d08277959af..218e56a26fb0 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -89,7 +89,7 @@ int proc_do_static_key(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); /* - * Register a set of sysctl names by calling register_sysctl_table + * Register a set of sysctl names by calling register_sysctl * with an initialised array of struct ctl_table's. An entry with * NULL procname terminates the table. table->de will be * set up by the registration and need not be initialised in advance. @@ -222,7 +222,6 @@ struct ctl_table_header *__register_sysctl_table( struct ctl_table_set *set, const char *path, struct ctl_table *table); struct ctl_table_header *register_sysctl(const char *path, struct ctl_table *table); -struct ctl_table_header *register_sysctl_table(struct ctl_table * table); void unregister_sysctl_table(struct ctl_table_header * table); extern int sysctl_init_bases(void); @@ -257,11 +256,6 @@ static inline int __register_sysctl_base(struct ctl_table *base_table) #define register_sysctl_base(table) __register_sysctl_base(table) -static inline struct ctl_table_header *register_sysctl_table(struct ctl_table * table) -{ - return NULL; -} - static inline void register_sysctl_init(const char *path, struct ctl_table *table) { } -- cgit v1.2.3 From 2f5edd03ca0d7221a88236b344b84f3fc301b1e3 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Tue, 23 May 2023 14:22:19 +0200 Subject: sysctl: Refactor base paths registrations This is part of the general push to deprecate register_sysctl_paths and register_sysctl_table. The old way of doing this through register_sysctl_base and DECLARE_SYSCTL_BASE macro is replaced with a call to register_sysctl_init. The 5 base paths affected are: "kernel", "vm", "debug", "dev" and "fs". We remove the register_sysctl_base function and the DECLARE_SYSCTL_BASE macro since they are no longer needed. In order to quickly acertain that the paths did not actually change I executed `find /proc/sys/ | sha1sum` and made sure that the sha was the same before and after the commit. We end up saving 563 bytes with this change: ./scripts/bloat-o-meter vmlinux.0.base vmlinux.1.refactor-base-paths add/remove: 0/5 grow/shrink: 2/0 up/down: 77/-640 (-563) Function old new delta sysctl_init_bases 55 111 +56 init_fs_sysctls 12 33 +21 vm_base_table 128 - -128 kernel_base_table 128 - -128 fs_base_table 128 - -128 dev_base_table 128 - -128 debug_base_table 128 - -128 Total: Before=21258215, After=21257652, chg -0.00% [mcgrof: modified to use register_sysctl_init() over register_sysctl() and add bloat-o-meter stats] Signed-off-by: Joel Granados Signed-off-by: Luis Chamberlain Tested-by: Stephen Rothwell Acked-by: Christian Brauner --- include/linux/sysctl.h | 23 ----------------------- 1 file changed, 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 218e56a26fb0..653b66c762b1 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -197,20 +197,6 @@ struct ctl_path { #ifdef CONFIG_SYSCTL -#define DECLARE_SYSCTL_BASE(_name, _table) \ -static struct ctl_table _name##_base_table[] = { \ - { \ - .procname = #_name, \ - .mode = 0555, \ - .child = _table, \ - }, \ - { }, \ -} - -extern int __register_sysctl_base(struct ctl_table *base_table); - -#define register_sysctl_base(_name) __register_sysctl_base(_name##_base_table) - void proc_sys_poll_notify(struct ctl_table_poll *poll); extern void setup_sysctl_set(struct ctl_table_set *p, @@ -247,15 +233,6 @@ extern struct ctl_table sysctl_mount_point[]; #else /* CONFIG_SYSCTL */ -#define DECLARE_SYSCTL_BASE(_name, _table) - -static inline int __register_sysctl_base(struct ctl_table *base_table) -{ - return 0; -} - -#define register_sysctl_base(table) __register_sysctl_base(table) - static inline void register_sysctl_init(const char *path, struct ctl_table *table) { } -- cgit v1.2.3 From 59088b5a946ee8a6603a9a84781670cedb01c40d Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Mon, 22 May 2023 16:58:08 +0100 Subject: net: phy: avoid kernel warning dump when stopping an errored PHY When taking a network interface down (or removing a SFP module) after the PHY has encountered an error, phy_stop() complains incorrectly that it was called from HALTED state. The reason this is incorrect is that the network driver will have called phy_start() when the interface was brought up, and the fact that the PHY has a problem bears no relationship to the administrative state of the interface. Taking the interface administratively down (which calls phy_stop()) is always the right thing to do after a successful phy_start() call, whether or not the PHY has encountered an error. Signed-off-by: Russell King (Oracle) Acked-by: Florian Fainelli Signed-off-by: David S. Miller --- include/linux/phy.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index 2da87a36200d..7addde5d14c0 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -497,14 +497,17 @@ struct phy_device *mdiobus_scan_c22(struct mii_bus *bus, int addr); * Once complete, move to UP to restart the PHY. * - phy_stop aborts the running test and moves to @PHY_HALTED * - * @PHY_HALTED: PHY is up, but no polling or interrupts are done. Or - * PHY is in an error state. + * @PHY_HALTED: PHY is up, but no polling or interrupts are done. * - phy_start moves to @PHY_UP + * + * @PHY_ERROR: PHY is up, but is in an error state. + * - phy_stop moves to @PHY_HALTED */ enum phy_state { PHY_DOWN = 0, PHY_READY, PHY_HALTED, + PHY_ERROR, PHY_UP, PHY_RUNNING, PHY_NOLINK, -- cgit v1.2.3 From bc06a9e0874239cb6d4eebcb0ecd1a91ad9272db Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Fri, 19 May 2023 08:49:00 -0500 Subject: genirq: Use hlist for managing resend handlers The current implementation utilizes a bitmap for managing interrupt resend handlers, which is allocated based on the SPARSE_IRQ/NR_IRQS macros. However, this method may not efficiently utilize memory during runtime, particularly when IRQ_BITMAP_BITS is large. Address this issue by using an hlist to manage interrupt resend handlers instead of relying on a static bitmap memory allocation. Additionally, a new function, clear_irq_resend(), is introduced and called from irq_shutdown to ensure a graceful teardown of the interrupt. Signed-off-by: Shanker Donthineni Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230519134902.1495562-2-sdonthineni@nvidia.com --- include/linux/irqdesc.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index 844a8e30e6de..d9451d456a73 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -102,6 +102,9 @@ struct irq_desc { int parent_irq; struct module *owner; const char *name; +#ifdef CONFIG_HARDIRQS_SW_RESEND + struct hlist_node resend_node; +#endif } ____cacheline_internodealigned_in_smp; #ifdef CONFIG_SPARSE_IRQ -- cgit v1.2.3 From c32c81f3dbdfd68f6ab20a29ad86f811aed36e4e Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sun, 30 Apr 2023 11:35:05 +0200 Subject: ARM/mfd/gpio: Fixup TPS65010 regression on OMAP1 OSK1 Aaro reports problems on the OSK1 board after we altered the dynamic base for GPIO allocations. It appears this happens because the OMAP driver now allocates GPIO numbers dynamically, so all that is references by number is a bit up in the air. Let's bite the bullet and try to just move the gpio_chip in the tps65010 MFD driver over to using dynamic allocations. Alter everything in the OSK1 board file to use a GPIO descriptor table and lookups. Utilize the NULL device to define some board-specific GPIO lookups and use these to immediately look up the same GPIOs, convert to IRQ numbers and pass as resources to the devices. This is ugly but should work. The .setup() callback for tps65010 was used for some GPIO hogging, but since the OSK1 is the only user in the entire kernel we can alter the signatures to something that is helpful and make a clean transition. Fixes: 92bf78b33b0b ("gpio: omap: use dynamic allocation of base") Cc: Christophe Leroy Cc: andy.shevchenko@gmail.com Cc: Andreas Kemnade Acked-by: Lee Jones Reviewed-by: Lee Jones Reported-by: Aaro Koskinen Reviewed-by: Andy Shevchenko Signed-off-by: Linus Walleij --- include/linux/mfd/tps65010.h | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/tps65010.h b/include/linux/mfd/tps65010.h index a1fb9bc5311d..5edf1aef1118 100644 --- a/include/linux/mfd/tps65010.h +++ b/include/linux/mfd/tps65010.h @@ -28,6 +28,8 @@ #ifndef __LINUX_I2C_TPS65010_H #define __LINUX_I2C_TPS65010_H +struct gpio_chip; + /* * ---------------------------------------------------------------------------- * Registers, all 8 bits @@ -176,12 +178,10 @@ struct i2c_client; /** * struct tps65010_board - packages GPIO and LED lines - * @base: the GPIO number to assign to GPIO-1 * @outmask: bit (N-1) is set to allow GPIO-N to be used as an * (open drain) output * @setup: optional callback issued once the GPIOs are valid * @teardown: optional callback issued before the GPIOs are invalidated - * @context: optional parameter passed to setup() and teardown() * * Board data may be used to package the GPIO (and LED) lines for use * in by the generic GPIO and LED frameworks. The first four GPIOs @@ -193,12 +193,9 @@ struct i2c_client; * devices in their initial states using these GPIOs. */ struct tps65010_board { - int base; unsigned outmask; - - int (*setup)(struct i2c_client *client, void *context); - int (*teardown)(struct i2c_client *client, void *context); - void *context; + int (*setup)(struct i2c_client *client, struct gpio_chip *gc); + void (*teardown)(struct i2c_client *client, struct gpio_chip *gc); }; #endif /* __LINUX_I2C_TPS65010_H */ -- cgit v1.2.3 From 767d83361aaa6a1ecb4d5b89eeb38a267239917a Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Mon, 8 May 2023 23:20:06 +0200 Subject: Input: ads7846 - Convert to use software nodes The Nokia 770 is using GPIOs from the global numberspace on the CBUS node to pass down to the LCD controller. This regresses when we let the OMAP GPIO driver use dynamic GPIO base. The Nokia 770 now has dynamic allocation of IRQ numbers, so this needs to be fixed for it to work. As this is the only user of LCD MIPID we can easily augment the driver to use a GPIO descriptor instead and resolve the issue. The platform data .shutdown() callback wasn't even used in the code, but we encode a shutdown asserting RESET in the remove() callback for completeness sake. The CBUS also has the ADS7846 touchscreen attached. Populate the devices on the Nokia 770 CBUS I2C using software nodes instead of platform data quirks. This includes the LCD and the ADS7846 touchscreen so the conversion just brings the LCD along with it as software nodes is an all-or-nothing design pattern. The ADS7846 has some limited support for using GPIO descriptors, let's convert it over completely to using device properties and then fix all remaining boardfile users to provide all platform data using software nodes. Dump the of includes and of_match_ptr() in the ADS7846 driver as part of the job. Since we have to move ADS7846 over to obtaining the GPIOs it is using exclusively from descriptors, we provide descriptor tables for the two remaining in-kernel boardfiles using ADS7846: - PXA Spitz - MIPS Alchemy DB1000 development board It was too hard for me to include software node conversion of these two remaining users at this time: the spitz is using a hscync callback in the platform data that would require further GPIO descriptor conversion of the Spitz, and moving the hsync callback down into the driver: it will just become too big of a job, but it can be done separately. The MIPS Alchemy DB1000 is simply something I cannot test, so take the easier approach of just providing some GPIO descriptors in this case as I don't want the patch to grow too intrusive. As we see that several device trees have incorrect polarity flags and just expect to bypass the gpiolib polarity handling, fix up all device trees too, in a separate patch. Suggested-by: Dmitry Torokhov Fixes: 92bf78b33b0b ("gpio: omap: use dynamic allocation of base") Acked-by: Dmitry Torokhov Reviewed-by: Dmitry Torokhov Signed-off-by: Linus Walleij --- include/linux/platform_data/lcd-mipid.h | 2 -- include/linux/spi/ads7846.h | 2 -- 2 files changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/lcd-mipid.h b/include/linux/platform_data/lcd-mipid.h index 63f05eb23827..4927cfc5158c 100644 --- a/include/linux/platform_data/lcd-mipid.h +++ b/include/linux/platform_data/lcd-mipid.h @@ -15,10 +15,8 @@ enum mipid_test_result { #ifdef __KERNEL__ struct mipid_platform_data { - int nreset_gpio; int data_lines; - void (*shutdown)(struct mipid_platform_data *pdata); void (*set_bklight_level)(struct mipid_platform_data *pdata, int level); int (*get_bklight_level)(struct mipid_platform_data *pdata); diff --git a/include/linux/spi/ads7846.h b/include/linux/spi/ads7846.h index d424c1aadf38..a04c1c34c344 100644 --- a/include/linux/spi/ads7846.h +++ b/include/linux/spi/ads7846.h @@ -35,8 +35,6 @@ struct ads7846_platform_data { u16 debounce_tol; /* tolerance used for filtering */ u16 debounce_rep; /* additional consecutive good readings * required after the first two */ - int gpio_pendown; /* the GPIO used to decide the pendown - * state if get_pendown_state == NULL */ int gpio_pendown_debounce; /* platform specific debounce time for * the gpio_pendown */ int (*get_pendown_state)(void); -- cgit v1.2.3 From e519f0bb64efc2c9c8b67bb2d114dda458bdc34d Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Mon, 8 May 2023 23:20:07 +0200 Subject: ARM/mmc: Convert old mmci-omap to GPIO descriptors A recent change to the OMAP driver making it use a dynamic GPIO base created problems with some old OMAP1 board files, among them Nokia 770, SX1 and also the OMAP2 Nokia n8x0. Fix up all instances of GPIOs being used for the MMC driver by pushing the handling of power, slot selection and MMC "cover" into the driver as optional GPIOs. This is maybe not the most perfect solution as the MMC framework have some central handlers for some of the stuff, but it at least makes the situtation better and solves the immediate issue. Fixes: 92bf78b33b0b ("gpio: omap: use dynamic allocation of base") Acked-by: Ulf Hansson Signed-off-by: Linus Walleij --- include/linux/platform_data/mmc-omap.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/platform_data/mmc-omap.h b/include/linux/platform_data/mmc-omap.h index 91051e9907f3..054d0c3c5ec5 100644 --- a/include/linux/platform_data/mmc-omap.h +++ b/include/linux/platform_data/mmc-omap.h @@ -20,8 +20,6 @@ struct omap_mmc_platform_data { * maximum frequency on the MMC bus */ unsigned int max_freq; - /* switch the bus to a new slot */ - int (*switch_slot)(struct device *dev, int slot); /* initialize board-specific MMC functionality, can be NULL if * not supported */ int (*init)(struct device *dev); -- cgit v1.2.3 From d5f4fa60d63aa54ae33339895b88d8932b6037ed Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Mon, 1 May 2023 11:05:21 +0200 Subject: ARM/gpio: Push OMAP2 quirk down into TWL4030 driver The TWL4030 GPIO driver has a custom platform data .set_up() callback to call back into the platform and do misc stuff such as hog and export a GPIO for WLAN PWR on a specific OMAP3 board. Avoid all the kludgery in the platform data and the boardfile and just put the quirks right into the driver. Make it conditional on OMAP3. I think the exported GPIO is used by some kind of userspace so ordinary DTS hogs will probably not work. Fixes: 92bf78b33b0b ("gpio: omap: use dynamic allocation of base") Signed-off-by: Linus Walleij --- include/linux/mfd/twl.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/twl.h b/include/linux/mfd/twl.h index 6e3d99b7a0ee..c062d91a67d9 100644 --- a/include/linux/mfd/twl.h +++ b/include/linux/mfd/twl.h @@ -593,9 +593,6 @@ struct twl4030_gpio_platform_data { */ u32 pullups; u32 pulldowns; - - int (*setup)(struct device *dev, - unsigned gpio, unsigned ngpio); }; struct twl4030_madc_platform_data { -- cgit v1.2.3 From 8e0285ab95a9baf374f2c13eb152221c8ecb3f28 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sun, 30 Apr 2023 21:38:24 +0200 Subject: ARM/musb: omap2: Remove global GPIO numbers from TUSB6010 The TUSB6010 (MUSB) device is picking up some GPIO lines hardcoded by number and passing on to the TUSB6010 device when registering it. Instead of nasty workarounds, provide a GPIO descriptor table and then make the TUSB6010 MUSB glue driver pick up the GPIO lines directly, convert it to an IRQ and pass down to the MUSB driver. OMAP2 is the only system using the TUSB6010. Stash the GPIO descriptors in the glue layer and use then to power up and down the TUSB6010 on-demand, instead of using boardfile callbacks. Since the OMAP2 boards are the only boards using the .set_power() and .board_set_power() callbacks, we can just delete them as the power is now handled directly in the TUSB6010 glue code. Cc: Bin Liu Cc: linux-usb@vger.kernel.org Fixes: 92bf78b33b0b ("gpio: omap: use dynamic allocation of base") Acked-by: Greg Kroah-Hartman Signed-off-by: Linus Walleij --- include/linux/usb/musb.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/musb.h b/include/linux/usb/musb.h index e4a3ad3c800f..3963e55e88a3 100644 --- a/include/linux/usb/musb.h +++ b/include/linux/usb/musb.h @@ -99,9 +99,6 @@ struct musb_hdrc_platform_data { /* (HOST or OTG) program PHY for external Vbus */ unsigned extvbus:1; - /* Power the device on or off */ - int (*set_power)(int state); - /* MUSB configuration-specific details */ const struct musb_hdrc_config *config; @@ -135,14 +132,4 @@ static inline int musb_mailbox(enum musb_vbus_id_status status) #define TUSB6010_REFCLK_24 41667 /* psec/clk @ 24.0 MHz XI */ #define TUSB6010_REFCLK_19 52083 /* psec/clk @ 19.2 MHz CLKIN */ -#ifdef CONFIG_ARCH_OMAP2 - -extern int __init tusb6010_setup_interface( - struct musb_hdrc_platform_data *data, - unsigned ps_refclk, unsigned waitpin, - unsigned async_cs, unsigned sync_cs, - unsigned irq, unsigned dmachan); - -#endif /* OMAP2 */ - #endif /* __LINUX_USB_MUSB_H */ -- cgit v1.2.3 From 69df79a4511117f377d6a5909b47bd4fb541b978 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 14:49:50 +0100 Subject: splice: Rename direct_splice_read() to copy_splice_read() Rename direct_splice_read() to copy_splice_read() to better reflect as to what it does. Suggested-by: Christoph Hellwig Signed-off-by: David Howells Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner cc: Steve French cc: Jens Axboe cc: Al Viro cc: linux-cifs@vger.kernel.org cc: linux-mm@kvack.org cc: linux-block@vger.kernel.org cc: linux-fsdevel@vger.kernel.org Link: https://lore.kernel.org/r/20230522135018.2742245-4-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/fs.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 21a981680856..e3c22efa413e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2752,9 +2752,9 @@ ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb, ssize_t filemap_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); -ssize_t direct_splice_read(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, - size_t len, unsigned int flags); +ssize_t copy_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags); extern ssize_t generic_file_splice_read(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); extern ssize_t iter_file_splice_write(struct pipe_inode_info *, -- cgit v1.2.3 From 6a3f30b8bdb23842aff5eea65b6a7693c49f5506 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 14:49:52 +0100 Subject: splice: Make do_splice_to() generic and export it Rename do_splice_to() to vfs_splice_read() and export it so that it can be used as a helper when calling down to a lower layer filesystem as it performs all the necessary checks[1]. Signed-off-by: David Howells Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner cc: Miklos Szeredi cc: Jens Axboe cc: Al Viro cc: John Hubbard cc: David Hildenbrand cc: Matthew Wilcox cc: linux-unionfs@vger.kernel.org cc: linux-block@vger.kernel.org cc: linux-fsdevel@vger.kernel.org cc: linux-mm@kvack.org Link: https://lore.kernel.org/r/CAJfpeguGksS3sCigmRi9hJdUec8qtM9f+_9jC1rJhsXT+dV01w@mail.gmail.com/ [1] Link: https://lore.kernel.org/r/20230522135018.2742245-6-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/splice.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/splice.h b/include/linux/splice.h index a55179fd60fc..8f052c3dae95 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -76,6 +76,9 @@ extern ssize_t splice_to_pipe(struct pipe_inode_info *, struct splice_pipe_desc *); extern ssize_t add_to_pipe(struct pipe_inode_info *, struct pipe_buffer *); +long vfs_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags); extern ssize_t splice_direct_to_actor(struct file *, struct splice_desc *, splice_direct_actor *); extern long do_splice(struct file *in, loff_t *off_in, -- cgit v1.2.3 From c6585011bc1d8934cc78046c50fc94590fb2ab24 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 14:50:16 +0100 Subject: splice: Remove generic_file_splice_read() Remove generic_file_splice_read() as it has been replaced with calls to filemap_splice_read() and copy_splice_read(). With this, ITER_PIPE is no longer used. Signed-off-by: David Howells Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner cc: Jens Axboe cc: Steve French cc: Al Viro cc: David Hildenbrand cc: John Hubbard cc: linux-mm@kvack.org cc: linux-block@vger.kernel.org cc: linux-cifs@vger.kernel.org cc: linux-fsdevel@vger.kernel.org Link: https://lore.kernel.org/r/20230522135018.2742245-30-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/fs.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index e3c22efa413e..08ba2ae1d3ce 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2755,8 +2755,6 @@ ssize_t filemap_splice_read(struct file *in, loff_t *ppos, ssize_t copy_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); -extern ssize_t generic_file_splice_read(struct file *, loff_t *, - struct pipe_inode_info *, size_t, unsigned int); extern ssize_t iter_file_splice_write(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, -- cgit v1.2.3 From 3fc40265ae2b48a7475c41c5c0b256374c419f4b Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 14:50:17 +0100 Subject: iov_iter: Kill ITER_PIPE The ITER_PIPE-type iterator was only used by generic_file_splice_read() and that has been replaced and removed. This leaves ITER_PIPE unused - so remove it too. Signed-off-by: David Howells Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner cc: Jens Axboe cc: Al Viro cc: David Hildenbrand cc: John Hubbard cc: linux-mm@kvack.org cc: linux-block@vger.kernel.org cc: linux-fsdevel@vger.kernel.org Link: https://lore.kernel.org/r/20230522135018.2742245-31-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/uio.h | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/uio.h b/include/linux/uio.h index 044c1d8c230c..60c342bb7ab8 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -11,7 +11,6 @@ #include struct page; -struct pipe_inode_info; typedef unsigned int __bitwise iov_iter_extraction_t; @@ -25,7 +24,6 @@ enum iter_type { ITER_IOVEC, ITER_KVEC, ITER_BVEC, - ITER_PIPE, ITER_XARRAY, ITER_DISCARD, ITER_UBUF, @@ -74,7 +72,6 @@ struct iov_iter { const struct kvec *kvec; const struct bio_vec *bvec; struct xarray *xarray; - struct pipe_inode_info *pipe; void __user *ubuf; }; size_t count; @@ -82,10 +79,6 @@ struct iov_iter { }; union { unsigned long nr_segs; - struct { - unsigned int head; - unsigned int start_head; - }; loff_t xarray_start; }; }; @@ -133,11 +126,6 @@ static inline bool iov_iter_is_bvec(const struct iov_iter *i) return iov_iter_type(i) == ITER_BVEC; } -static inline bool iov_iter_is_pipe(const struct iov_iter *i) -{ - return iov_iter_type(i) == ITER_PIPE; -} - static inline bool iov_iter_is_discard(const struct iov_iter *i) { return iov_iter_type(i) == ITER_DISCARD; @@ -286,8 +274,6 @@ void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec unsigned long nr_segs, size_t count); void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec, unsigned long nr_segs, size_t count); -void iov_iter_pipe(struct iov_iter *i, unsigned int direction, struct pipe_inode_info *pipe, - size_t count); void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count); void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, loff_t start, size_t count); -- cgit v1.2.3 From 09e8c253415b8eb9ca29a2131d2ebf17743534c5 Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 21:57:40 +0100 Subject: block: Fix bio_flagged() so that gcc can better optimise it Fix bio_flagged() so that multiple instances of it, such as: if (bio_flagged(bio, BIO_PAGE_REFFED) || bio_flagged(bio, BIO_PAGE_PINNED)) can be combined by the gcc optimiser into a single test in assembly (arguably, this is a compiler optimisation issue[1]). The missed optimisation stems from bio_flagged() comparing the result of the bitwise-AND to zero. This results in an out-of-line bio_release_page() being compiled to something like: <+0>: mov 0x14(%rdi),%eax <+3>: test $0x1,%al <+5>: jne 0xffffffff816dac53 <+7>: test $0x2,%al <+9>: je 0xffffffff816dac5c <+11>: movzbl %sil,%esi <+15>: jmp 0xffffffff816daba1 <__bio_release_pages> <+20>: jmp 0xffffffff81d0b800 <__x86_return_thunk> However, the test is superfluous as the return type is bool. Removing it results in: <+0>: testb $0x3,0x14(%rdi) <+4>: je 0xffffffff816e4af4 <+6>: movzbl %sil,%esi <+10>: jmp 0xffffffff816dab7c <__bio_release_pages> <+15>: jmp 0xffffffff81d0b7c0 <__x86_return_thunk> instead. Also, the MOVZBL instruction looks unnecessary[2] - I think it's just 're-booling' the mark_dirty parameter. Signed-off-by: David Howells Reviewed-by: Christoph Hellwig Reviewed-by: John Hubbard cc: Jens Axboe cc: linux-block@vger.kernel.org Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108370 [1] Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108371 [2] Link: https://lore.kernel.org/r/167391056756.2311931.356007731815807265.stgit@warthog.procyon.org.uk/ # v6 Reviewed-by: Christian Brauner Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230522205744.2825689-3-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/bio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index b3e7529ff55e..7f53be035cf0 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -229,7 +229,7 @@ static inline void bio_cnt_set(struct bio *bio, unsigned int count) static inline bool bio_flagged(struct bio *bio, unsigned int bit) { - return (bio->bi_flags & (1U << bit)) != 0; + return bio->bi_flags & (1U << bit); } static inline void bio_set_flag(struct bio *bio, unsigned int bit) -- cgit v1.2.3 From e51bab4e20586fb3afc30536b776a97ed8ffb681 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 May 2023 21:57:41 +0100 Subject: block: Replace BIO_NO_PAGE_REF with BIO_PAGE_REFFED with inverted logic Replace BIO_NO_PAGE_REF with a BIO_PAGE_REFFED flag that has the inverted meaning is only set when a page reference has been acquired that needs to be released by bio_release_pages(). Signed-off-by: Christoph Hellwig Signed-off-by: David Howells Reviewed-by: John Hubbard cc: Al Viro cc: Jens Axboe cc: Jan Kara cc: Matthew Wilcox cc: Logan Gunthorpe cc: linux-block@vger.kernel.org Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230522205744.2825689-4-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/bio.h | 2 +- include/linux/blk_types.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index 7f53be035cf0..0922729acd26 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -488,7 +488,7 @@ void zero_fill_bio(struct bio *bio); static inline void bio_release_pages(struct bio *bio, bool mark_dirty) { - if (!bio_flagged(bio, BIO_NO_PAGE_REF)) + if (bio_flagged(bio, BIO_PAGE_REFFED)) __bio_release_pages(bio, mark_dirty); } diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 740afe80f297..dfd2c2cb909d 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -323,7 +323,7 @@ struct bio { * bio flags */ enum { - BIO_NO_PAGE_REF, /* don't put release vec pages */ + BIO_PAGE_REFFED, /* put pages in bio_release_pages() */ BIO_CLONED, /* doesn't own data */ BIO_BOUNCED, /* bio is a bounce bio */ BIO_QUIET, /* Make BIO Quiet */ -- cgit v1.2.3 From fd363244e883323e1ac9412d96fd22b51e255b0c Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 22 May 2023 21:57:42 +0100 Subject: block: Add BIO_PAGE_PINNED and associated infrastructure Add BIO_PAGE_PINNED to indicate that the pages in a bio are pinned (FOLL_PIN) and that the pin will need removing. Signed-off-by: David Howells Reviewed-by: Christoph Hellwig Reviewed-by: John Hubbard cc: Al Viro cc: Jens Axboe cc: Jan Kara cc: Matthew Wilcox cc: Logan Gunthorpe cc: linux-block@vger.kernel.org Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230522205744.2825689-5-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/bio.h | 3 ++- include/linux/blk_types.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index 0922729acd26..8588bcfbc6ef 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -488,7 +488,8 @@ void zero_fill_bio(struct bio *bio); static inline void bio_release_pages(struct bio *bio, bool mark_dirty) { - if (bio_flagged(bio, BIO_PAGE_REFFED)) + if (bio_flagged(bio, BIO_PAGE_REFFED) || + bio_flagged(bio, BIO_PAGE_PINNED)) __bio_release_pages(bio, mark_dirty); } diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index dfd2c2cb909d..8ef209e3aa96 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -323,6 +323,7 @@ struct bio { * bio flags */ enum { + BIO_PAGE_PINNED, /* Unpin pages in bio_release_pages() */ BIO_PAGE_REFFED, /* put pages in bio_release_pages() */ BIO_CLONED, /* doesn't own data */ BIO_BOUNCED, /* bio is a bounce bio */ -- cgit v1.2.3 From e9261467ae86a6544bb602a55a1eab52696e71e3 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Tue, 23 May 2023 11:15:48 +0100 Subject: net: mdio: add clause 73 to ethtool conversion helper Add a helper to convert a clause 73 advertisement to an ethtool bitmap. Reviewed-by: Andrew Lunn Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- include/linux/mdio.h | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 27013d6bf24a..0670cc6e067c 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -486,6 +486,45 @@ static inline u32 linkmode_adv_to_mii_10base_t1_t(unsigned long *adv) return result; } +/** + * mii_c73_mod_linkmode - convert a Clause 73 advertisement to linkmodes + * @adv: linkmode advertisement setting + * @lpa: array of three u16s containing the advertisement + * + * Convert an IEEE 802.3 Clause 73 advertisement to ethtool link modes. + */ +static inline void mii_c73_mod_linkmode(unsigned long *adv, u16 *lpa) +{ + linkmode_mod_bit(ETHTOOL_LINK_MODE_Pause_BIT, + adv, lpa[0] & MDIO_AN_C73_0_PAUSE); + linkmode_mod_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, + adv, lpa[0] & MDIO_AN_C73_0_ASM_DIR); + linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_1000BASE_KX); + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_10GBASE_KX4); + linkmode_mod_bit(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_40GBASE_KR4); + linkmode_mod_bit(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_40GBASE_CR4); + /* 100GBASE_CR10 and 100GBASE_KP4 not implemented */ + linkmode_mod_bit(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_100GBASE_KR4); + linkmode_mod_bit(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_100GBASE_CR4); + /* 25GBASE_R_S not implemented */ + /* The 25GBASE_R bit can be used for 25Gbase KR or CR modes */ + linkmode_mod_bit(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_25GBASE_R); + linkmode_mod_bit(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_25GBASE_R); + linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, + adv, lpa[1] & MDIO_AN_C73_1_10GBASE_KR); + linkmode_mod_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, + adv, lpa[2] & MDIO_AN_C73_2_2500BASE_KX); + /* 5GBASE_KR not implemented */ +} + int __mdiobus_read(struct mii_bus *bus, int addr, u32 regnum); int __mdiobus_write(struct mii_bus *bus, int addr, u32 regnum, u16 val); int __mdiobus_modify_changed(struct mii_bus *bus, int addr, u32 regnum, -- cgit v1.2.3 From dad987484eaaa7cd7f7f7459f4aee1470d8ec8ef Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Tue, 23 May 2023 11:15:58 +0100 Subject: net: phylink: add function to resolve clause 73 negotiation Add a function to resolve clause 73 negotiation according to the priority resolution function described in clause 73.3.6. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Signed-off-by: Jakub Kicinski --- include/linux/phylink.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index bb782f05ad08..0cf07d7d11b8 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -656,6 +656,8 @@ int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, const unsigned long *advertising); void phylink_mii_c22_pcs_an_restart(struct mdio_device *pcs); +void phylink_resolve_c73(struct phylink_link_state *state); + void phylink_mii_c45_pcs_get_state(struct mdio_device *pcs, struct phylink_link_state *state); -- cgit v1.2.3 From 31cb1304ad8bd27b7d2abd8669fb887fb47d8eaf Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:05 +0800 Subject: powercap: intel_rapl: Remove unused field in struct rapl_if_priv After commit f1e8d7560d30 ("powercap/intel_rapl: enumerate Psys RAPL domain together with package RAPL domain"), the platform_rapl_domain field is not used anymore. Remove it from rapl_if_priv structure. Fixes: f1e8d7560d30 ("powercap/intel_rapl: enumerate Psys RAPL domain together with package RAPL domain") Signed-off-by: Zhang Rui Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 9f4b6f5b822f..828557645770 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -124,7 +124,6 @@ struct reg_action { */ struct rapl_if_priv { struct powercap_control_type *control_type; - struct rapl_domain *platform_rapl_domain; enum cpuhp_state pcap_rapl_online; u64 reg_unit; u64 regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; -- cgit v1.2.3 From e8e28c2af16b279b6c37d533e1e73effb197cf2e Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:07 +0800 Subject: powercap: intel_rapl: Support per Interface rapl_defaults rapl_defaults is Interface specific. Although current MSR and MMIO Interface share the same rapl_defaults, new Interface like TPMI need its own rapl_defaults callbacks. Save the rapl_defaults information in the Interface private structure. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 828557645770..ebd1cad78212 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -121,6 +121,7 @@ struct reg_action { * registers. * @write_raw: Callback for writing RAPL interface specific * registers. + * @defaults: internal pointer to interface default settings */ struct rapl_if_priv { struct powercap_control_type *control_type; @@ -130,6 +131,7 @@ struct rapl_if_priv { int limits[RAPL_DOMAIN_MAX]; int (*read_raw)(int cpu, struct reg_action *ra); int (*write_raw)(int cpu, struct reg_action *ra); + void *defaults; }; /* maximum rapl package domain name: package-%d-die-%d */ -- cgit v1.2.3 From 98ff639a7289067247b3ef9dd5d1e922361e7365 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:08 +0800 Subject: powercap: intel_rapl: Support per Interface primitive information RAPL primitive information is Interface specific. Although current MSR and MMIO Interface share the same RAPL primitives, new Interface like TPMI has its own RAPL primitive information. Save the primitive information in the Interface private structure. Plus, using variant name "rp" for struct rapl_primitive_info is confusing because "rp" is also used for struct rapl_package. Use "rpi" as the variant name for struct rapl_primitive_info, and rename the previous rpi[] array to avoid conflict. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index ebd1cad78212..f51e2df7130e 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -122,6 +122,7 @@ struct reg_action { * @write_raw: Callback for writing RAPL interface specific * registers. * @defaults: internal pointer to interface default settings + * @rpi: internal pointer to interface primitive info */ struct rapl_if_priv { struct powercap_control_type *control_type; @@ -132,6 +133,7 @@ struct rapl_if_priv { int (*read_raw)(int cpu, struct reg_action *ra); int (*write_raw)(int cpu, struct reg_action *ra); void *defaults; + void *rpi; }; /* maximum rapl package domain name: package-%d-die-%d */ -- cgit v1.2.3 From cb532e728ee2880be53264752e74945fd2d917ac Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:09 +0800 Subject: powercap: intel_rapl: Support per domain energy/power/time unit RAPL MSR/MMIO Interface has package scope unit register but some RAPL domains like Dram/Psys may use a fixed energy unit value instead of the default unit value on certain platforms. RAPL TPMI Interface supports per domain unit register. For the above reasons, add support for per domain unit register and per domain energy/power/time unit. When per domain unit register is not available, use the package scope unit register as the per domain unit register for each RAPL domain so that this change is transparent to MSR/MMIO Interface. No functional change intended. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index f51e2df7130e..936fb8c3082c 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -30,6 +30,7 @@ enum rapl_domain_reg_id { RAPL_DOMAIN_REG_POLICY, RAPL_DOMAIN_REG_INFO, RAPL_DOMAIN_REG_PL4, + RAPL_DOMAIN_REG_UNIT, RAPL_DOMAIN_REG_MAX, }; @@ -96,7 +97,9 @@ struct rapl_domain { struct rapl_power_limit rpl[NR_POWER_LIMITS]; u64 attr_map; /* track capabilities */ unsigned int state; - unsigned int domain_energy_unit; + unsigned int power_unit; + unsigned int energy_unit; + unsigned int time_unit; struct rapl_package *rp; }; @@ -143,9 +146,6 @@ struct rapl_package { unsigned int id; /* logical die id, equals physical 1-die systems */ unsigned int nr_domains; unsigned long domain_map; /* bit map of active domains */ - unsigned int power_unit; - unsigned int energy_unit; - unsigned int time_unit; struct rapl_domain *domains; /* array of domains, sized at runtime */ struct powercap_zone *power_zone; /* keep track of parent zone */ unsigned long power_limit_irq; /* keep track of package power limit -- cgit v1.2.3 From 045610c383bd6b740bb7e7c780d6f7729249e60d Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:11 +0800 Subject: powercap: intel_rapl: Change primitive order The same set of operations are shared by different Powert Limits, including Power Limit get/set, Power Limit enable/disable, clamping enable/disable, time window get/set, and max power get/set, etc. But the same operation for different Power Limit has different primitives because they use different registers/register bits. A lot of dirty/duplicate code was introduced to handle this difference. Instead of using hardcoded primitive name directly, using Power Limit id + operation type is much cleaner. For this sense, move POWER_LIMIT1/POWER_LIMIT2/POWER_LIMIT4 to the beginning of enum rapl_primitives so that they can be reused as Power Limit ids. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 936fb8c3082c..bbd03b17dc8d 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -37,10 +37,10 @@ enum rapl_domain_reg_id { struct rapl_domain; enum rapl_primitives { - ENERGY_COUNTER, POWER_LIMIT1, POWER_LIMIT2, POWER_LIMIT4, + ENERGY_COUNTER, FW_LOCK, PL1_ENABLE, /* power limit 1, aka long term */ @@ -75,7 +75,8 @@ struct rapl_domain_data { unsigned long timestamp; }; -#define NR_POWER_LIMITS (3) +#define NR_POWER_LIMITS (POWER_LIMIT4 + 1) + struct rapl_power_limit { struct powercap_zone_constraint *constraint; int prim_id; /* primitive ID used to enable */ -- cgit v1.2.3 From 9050a9cd5e4c848e265915d6e7b1f731e6e1e0e6 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:13 +0800 Subject: powercap: intel_rapl: Cleanup Power Limits support The same set of operations are shared by different Powert Limits, including Power Limit get/set, Power Limit enable/disable, clamping enable/disable, time window get/set, and max power get/set, etc. But the same operation for different Power Limit has different primitives because they use different registers/register bits. A lot of dirty/duplicate code was introduced to handle this difference. Introduce a universal way to issue Power Limit operations. Instead of using hardcoded primitive name directly, use Power Limit id + operation type, and hide all the Power Limit difference details in a central place, get_pl_prim(). Two helpers, rapl_read_pl_data() and rapl_write_pl_data(), are introduced at the same time to simplify the code for issuing Power Limit operations. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index bbd03b17dc8d..df17f4e51dbf 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -79,7 +79,6 @@ struct rapl_domain_data { struct rapl_power_limit { struct powercap_zone_constraint *constraint; - int prim_id; /* primitive ID used to enable */ struct rapl_domain *domain; const char *name; u64 last_power_limit; -- cgit v1.2.3 From f442bd2742174eed6993315ec621275df13f311d Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:14 +0800 Subject: powercap: intel_rapl: Add support for lock bit per Power Limit With RAPL MSR/MMIO Interface, each RAPL domain has one Power Limit register. Each Power Limit register has one lock bit which tells the OS if the power limit register can be used or not. Depending on the number of power limits supported by the power limit register, the lock bit may apply to one or more power limits. With RAPL TPMI Interface, each RAPL domain has multiple Power Limits, and each Power Limit has its own register, with a lock bit. To handle this, introduce support for lock bit per Power Limit. For existing RAPL MSR/MMIO Interfaces, the lock bit in the Power Limit register applies to all the Power Limits controlled by this register. Remove the per domain DOMAIN_STATE_BIOS_LOCKED flag at the same time because it can be replaced by the per Power Limit lock. No functional change intended. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index df17f4e51dbf..d07b460bac3b 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -42,6 +42,7 @@ enum rapl_primitives { POWER_LIMIT4, ENERGY_COUNTER, FW_LOCK, + FW_HIGH_LOCK, PL1_ENABLE, /* power limit 1, aka long term */ PL1_CLAMP, /* allow frequency to go below OS request */ @@ -81,6 +82,7 @@ struct rapl_power_limit { struct powercap_zone_constraint *constraint; struct rapl_domain *domain; const char *name; + bool locked; u64 last_power_limit; }; -- cgit v1.2.3 From bf44b9011df3d6e34a23be77d86540553ba2bbe2 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:16 +0800 Subject: powercap: intel_rapl: Make cpu optional for rapl_package MSR RAPL Interface always removes a rapl_package when all the CPUs in that rapl_package are offlined. This is because it relies on an online CPU to access the MSR. But for RAPL Interface using MMIO registers, when all the cpus within the rapl_package are offlined, 1. the register can still be accessed 2. monitoring and setting the Power Pimits for the rapl_package is still meaningful because of uncore power. This means that, a valid rapl_package doesn't rely on one or more cpus being onlined. For this sense, make cpu optional for rapl_package. A rapl_package can be registered either using a CPU id to represent the physical package/die, or using the physical package id directly. Note that, the thermal throttling interrupt is not disabled via MSR_IA32_PACKAGE_THERM_INTERRUPT for such rapl_package at the moment. If it is still needed in the future, this can be achieved by selecting an onlined CPU using the physical package id. Note that, processor_thermal_rapl, the current MMIO RAPL Interface driver, can also be converted to register using a package id instead. But this is not done right now because processor_thermal_rapl driver works on single-package systems only, and offlining the only package will not happen. So keep the previous logic. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index d07b460bac3b..51509f35027b 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -135,8 +135,8 @@ struct rapl_if_priv { u64 reg_unit; u64 regs[RAPL_DOMAIN_MAX][RAPL_DOMAIN_REG_MAX]; int limits[RAPL_DOMAIN_MAX]; - int (*read_raw)(int cpu, struct reg_action *ra); - int (*write_raw)(int cpu, struct reg_action *ra); + int (*read_raw)(int id, struct reg_action *ra); + int (*write_raw)(int id, struct reg_action *ra); void *defaults; void *rpi; }; @@ -161,8 +161,8 @@ struct rapl_package { struct rapl_if_priv *priv; }; -struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv); -struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv); +struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu); +struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu); void rapl_remove_package(struct rapl_package *rp); #endif /* __INTEL_RAPL_H__ */ -- cgit v1.2.3 From b4288ce788aaf160f2a706672af2eaef417bb057 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:17 +0800 Subject: powercap: intel_rapl: Introduce RAPL I/F type Different RAPL Interfaces may have different primitive information and rapl_defaults calls. To better distinguish this difference in the RAPL framework code, introduce a new enum to represent different types of RAPL Interfaces. No functional change. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 51509f35027b..65f358b64096 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -14,6 +14,11 @@ #include #include +enum rapl_if_type { + RAPL_IF_MSR, /* RAPL I/F using MSR registers */ + RAPL_IF_MMIO, /* RAPL I/F using MMIO registers */ +}; + enum rapl_domain_type { RAPL_DOMAIN_PACKAGE, /* entire package/socket */ RAPL_DOMAIN_PP0, /* core power plane */ @@ -130,6 +135,7 @@ struct reg_action { * @rpi: internal pointer to interface primitive info */ struct rapl_if_priv { + enum rapl_if_type type; struct powercap_control_type *control_type; enum cpuhp_state pcap_rapl_online; u64 reg_unit; -- cgit v1.2.3 From e12dee18b89f1b0d4fc070eda4843f9d806645ca Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Wed, 19 Apr 2023 10:44:18 +0800 Subject: powercap: intel_rapl: Introduce core support for TPMI interface Compared with existing RAPL MSR/MMIO Interface, the RAPL TPMI Interface 1. has per Power Limit register, thus has per Power Limit Lock and Enable bit. 2. doesn't have Power Limit Clamp bit. 3. the Power Limit Lock and Enable bits have different bit offsets. These mean RAPL TPMI Interface needs its own primitive information. RAPL TPMI Interface also has per domain unit register but with a different register layout. This requires a TPMI specific rapl_defaults call to decode the unit register. Introduce the RAPL core support for TPMI Interface. Signed-off-by: Zhang Rui Tested-by: Wang Wendy Signed-off-by: Rafael J. Wysocki --- include/linux/intel_rapl.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index 65f358b64096..e6936cb25047 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -17,6 +17,7 @@ enum rapl_if_type { RAPL_IF_MSR, /* RAPL I/F using MSR registers */ RAPL_IF_MMIO, /* RAPL I/F using MMIO registers */ + RAPL_IF_TPMI, /* RAPL I/F using TPMI registers */ }; enum rapl_domain_type { @@ -36,6 +37,7 @@ enum rapl_domain_reg_id { RAPL_DOMAIN_REG_INFO, RAPL_DOMAIN_REG_PL4, RAPL_DOMAIN_REG_UNIT, + RAPL_DOMAIN_REG_PL2, RAPL_DOMAIN_REG_MAX, }; @@ -48,6 +50,9 @@ enum rapl_primitives { ENERGY_COUNTER, FW_LOCK, FW_HIGH_LOCK, + PL1_LOCK, + PL2_LOCK, + PL4_LOCK, PL1_ENABLE, /* power limit 1, aka long term */ PL1_CLAMP, /* allow frequency to go below OS request */ -- cgit v1.2.3 From 2e41e3ca4729455e002bcb585f0d3749ee66d572 Mon Sep 17 00:00:00 2001 From: Kai-Heng Feng Date: Tue, 2 May 2023 17:04:34 +0200 Subject: PM: suspend: Fix pm_suspend_target_state handling for !CONFIG_PM Move the pm_suspend_target_state definition for CONFIG_SUSPEND unset from the wakeup code into the headers so as to allow it to still be used elsewhere when CONFIG_SUSPEND is not set. Signed-off-by: Kai-Heng Feng [ rjw: Changelog and subject edits ] Signed-off-by: Rafael J. Wysocki --- include/linux/suspend.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index d0d4598a7b3f..474ecfbbaa62 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -202,6 +202,7 @@ struct platform_s2idle_ops { }; #ifdef CONFIG_SUSPEND +extern suspend_state_t pm_suspend_target_state; extern suspend_state_t mem_sleep_current; extern suspend_state_t mem_sleep_default; @@ -337,6 +338,8 @@ extern bool sync_on_suspend_enabled; #else /* !CONFIG_SUSPEND */ #define suspend_valid_only_mem NULL +#define pm_suspend_target_state (PM_SUSPEND_ON) + static inline void pm_suspend_clear_flags(void) {} static inline void pm_set_suspend_via_firmware(void) {} static inline void pm_set_resume_via_firmware(void) {} @@ -503,7 +506,6 @@ extern void pm_report_max_hw_sleep(u64 t); /* drivers/base/power/wakeup.c */ extern bool events_check_enabled; -extern suspend_state_t pm_suspend_target_state; extern bool pm_wakeup_pending(void); extern void pm_system_wakeup(void); -- cgit v1.2.3 From ab23ed6e73ecd198bf577077677beaded0a9e718 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:58 +0200 Subject: PM: suspend: add a arch_resume_nosmt() prototype The arch_resume_nosmt() has a __weak definition, plus an x86 specific override, but no prototype that ensures the two have the same arguments. This causes a W=1 warning: arch/x86/power/hibernate.c:189:5: error: no previous prototype for 'arch_resume_nosmt' [-Werror=missing-prototypes] Add the prototype in linux/suspend.h, which is included in both places. Signed-off-by: Arnd Bergmann Signed-off-by: Rafael J. Wysocki --- include/linux/suspend.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 474ecfbbaa62..1a0426e6761c 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -471,6 +471,8 @@ static inline int hibernate_quiet_exec(int (*func)(void *data), void *data) { } #endif /* CONFIG_HIBERNATION */ +int arch_resume_nosmt(void); + #ifdef CONFIG_HIBERNATION_SNAPSHOT_DEV int is_hibernate_resume_dev(dev_t dev); #else -- cgit v1.2.3 From 75c8cb2f4cb218aaf4ea68cab08d6dbc96eeae15 Mon Sep 17 00:00:00 2001 From: Martin Botka Date: Wed, 24 May 2023 01:00:10 +0100 Subject: mfd: axp20x: Add support for AXP313a PMIC The AXP313a is a PMIC chip produced by X-Powers, it can be connected via an I2C bus. The name AXP1530 seems to appear as well, and this is what is used in the BSP driver. From all we know it's the same chip, just a different name. However we have only seen AXP313a chips in the wild, so go with this name. Compared to the other AXP PMICs it's a rather simple affair: just three DCDC converters, three LDOs, and no battery charging support. Describe the regmap and the MFD bits, along with the registers exposed via I2C. Aside from the various regulators, also describe the power key interrupts, and adjust the shutdown handler routine to use a different register than the other PMICs. Eventually advertise the device using the new compatible string. Signed-off-by: Martin Botka Signed-off-by: Andre Przywara Reviewed-by: Chen-Yu Tsai Link: https://lore.kernel.org/r/20230524000012.15028-2-andre.przywara@arm.com Signed-off-by: Lee Jones --- include/linux/mfd/axp20x.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mfd/axp20x.h b/include/linux/mfd/axp20x.h index beb3f44f85c5..fff7fa6b7c5d 100644 --- a/include/linux/mfd/axp20x.h +++ b/include/linux/mfd/axp20x.h @@ -17,6 +17,7 @@ enum axp20x_variants { AXP221_ID, AXP223_ID, AXP288_ID, + AXP313A_ID, AXP803_ID, AXP806_ID, AXP809_ID, @@ -92,6 +93,17 @@ enum axp20x_variants { #define AXP22X_ALDO3_V_OUT 0x2a #define AXP22X_CHRG_CTRL3 0x35 +#define AXP313A_ON_INDICATE 0x00 +#define AXP313A_OUTPUT_CONTROL 0x10 +#define AXP313A_DCDC1_CONRTOL 0x13 +#define AXP313A_DCDC2_CONRTOL 0x14 +#define AXP313A_DCDC3_CONRTOL 0x15 +#define AXP313A_ALDO1_CONRTOL 0x16 +#define AXP313A_DLDO1_CONRTOL 0x17 +#define AXP313A_SHUTDOWN_CTRL 0x1a +#define AXP313A_IRQ_EN 0x20 +#define AXP313A_IRQ_STATE 0x21 + #define AXP806_STARTUP_SRC 0x00 #define AXP806_CHIP_ID 0x03 #define AXP806_PWR_OUT_CTRL1 0x10 @@ -363,6 +375,16 @@ enum { AXP22X_REG_ID_MAX, }; +enum { + AXP313A_DCDC1 = 0, + AXP313A_DCDC2, + AXP313A_DCDC3, + AXP313A_ALDO1, + AXP313A_DLDO1, + AXP313A_RTC_LDO, + AXP313A_REG_ID_MAX, +}; + enum { AXP806_DCDCA = 0, AXP806_DCDCB, @@ -616,6 +638,16 @@ enum axp288_irqs { AXP288_IRQ_BC_USB_CHNG, }; +enum axp313a_irqs { + AXP313A_IRQ_DIE_TEMP_HIGH, + AXP313A_IRQ_DCDC2_V_LOW = 2, + AXP313A_IRQ_DCDC3_V_LOW, + AXP313A_IRQ_PEK_LONG, + AXP313A_IRQ_PEK_SHORT, + AXP313A_IRQ_PEK_FAL_EDGE, + AXP313A_IRQ_PEK_RIS_EDGE, +}; + enum axp803_irqs { AXP803_IRQ_ACIN_OVER_V = 1, AXP803_IRQ_ACIN_PLUGIN, -- cgit v1.2.3 From 5f3139fc46993b2d653a7aa5cdfe66a91881fd06 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Mon, 15 May 2023 13:54:42 +0100 Subject: io_uring/cmd: add cmd lazy tw wake helper We want to use IOU_F_TWQ_LAZY_WAKE in commands. First, introduce a new cmd tw helper accepting TWQ flags, and then add io_uring_cmd_do_in_task_laz() that will pass IOU_F_TWQ_LAZY_WAKE and imply the "lazy" semantics, i.e. it posts no more than 1 CQE and delaying execution of this tw should not prevent forward progress. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/5b9f6716006df7e817f18bd555aee2f8f9c8b0c3.1684154817.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- include/linux/io_uring.h | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 7fe31b2cd02f..bb9c666bd584 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -46,13 +46,23 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd); void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2, unsigned issue_flags); -void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, - void (*task_work_cb)(struct io_uring_cmd *, unsigned)); struct sock *io_uring_get_socket(struct file *file); void __io_uring_cancel(bool cancel_all); void __io_uring_free(struct task_struct *tsk); void io_uring_unreg_ringfd(void); const char *io_uring_get_opcode(u8 opcode); +void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, + void (*task_work_cb)(struct io_uring_cmd *, unsigned), + unsigned flags); +/* users should follow semantics of IOU_F_TWQ_LAZY_WAKE */ +void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd, + void (*task_work_cb)(struct io_uring_cmd *, unsigned)); + +static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, + void (*task_work_cb)(struct io_uring_cmd *, unsigned)) +{ + __io_uring_cmd_do_in_task(ioucmd, task_work_cb, 0); +} static inline void io_uring_files_cancel(void) { @@ -85,6 +95,10 @@ static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, void (*task_work_cb)(struct io_uring_cmd *, unsigned)) { } +static inline void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd, + void (*task_work_cb)(struct io_uring_cmd *, unsigned)) +{ +} static inline struct sock *io_uring_get_socket(struct file *file) { return NULL; -- cgit v1.2.3 From 09fcdbd28404b7e02cc9fc4862ae5b43b76867c0 Mon Sep 17 00:00:00 2001 From: Martin Blumenstingl Date: Mon, 22 May 2023 22:24:24 +0200 Subject: mmc: sdio: Add/rename SDIO ID of the RTL8723DS SDIO wifi cards RTL8723DS comes in two variant and each of them has their own SDIO ID: - 0xd723 can connect two antennas. The WiFi part is still 1x1 so the second antenna can be dedicated to Bluetooth - 0xd724 can only connect one antenna so it's shared between WiFi and Bluetooth Add a new entry for the single antenna RTL8723DS (0xd724) which can be found on the MangoPi MQ-Quad. Also rename the existing RTL8723DS entry (0xd723) so it's name reflects that it's the variant with support for two antennas. Reviewed-by: Ping-Ke Shih Signed-off-by: Martin Blumenstingl Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20230522202425.1827005-4-martin.blumenstingl@googlemail.com --- include/linux/mmc/sdio_ids.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index c653accdc7fd..7fada7a714fe 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -121,7 +121,8 @@ #define SDIO_DEVICE_ID_REALTEK_RTW8822BS 0xb822 #define SDIO_DEVICE_ID_REALTEK_RTW8821CS 0xc821 #define SDIO_DEVICE_ID_REALTEK_RTW8822CS 0xc822 -#define SDIO_DEVICE_ID_REALTEK_RTW8723DS 0xd723 +#define SDIO_DEVICE_ID_REALTEK_RTW8723DS_2ANT 0xd723 +#define SDIO_DEVICE_ID_REALTEK_RTW8723DS_1ANT 0xd724 #define SDIO_DEVICE_ID_REALTEK_RTW8821DS 0xd821 #define SDIO_VENDOR_ID_SIANO 0x039a -- cgit v1.2.3 From 30955b4afc2bbea9046c60df994297fac5edc02c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 17:30:57 +0200 Subject: ARM: davinci: fix davinci_cpufreq_init() declaration The davinci_cpufreq_init() declaration is only seen by its caller but not the definition: drivers/cpufreq/davinci-cpufreq.c:153:12: error: no previous prototype for 'davinci_cpufreq_init' Move it into the platform_data header that is already used an interface between the two places. Acked-by: Bartosz Golaszewski Link: https://lore.kernel.org/r/20230516153109.514251-2-arnd@kernel.org Signed-off-by: Arnd Bergmann --- include/linux/platform_data/davinci-cpufreq.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/platform_data/davinci-cpufreq.h b/include/linux/platform_data/davinci-cpufreq.h index bc208c64e3d7..1ef91c36f609 100644 --- a/include/linux/platform_data/davinci-cpufreq.h +++ b/include/linux/platform_data/davinci-cpufreq.h @@ -16,4 +16,10 @@ struct davinci_cpufreq_config { int (*init)(void); }; +#ifdef CONFIG_CPU_FREQ +int davinci_cpufreq_init(void); +#else +static inline int davinci_cpufreq_init(void) { return 0; } +#endif + #endif /* _MACH_DAVINCI_CPUFREQ_H */ -- cgit v1.2.3 From a9ae9c526cc232b69b0bc9d668e303c90600e848 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 17:31:06 +0200 Subject: ARM: pxa: fix missing-prototypes warnings The PXA platform has a number of configurations that end up with a warning like these when building with W=1: drivers/hwmon/max1111.c:83:5: error: no previous prototype for 'max1111_read_channel' [-Werror=missing-prototypes] arch/arm/mach-pxa/reset.c:86:6: error: no previous prototype for 'pxa_restart' [-Werror=missing-prototypes] arch/arm/mach-pxa/mfp-pxa2xx.c:254:5: error: no previous prototype for 'keypad_set_wake' [-Werror=missing-prototypes] drivers/clk/pxa/clk-pxa25x.c:70:14: error: no previous prototype for 'pxa25x_get_clk_frequency_khz' [-Werror=missing-prototypes] drivers/clk/pxa/clk-pxa25x.c:325:12: error: no previous prototype for 'pxa25x_clocks_init' [-Werror=missing-prototypes] drivers/clk/pxa/clk-pxa27x.c:74:14: error: no previous prototype for 'pxa27x_get_clk_frequency_khz' [-Werror=missing-prototypes] drivers/clk/pxa/clk-pxa27x.c:102:6: error: no previous prototype for 'pxa27x_is_ppll_disabled' [-Werror=missing-prototypes] drivers/clk/pxa/clk-pxa27x.c:470:12: error: no previous prototype for 'pxa27x_clocks_init' [-Werror=missing-prototypes] arch/arm/mach-pxa/pxa27x.c:44:6: error: no previous prototype for 'pxa27x_clear_otgph' [-Werror=missing-prototypes] arch/arm/mach-pxa/pxa27x.c:58:6: error: no previous prototype for 'pxa27x_configure_ac97reset' [-Werror=missing-prototypes] arch/arm/mach-pxa/spitz_pm.c:170:15: error: no previous prototype for 'spitzpm_read_devdata' [-Werror=missing-prototypes] The problem is that there is a declaration for each of these, but it's only seen by the caller and not the callee. Moving these into appropriate header files ensures that both use the same calling conventions and it avoids the warnings. Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/20230516153109.514251-11-arnd@kernel.org Signed-off-by: Arnd Bergmann --- include/linux/platform_data/asoc-pxa.h | 1 + include/linux/platform_data/pxa2xx_udc.h | 6 ++++++ include/linux/soc/pxa/smemc.h | 16 ++++++++++++++++ 3 files changed, 23 insertions(+) (limited to 'include/linux') diff --git a/include/linux/platform_data/asoc-pxa.h b/include/linux/platform_data/asoc-pxa.h index 327454cd8246..7b5b9e20fbf5 100644 --- a/include/linux/platform_data/asoc-pxa.h +++ b/include/linux/platform_data/asoc-pxa.h @@ -27,5 +27,6 @@ typedef struct { } pxa2xx_audio_ops_t; extern void pxa_set_ac97_info(pxa2xx_audio_ops_t *ops); +extern void pxa27x_configure_ac97reset(int reset_gpio, bool to_gpio); #endif diff --git a/include/linux/platform_data/pxa2xx_udc.h b/include/linux/platform_data/pxa2xx_udc.h index ff9c35dca59d..bc99cc6a3c5f 100644 --- a/include/linux/platform_data/pxa2xx_udc.h +++ b/include/linux/platform_data/pxa2xx_udc.h @@ -25,4 +25,10 @@ struct pxa2xx_udc_mach_info { int gpio_pullup; /* high == pullup activated */ }; +#ifdef CONFIG_PXA27x +extern void pxa27x_clear_otgph(void); +#else +#define pxa27x_clear_otgph() do {} while (0) +#endif + #endif diff --git a/include/linux/soc/pxa/smemc.h b/include/linux/soc/pxa/smemc.h index f1ffea236c15..4feb1dded3ec 100644 --- a/include/linux/soc/pxa/smemc.h +++ b/include/linux/soc/pxa/smemc.h @@ -10,4 +10,20 @@ int pxa2xx_smemc_get_sdram_rows(void); unsigned int pxa3xx_smemc_get_memclkdiv(void); void __iomem *pxa_smemc_get_mdrefr(void); +/* + * Once fully converted to the clock framework, all these functions should be + * removed, and replaced with a clk_get(NULL, "core"). + */ +#ifdef CONFIG_PXA25x +extern unsigned pxa25x_get_clk_frequency_khz(int); +#else +#define pxa25x_get_clk_frequency_khz(x) (0) +#endif + +#ifdef CONFIG_PXA27x +extern unsigned pxa27x_get_clk_frequency_khz(int); +#else +#define pxa27x_get_clk_frequency_khz(x) (0) +#endif + #endif -- cgit v1.2.3 From db967cf828fc134ba17c5e4539b1a3687cdd3f2d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 22:12:12 +0200 Subject: arm-cci: add cci_enable_port_for_self prototype The cci_enable_port_for_self() is called from assembler, so add the prototype only to shut up the W=1 warning: drivers/bus/arm-cci.c:298:25: error: no previous prototype for 'cci_enable_port_for_self' [-Werror=missing-prototypes] Link: https://lore.kernel.org/r/20230516201218.556437-1-arnd@kernel.org Signed-off-by: Arnd Bergmann --- include/linux/arm-cci.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/arm-cci.h b/include/linux/arm-cci.h index d0e44201d855..7f7a576267bc 100644 --- a/include/linux/arm-cci.h +++ b/include/linux/arm-cci.h @@ -43,6 +43,8 @@ static inline int __cci_control_port_by_index(u32 port, bool enable) } #endif +void cci_enable_port_for_self(void); + #define cci_disable_port_by_device(dev) \ __cci_control_port_by_device(dev, false) #define cci_enable_port_by_device(dev) \ -- cgit v1.2.3 From 62c68e7cee332e08e625af3bca3318814086490d Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Tue, 23 May 2023 14:04:32 -0700 Subject: HID: ensure timely release of driver-allocated resources More and more drivers rely on devres to manage their resources, however if bus' probe() and release() methods are not trivial and control some of resources as well (for example enable or disable clocks, or attach device to a power domain), we need to make sure that driver-allocated resources are released immediately after driver's remove() method returns, and not postponed until driver core gets around to releasing resources. In case of HID we should not try to close the report and release associated memory until after all devres callbacks are executed. To fix that we open a new devres group before calling driver's probe() and explicitly release it when we return from driver's remove(). This is similar to what we did for I2C bus in commit 5b5475826c52 ("i2c: ensure timely release of driver-allocated resources"). It is tempting to try and move this into driver core, but actually doing so is challenging, we need to split bus' remove() method into pre- and post-remove methods, which would make the logic even less clear. Reported-by: Stephen Boyd Link: https://lore.kernel.org/r/20230505232417.1377393-1-swboyd@chromium.org Signed-off-by: Dmitry Torokhov Signed-off-by: Jiri Kosina --- include/linux/hid.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/hid.h b/include/linux/hid.h index 4e4c4fe36911..39e21e3815ad 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -597,6 +597,7 @@ struct hid_device { /* device report descriptor */ struct semaphore driver_input_lock; /* protects the current driver */ struct device dev; /* device */ struct hid_driver *driver; + void *devres_group_id; /* ID of probe devres group */ const struct hid_ll_driver *ll_driver; struct mutex ll_open_lock; -- cgit v1.2.3 From d67790ddf0219aa0ad3e13b53ae0a7619b3425a2 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 22 May 2023 14:18:13 -0700 Subject: overflow: Add struct_size_t() helper While struct_size() is normally used in situations where the structure type already has a pointer instance, there are places where no variable is available. In the past, this has been worked around by using a typed NULL first argument, but this is a bit ugly. Add a helper to do this, and replace the handful of instances of the code pattern with it. Instances were found with this Coccinelle script: @struct_size_t@ identifier STRUCT, MEMBER; expression COUNT; @@ - struct_size((struct STRUCT *)\(0\|NULL\), + struct_size_t(struct STRUCT, MEMBER, COUNT) Suggested-by: Christoph Hellwig Cc: Jesse Brandeburg Cc: Tony Nguyen Cc: "David S. Miller" Cc: Eric Dumazet Cc: Paolo Abeni Cc: James Smart Cc: Keith Busch Cc: Jens Axboe Cc: Sagi Grimberg Cc: HighPoint Linux Team Cc: "James E.J. Bottomley" Cc: "Martin K. Petersen" Cc: Kashyap Desai Cc: Sumit Saxena Cc: Shivasharan S Cc: Don Brace Cc: "Darrick J. Wong" Cc: Dave Chinner Cc: Guo Xuenan Cc: Gwan-gyeong Mun Cc: Nick Desaulniers Cc: Daniel Latypov Cc: kernel test robot Cc: intel-wired-lan@lists.osuosl.org Cc: netdev@vger.kernel.org Cc: linux-nvme@lists.infradead.org Cc: linux-scsi@vger.kernel.org Cc: megaraidlinux.pdl@broadcom.com Cc: storagedev@microchip.com Cc: linux-xfs@vger.kernel.org Cc: linux-hardening@vger.kernel.org Signed-off-by: Kees Cook Acked-by: Martin K. Petersen Reviewed-by: Darrick J. Wong Reviewed-by: Gustavo A. R. Silva Reviewed-by: Christoph Hellwig Acked-by: Jakub Kicinski Reviewed-by: Alexander Lobakin Link: https://lore.kernel.org/r/20230522211810.never.421-kees@kernel.org --- include/linux/overflow.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/overflow.h b/include/linux/overflow.h index 0e33b5cbdb9f..f9b60313eaea 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -283,7 +283,7 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) * @member: Name of the array member. * @count: Number of elements in the array. * - * Calculates size of memory needed for structure @p followed by an + * Calculates size of memory needed for structure of @p followed by an * array of @count number of @member elements. * * Return: number of bytes needed or SIZE_MAX on overflow. @@ -293,4 +293,20 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) sizeof(*(p)) + flex_array_size(p, member, count), \ size_add(sizeof(*(p)), flex_array_size(p, member, count))) +/** + * struct_size_t() - Calculate size of structure with trailing flexible array + * @type: structure type name. + * @member: Name of the array member. + * @count: Number of elements in the array. + * + * Calculates size of memory needed for structure @type followed by an + * array of @count number of @member elements. Prefer using struct_size() + * when possible instead, to keep calculations associated with a specific + * instance variable of type @type. + * + * Return: number of bytes needed or SIZE_MAX on overflow. + */ +#define struct_size_t(type, member, count) \ + struct_size((type *)NULL, member, count) + #endif /* __LINUX_OVERFLOW_H */ -- cgit v1.2.3 From 4f521bab5bfc854ec0dab7ef560dfa75247e615d Mon Sep 17 00:00:00 2001 From: Maninder Singh Date: Fri, 26 May 2023 12:51:23 +0530 Subject: kallsyms: remove unsed API lookup_symbol_attrs with commit '7878c231dae0 ("slab: remove /proc/slab_allocators")' lookup_symbol_attrs usage is removed. Thus removing redundant API. Signed-off-by: Maninder Singh Reviewed-by: Kees Cook Signed-off-by: Luis Chamberlain --- include/linux/kallsyms.h | 6 ------ include/linux/module.h | 9 --------- 2 files changed, 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h index fe3c9993b5bf..1037f4957caa 100644 --- a/include/linux/kallsyms.h +++ b/include/linux/kallsyms.h @@ -93,7 +93,6 @@ extern int sprint_backtrace(char *buffer, unsigned long address); extern int sprint_backtrace_build_id(char *buffer, unsigned long address); int lookup_symbol_name(unsigned long addr, char *symname); -int lookup_symbol_attrs(unsigned long addr, unsigned long *size, unsigned long *offset, char *modname, char *name); /* How and when do we show kallsyms values? */ extern bool kallsyms_show_value(const struct cred *cred); @@ -155,11 +154,6 @@ static inline int lookup_symbol_name(unsigned long addr, char *symname) return -ERANGE; } -static inline int lookup_symbol_attrs(unsigned long addr, unsigned long *size, unsigned long *offset, char *modname, char *name) -{ - return -ERANGE; -} - static inline bool kallsyms_show_value(const struct cred *cred) { return false; diff --git a/include/linux/module.h b/include/linux/module.h index 9e56763dff81..a98e188cf37b 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -968,15 +968,6 @@ static inline int lookup_module_symbol_name(unsigned long addr, char *symname) return -ERANGE; } -static inline int lookup_module_symbol_attrs(unsigned long addr, - unsigned long *size, - unsigned long *offset, - char *modname, - char *name) -{ - return -ERANGE; -} - static inline int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *name, char *module_name, int *exported) -- cgit v1.2.3 From ec001bb71e4476f7f5be9db693d5f43e65b9d8cb Mon Sep 17 00:00:00 2001 From: Robert Marko Date: Fri, 26 May 2023 22:47:58 +0200 Subject: soc: qcom: socinfo: move SMEM item struct and defines to a header Move SMEM item struct and related defines to a header in order to be able to reuse them in the SMEM driver instead of duplicating them. Signed-off-by: Robert Marko Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20230526204802.3081168-1-robimarko@gmail.com --- include/linux/soc/qcom/socinfo.h | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 include/linux/soc/qcom/socinfo.h (limited to 'include/linux') diff --git a/include/linux/soc/qcom/socinfo.h b/include/linux/soc/qcom/socinfo.h new file mode 100644 index 000000000000..d1cbc49a2a2d --- /dev/null +++ b/include/linux/soc/qcom/socinfo.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __QCOM_SOCINFO_H__ +#define __QCOM_SOCINFO_H__ + +/* + * SMEM item id, used to acquire handles to respective + * SMEM region. + */ +#define SMEM_HW_SW_BUILD_ID 137 + +#define SMEM_SOCINFO_BUILD_ID_LENGTH 32 +#define SMEM_SOCINFO_CHIP_ID_LENGTH 32 + +/* Socinfo SMEM item structure */ +struct socinfo { + __le32 fmt; + __le32 id; + __le32 ver; + char build_id[SMEM_SOCINFO_BUILD_ID_LENGTH]; + /* Version 2 */ + __le32 raw_id; + __le32 raw_ver; + /* Version 3 */ + __le32 hw_plat; + /* Version 4 */ + __le32 plat_ver; + /* Version 5 */ + __le32 accessory_chip; + /* Version 6 */ + __le32 hw_plat_subtype; + /* Version 7 */ + __le32 pmic_model; + __le32 pmic_die_rev; + /* Version 8 */ + __le32 pmic_model_1; + __le32 pmic_die_rev_1; + __le32 pmic_model_2; + __le32 pmic_die_rev_2; + /* Version 9 */ + __le32 foundry_id; + /* Version 10 */ + __le32 serial_num; + /* Version 11 */ + __le32 num_pmics; + __le32 pmic_array_offset; + /* Version 12 */ + __le32 chip_family; + __le32 raw_device_family; + __le32 raw_device_num; + /* Version 13 */ + __le32 nproduct_id; + char chip_id[SMEM_SOCINFO_CHIP_ID_LENGTH]; + /* Version 14 */ + __le32 num_clusters; + __le32 ncluster_array_offset; + __le32 num_defective_parts; + __le32 ndefective_parts_array_offset; + /* Version 15 */ + __le32 nmodem_supported; + /* Version 16 */ + __le32 feature_code; + __le32 pcode; + __le32 npartnamemap_offset; + __le32 nnum_partname_mapping; + /* Version 17 */ + __le32 oem_variant; +}; + +#endif -- cgit v1.2.3 From 17051d2c3cd696439adb900e9af547ba162fb982 Mon Sep 17 00:00:00 2001 From: Robert Marko Date: Fri, 26 May 2023 22:48:00 +0200 Subject: soc: qcom: smem: introduce qcom_smem_get_soc_id() Introduce a helper to return the SoC SMEM ID, which is used to identify the exact SoC model as there may be differences in the same SoC family. Currently, cpufreq-nvmem does this completely in the driver and there has been more interest expresed for other drivers to use this information so lets expose a common helper to prevent redoing it in individual drivers since this field is present on every SMEM table version. Signed-off-by: Robert Marko Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20230526204802.3081168-3-robimarko@gmail.com --- include/linux/soc/qcom/smem.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/smem.h b/include/linux/soc/qcom/smem.h index 86e1b358688a..223db6a9c733 100644 --- a/include/linux/soc/qcom/smem.h +++ b/include/linux/soc/qcom/smem.h @@ -11,4 +11,6 @@ int qcom_smem_get_free_space(unsigned host); phys_addr_t qcom_smem_virt_to_phys(void *p); +int qcom_smem_get_soc_id(u32 *id); + #endif -- cgit v1.2.3 From 4a3a2c32a5ee163bc8f195b04751f165aa4d9c83 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 8 May 2023 09:42:15 +0200 Subject: PM / devfreq: Reorder fields in 'struct devfreq_dev_status' Group some variables based on their sizes to reduce holes. On x86_64, this shrinks the size of 'struct devfreq_dev_status' from 72 to 64 bytes. This structure is used both to allocate static variables or is embedded in some other structures. In both cases, reducing its size is nice to have. Moreover, the whole structure now fits in a single cache line on x86_64. Finally, it makes the order of code match the order of the above kernel doc. Signed-off-by: Christophe JAILLET Acked-by: MyungJoo Ham Signed-off-by: Chanwoo Choi --- include/linux/devfreq.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h index 7fd704bb8f3d..d312ffbac4dd 100644 --- a/include/linux/devfreq.h +++ b/include/linux/devfreq.h @@ -108,7 +108,6 @@ struct devfreq_dev_profile { unsigned long initial_freq; unsigned int polling_ms; enum devfreq_timer timer; - bool is_cooling_device; int (*target)(struct device *dev, unsigned long *freq, u32 flags); int (*get_dev_status)(struct device *dev, @@ -118,6 +117,8 @@ struct devfreq_dev_profile { unsigned long *freq_table; unsigned int max_state; + + bool is_cooling_device; }; /** -- cgit v1.2.3 From c4933fa88a68c69205753601044949d516c4db10 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 26 May 2023 11:14:24 +0100 Subject: net: mdio: add mdio_device_get() and mdio_device_put() Add two new operations for a mdio device to manage the refcount on the underlying struct device. This will be used by mdio PCS drivers to simplify the creation and destruction handling, making it easier for users to get it correct. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Signed-off-by: Jakub Kicinski --- include/linux/mdio.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 0670cc6e067c..c1b7008826e5 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -106,6 +106,16 @@ int mdio_driver_register(struct mdio_driver *drv); void mdio_driver_unregister(struct mdio_driver *drv); int mdio_device_bus_match(struct device *dev, struct device_driver *drv); +static inline void mdio_device_get(struct mdio_device *mdiodev) +{ + get_device(&mdiodev->dev); +} + +static inline void mdio_device_put(struct mdio_device *mdiodev) +{ + mdio_device_free(mdiodev); +} + static inline bool mdio_phy_id_is_c45(int phy_id) { return (phy_id & MDIO_PHY_ID_C45) && !(phy_id & ~MDIO_PHY_ID_C45_MASK); -- cgit v1.2.3 From 9a5d500cffdb3652215172b7c5829ca7b41e9efe Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 26 May 2023 11:14:29 +0100 Subject: net: pcs: xpcs: add xpcs_create_mdiodev() Add xpcs_create_mdiodev() to simplify the creation of the mdio device associated with the XPCS. In order to allow xpcs_destroy() to clean this up, we need to arrange for xpcs_create() to take a refcount on the mdiodev, and xpcs_destroy() to put it. Adding the refcounting to xpcs_create()..xpcs_destroy() will be transparent to existing users of these interfaces. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Signed-off-by: Jakub Kicinski --- include/linux/pcs/pcs-xpcs.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pcs/pcs-xpcs.h b/include/linux/pcs/pcs-xpcs.h index d2da1e0b4a92..a99972a6d046 100644 --- a/include/linux/pcs/pcs-xpcs.h +++ b/include/linux/pcs/pcs-xpcs.h @@ -37,6 +37,8 @@ int xpcs_config_eee(struct dw_xpcs *xpcs, int mult_fact_100ns, int enable); struct dw_xpcs *xpcs_create(struct mdio_device *mdiodev, phy_interface_t interface); +struct dw_xpcs *xpcs_create_mdiodev(struct mii_bus *bus, int addr, + phy_interface_t interface); void xpcs_destroy(struct dw_xpcs *xpcs); #endif /* __LINUX_PCS_XPCS_H */ -- cgit v1.2.3 From 86b5f2d8cd7828c881036d30ce3a4e711a071726 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 26 May 2023 11:14:39 +0100 Subject: net: pcs: lynx: add lynx_pcs_create_mdiodev() Add lynx_pcs_create_mdiodev() to simplify the creation of the mdio device associated with lynx PCS. In order to allow lynx_pcs_destroy() to clean this up, we need to arrange for lynx_pcs_create() to take a refcount on the mdiodev, and lynx_pcs_destroy() to put it. Adding the refcounting to lynx_pcs_create()..lynx_pcs_destroy() will be transparent to existing users of these interfaces. Signed-off-by: Russell King (Oracle) Reviewed-by: Andrew Lunn Reviewed-by: Ioana Ciornei Tested-by: Ioana Ciornei Signed-off-by: Jakub Kicinski --- include/linux/pcs-lynx.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pcs-lynx.h b/include/linux/pcs-lynx.h index 5712cc2ce775..885b59d10581 100644 --- a/include/linux/pcs-lynx.h +++ b/include/linux/pcs-lynx.h @@ -12,6 +12,7 @@ struct mdio_device *lynx_get_mdio_device(struct phylink_pcs *pcs); struct phylink_pcs *lynx_pcs_create(struct mdio_device *mdio); +struct phylink_pcs *lynx_pcs_create_mdiodev(struct mii_bus *bus, int addr); void lynx_pcs_destroy(struct phylink_pcs *pcs); -- cgit v1.2.3 From 576215cffdefc1f0ceebffd87abb390926e6b037 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 25 May 2023 16:17:10 +0200 Subject: fs: Drop wait_unfrozen wait queue wait_unfrozen waitqueue is used only in quota code to wait for filesystem to become unfrozen. In that place we can just use sb_start_write() - sb_end_write() pair to achieve the same. So just remove the waitqueue. Reviewed-by: Christian Brauner Message-Id: <20230525141710.7595-1-jack@suse.cz> Signed-off-by: Jan Kara --- include/linux/fs.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 21a981680856..3b65a6194485 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1146,7 +1146,6 @@ enum { struct sb_writers { int frozen; /* Is sb frozen? */ - wait_queue_head_t wait_unfrozen; /* wait for thaw */ struct percpu_rw_semaphore rw_sem[SB_FREEZE_LEVELS]; }; -- cgit v1.2.3 From cdb37f73cf05631c4f7401f2cd99878733c0c3d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Wei=C3=9Fschuh?= Date: Tue, 30 May 2023 19:09:58 +0200 Subject: block: constify struct part_type part_type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The struct is never modified so it can be const. Signed-off-by: Thomas Weißschuh Link: https://lore.kernel.org/r/20230419-const-partition-v3-2-4e14e48be367@weissschuh.net Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b2ac587e3402..d89c2da14698 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -41,7 +41,7 @@ struct blk_stat_callback; struct blk_crypto_profile; extern const struct device_type disk_type; -extern struct device_type part_type; +extern const struct device_type part_type; extern struct class block_class; /* -- cgit v1.2.3 From dd06e72e68bcb4070ef211be100d2896e236c8fb Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 17 May 2023 12:08:44 -0700 Subject: Compiler Attributes: Add __counted_by macro In an effort to annotate all flexible array members with their run-time size information, the "element_count" attribute is being introduced by Clang[1] and GCC[2] in future releases. This annotation will provide the CONFIG_UBSAN_BOUNDS and CONFIG_FORTIFY_SOURCE features the ability to perform run-time bounds checking on otherwise unknown-size flexible arrays. Even though the attribute is under development, we can start the annotation process in the kernel. This requires defining a macro for it, even if we have to change the name of the actual attribute later. Since it is likely that this attribute may change its name to "counted_by" in the future (to better align with a future total bytes "sized_by" attribute), name the wrapper macro "__counted_by", which also reads more clearly (and concisely) in structure definitions. [1] https://reviews.llvm.org/D148381 [2] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896 Cc: Bill Wendling Cc: Qing Zhao Cc: Gustavo A. R. Silva Cc: Nick Desaulniers Cc: Nathan Chancellor Cc: Tom Rix Cc: llvm@lists.linux.dev Signed-off-by: Kees Cook Reviewed-by: Gustavo A. R. Silva Reviewed-by: Nathan Chancellor Acked-by: Miguel Ojeda Link: https://lore.kernel.org/r/20230517190841.gonna.796-kees@kernel.org --- include/linux/compiler_attributes.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index e659cb6fded3..5588bffe53b9 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -123,6 +123,19 @@ # define __designated_init #endif +/* + * Optional: only supported since gcc >= 14 + * Optional: only supported since clang >= 17 + * + * gcc: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896 + * clang: https://reviews.llvm.org/D148381 + */ +#if __has_attribute(__element_count__) +# define __counted_by(member) __attribute__((__element_count__(#member))) +#else +# define __counted_by(member) +#endif + /* * Optional: only supported since clang >= 14.0 * -- cgit v1.2.3 From 59272ad8d9e8ea6398a96f8c6d62da284bf2ae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Fri, 10 Mar 2023 23:41:28 +0100 Subject: bus: fsl-mc: Make remove function return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The value returned by an fsl-mc driver's remove function is mostly ignored. (Only an error message is printed if the value is non-zero and then device removal continues unconditionally.) So change the prototype of the remove function to return no value. This way driver authors are not tempted to assume that passing an error to the upper layer is a good idea. All drivers are adapted accordingly. There is no intended change of behaviour, all callbacks were prepared to return 0 before. Signed-off-by: Uwe Kleine-König Reviewed-by: Ioana Ciornei Tested-by: Ioana Ciornei # sanity checks Reviewed-by: Laurentiu Tudor Tested-by: Laurentiu Tudor Signed-off-by: Li Yang --- include/linux/fsl/mc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index a86115bc799c..a1b3de87a3d1 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -48,7 +48,7 @@ struct fsl_mc_driver { struct device_driver driver; const struct fsl_mc_device_id *match_id_table; int (*probe)(struct fsl_mc_device *dev); - int (*remove)(struct fsl_mc_device *dev); + void (*remove)(struct fsl_mc_device *dev); void (*shutdown)(struct fsl_mc_device *dev); int (*suspend)(struct fsl_mc_device *dev, pm_message_t state); int (*resume)(struct fsl_mc_device *dev); -- cgit v1.2.3 From 7b4858df3bf7a8d43ed6b58f411543a040c56f10 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Mon, 29 May 2023 14:48:28 +0300 Subject: skbuff: bridge: Add layer 2 miss indication For EVPN non-DF (Designated Forwarder) filtering we need to be able to prevent decapsulated traffic from being flooded to a multi-homed host. Filtering of multicast and broadcast traffic can be achieved using the following flower filter: # tc filter add dev bond0 egress pref 1 proto all flower indev vxlan0 dst_mac 01:00:00:00:00:00/01:00:00:00:00:00 action drop Unlike broadcast and multicast traffic, it is not currently possible to filter unknown unicast traffic. The classification into unknown unicast is performed by the bridge driver, but is not visible to other layers such as tc. Solve this by adding a new 'l2_miss' bit to the tc skb extension. Clear the bit whenever a packet enters the bridge (received from a bridge port or transmitted via the bridge) and set it if the packet did not match an FDB or MDB entry. If there is no skb extension and the bit needs to be cleared, then do not allocate one as no extension is equivalent to the bit being cleared. The bit is not set for broadcast packets as they never perform a lookup and therefore never incur a miss. A bit that is set for every flooded packet would also work for the current use case, but it does not allow us to differentiate between registered and unregistered multicast traffic, which might be useful in the future. To keep the performance impact to a minimum, the marking of packets is guarded by the 'tc_skb_ext_tc' static key. When 'false', the skb is not touched and an skb extension is not allocated. Instead, only a 5 bytes nop is executed, as demonstrated below for the call site in br_handle_frame(). Before the patch: ``` memset(skb->cb, 0, sizeof(struct br_input_skb_cb)); c37b09: 49 c7 44 24 28 00 00 movq $0x0,0x28(%r12) c37b10: 00 00 p = br_port_get_rcu(skb->dev); c37b12: 49 8b 44 24 10 mov 0x10(%r12),%rax memset(skb->cb, 0, sizeof(struct br_input_skb_cb)); c37b17: 49 c7 44 24 30 00 00 movq $0x0,0x30(%r12) c37b1e: 00 00 c37b20: 49 c7 44 24 38 00 00 movq $0x0,0x38(%r12) c37b27: 00 00 ``` After the patch (when static key is disabled): ``` memset(skb->cb, 0, sizeof(struct br_input_skb_cb)); c37c29: 49 c7 44 24 28 00 00 movq $0x0,0x28(%r12) c37c30: 00 00 c37c32: 49 8d 44 24 28 lea 0x28(%r12),%rax c37c37: 48 c7 40 08 00 00 00 movq $0x0,0x8(%rax) c37c3e: 00 c37c3f: 48 c7 40 10 00 00 00 movq $0x0,0x10(%rax) c37c46: 00 #ifdef CONFIG_HAVE_JUMP_LABEL_HACK static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" c37c47: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) br_tc_skb_miss_set(skb, false); p = br_port_get_rcu(skb->dev); c37c4c: 49 8b 44 24 10 mov 0x10(%r12),%rax ``` Subsequent patches will extend the flower classifier to be able to match on the new 'l2_miss' bit and enable / disable the static key when filters that match on it are added / deleted. Signed-off-by: Ido Schimmel Acked-by: Nikolay Aleksandrov Acked-by: Jakub Kicinski Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 5951904413ab..e2f48ddb2f7c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -330,6 +330,7 @@ struct tc_skb_ext { u8 post_ct_snat:1; u8 post_ct_dnat:1; u8 act_miss:1; /* Set if act_miss_cookie is used */ + u8 l2_miss:1; /* Set by bridge upon FDB or MDB miss */ }; #endif -- cgit v1.2.3 From ed554d3f945179c5b159bddfad7be34b403fe11a Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Mon, 29 May 2023 18:32:31 +0200 Subject: leds: add APIs for LEDs hw control Add an option to permit LED driver to declare support for a specific trigger to use hw control and setup the LED to blink based on specific provided modes. Add APIs for LEDs hw control. These functions will be used to activate hardware control where a LED will use the provided flags, from an unique defined supported trigger, to setup the LED to be driven by hardware. Add hw_control_is_supported() to ask the LED driver if the requested mode by the trigger are supported and the LED can be setup to follow the requested modes. Deactivate hardware blink control by setting brightness to LED_OFF via the brightness_set() callback. Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- include/linux/leds.h | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index c39bbf17a25b..4caf559b1922 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -183,6 +183,43 @@ struct led_classdev { /* LEDs that have private triggers have this set */ struct led_hw_trigger_type *trigger_type; + + /* Unique trigger name supported by LED set in hw control mode */ + const char *hw_control_trigger; + /* + * Check if the LED driver supports the requested mode provided by the + * defined supported trigger to setup the LED to hw control mode. + * + * Return 0 on success. Return -EOPNOTSUPP when the passed flags are not + * supported and software fallback needs to be used. + * Return a negative error number on any other case for check fail due + * to various reason like device not ready or timeouts. + */ + int (*hw_control_is_supported)(struct led_classdev *led_cdev, + unsigned long flags); + /* + * Activate hardware control, LED driver will use the provided flags + * from the supported trigger and setup the LED to be driven by hardware + * following the requested mode from the trigger flags. + * Deactivate hardware blink control by setting brightness to LED_OFF via + * the brightness_set() callback. + * + * Return 0 on success, a negative error number on flags apply fail. + */ + int (*hw_control_set)(struct led_classdev *led_cdev, + unsigned long flags); + /* + * Get from the LED driver the current mode that the LED is set in hw + * control mode and put them in flags. + * Trigger can use this to get the initial state of a LED already set in + * hardware blink control. + * + * Return 0 on success, a negative error number on failing parsing the + * initial mode. Error from this function is NOT FATAL as the device + * may be in a not supported initial state by the attached LED trigger. + */ + int (*hw_control_get)(struct led_classdev *led_cdev, + unsigned long *flags); #endif #ifdef CONFIG_LEDS_BRIGHTNESS_HW_CHANGED -- cgit v1.2.3 From 052c38eb17e866c5b4cd43924e7a5e20167b55c0 Mon Sep 17 00:00:00 2001 From: Andrew Lunn Date: Mon, 29 May 2023 18:32:32 +0200 Subject: leds: add API to get attached device for LED hw control Some specific LED triggers blink the LED based on events from a device or subsystem. For example, an LED could be blinked to indicate a network device is receiving packets, or a disk is reading blocks. To correctly enable and request the hw control of the LED, the trigger has to check if the network interface or block device configured via a /sys/class/led file match the one the LED driver provide for hw control for. Provide an API call to get the device which the LED blinks for. Signed-off-by: Andrew Lunn Signed-off-by: Christian Marangi Signed-off-by: David S. Miller --- include/linux/leds.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index 4caf559b1922..3268b4e789d6 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -220,6 +220,12 @@ struct led_classdev { */ int (*hw_control_get)(struct led_classdev *led_cdev, unsigned long *flags); + /* + * Get the device this LED blinks in response to. + * e.g. for a PHY LED, it is the network device. If the LED is + * not yet associated to a device, return NULL. + */ + struct device *(*hw_control_get_device)(struct led_classdev *led_cdev); #endif #ifdef CONFIG_LEDS_BRIGHTNESS_HW_CHANGED -- cgit v1.2.3 From 947acacab5ea151291b861cdfbde16ff5cf1b08c Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Mon, 29 May 2023 18:32:41 +0200 Subject: leds: trigger: netdev: expose netdev trigger modes in linux include Expose netdev trigger modes to make them accessible by LED driver that will support netdev trigger for hw control. Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- include/linux/leds.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index 3268b4e789d6..8af62ff431f0 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -552,6 +552,16 @@ static inline void *led_get_trigger_data(struct led_classdev *led_cdev) #endif /* CONFIG_LEDS_TRIGGERS */ +/* Trigger specific enum */ +enum led_trigger_netdev_modes { + TRIGGER_NETDEV_LINK = 0, + TRIGGER_NETDEV_TX, + TRIGGER_NETDEV_RX, + + /* Keep last */ + __TRIGGER_NETDEV_MAX, +}; + /* Trigger specific functions */ #ifdef CONFIG_LEDS_TRIGGER_DISK void ledtrig_disk_activity(bool write); -- cgit v1.2.3 From 9b68f30b68701e98abcec331a2cf3df972d910f8 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 26 May 2023 14:21:02 +0300 Subject: net: Use umd_cleanup_helper() bpfilter_umh_cleanup() is the same function as umd_cleanup_helper(). Drop the redundant function. Signed-off-by: Jarkko Sakkinen Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20230526112104.1044686-1-jarkko@kernel.org --- include/linux/bpfilter.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpfilter.h b/include/linux/bpfilter.h index 2ae3c8e1d83c..736ded4905e0 100644 --- a/include/linux/bpfilter.h +++ b/include/linux/bpfilter.h @@ -11,7 +11,6 @@ int bpfilter_ip_set_sockopt(struct sock *sk, int optname, sockptr_t optval, unsigned int optlen); int bpfilter_ip_get_sockopt(struct sock *sk, int optname, char __user *optval, int __user *optlen); -void bpfilter_umh_cleanup(struct umd_info *info); struct bpfilter_umh_ops { struct umd_info info; -- cgit v1.2.3 From c8070b78751955e59b42457b974bea4a4fe00187 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 26 May 2023 22:41:40 +0100 Subject: mm: Don't pin ZERO_PAGE in pin_user_pages() Make pin_user_pages*() leave a ZERO_PAGE unpinned if it extracts a pointer to it from the page tables and make unpin_user_page*() correspondingly ignore a ZERO_PAGE when unpinning. We don't want to risk overrunning a zero page's refcount as we're only allowed ~2 million pins on it - something that userspace can conceivably trigger. Add a pair of functions to test whether a page or a folio is a ZERO_PAGE. Signed-off-by: David Howells cc: Christoph Hellwig cc: David Hildenbrand cc: Lorenzo Stoakes cc: Andrew Morton cc: Jens Axboe cc: Al Viro cc: Matthew Wilcox cc: Jan Kara cc: Jeff Layton cc: Jason Gunthorpe cc: Logan Gunthorpe cc: Hillf Danton cc: Christian Brauner cc: Linus Torvalds cc: linux-fsdevel@vger.kernel.org cc: linux-block@vger.kernel.org cc: linux-kernel@vger.kernel.org cc: linux-mm@kvack.org Reviewed-by: Lorenzo Stoakes Reviewed-by: Christoph Hellwig Acked-by: David Hildenbrand Link: https://lore.kernel.org/r/20230526214142.958751-2-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/mm.h | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 27ce77080c79..3c2f6b452586 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1910,6 +1910,28 @@ static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma, return page_maybe_dma_pinned(page); } +/** + * is_zero_page - Query if a page is a zero page + * @page: The page to query + * + * This returns true if @page is one of the permanent zero pages. + */ +static inline bool is_zero_page(const struct page *page) +{ + return is_zero_pfn(page_to_pfn(page)); +} + +/** + * is_zero_folio - Query if a folio is a zero page + * @folio: The folio to query + * + * This returns true if @folio is one of the permanent zero pages. + */ +static inline bool is_zero_folio(const struct folio *folio) +{ + return is_zero_page(&folio->page); +} + /* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */ #ifdef CONFIG_MIGRATION static inline bool is_longterm_pinnable_page(struct page *page) @@ -1920,8 +1942,8 @@ static inline bool is_longterm_pinnable_page(struct page *page) if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) return false; #endif - /* The zero page may always be pinned */ - if (is_zero_pfn(page_to_pfn(page))) + /* The zero page can be "pinned" but gets special handling. */ + if (is_zero_page(page)) return true; /* Coherent device memory must always allow eviction. */ -- cgit v1.2.3 From 1101fb8f89e5fc548c4d0ad66750e98980291815 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 26 May 2023 22:41:41 +0100 Subject: mm: Provide a function to get an additional pin on a page Provide a function to get an additional pin on a page that we already have a pin on. This will be used in fs/direct-io.c when dispatching multiple bios to a page we've extracted from a user-backed iter rather than redoing the extraction. Signed-off-by: David Howells cc: Christoph Hellwig cc: David Hildenbrand cc: Lorenzo Stoakes cc: Andrew Morton cc: Jens Axboe cc: Al Viro cc: Matthew Wilcox cc: Jan Kara cc: Jeff Layton cc: Jason Gunthorpe cc: Logan Gunthorpe cc: Hillf Danton cc: Christian Brauner cc: Linus Torvalds cc: linux-fsdevel@vger.kernel.org cc: linux-block@vger.kernel.org cc: linux-kernel@vger.kernel.org cc: linux-mm@kvack.org Reviewed-by: Christoph Hellwig Acked-by: David Hildenbrand Link: https://lore.kernel.org/r/20230526214142.958751-3-dhowells@redhat.com Signed-off-by: Jens Axboe --- include/linux/mm.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 3c2f6b452586..200068d98686 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2405,6 +2405,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages); int pin_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages); +void folio_add_pin(struct folio *folio); int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, -- cgit v1.2.3 From b7a7ce1bb77b19ff2859d365da96285340fbc145 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:11:02 +0200 Subject: vdso/timens: Always provide arch_get_vdso_data() prototype for vdso The arch_get_vdso_data() function is defined separately on each architecture, but only called when CONFIG_TIME_NS is set. If the definition is a global function, this causes a W=1 warning without TIME_NS: arch/x86/entry/vdso/vma.c:35:19: error: no previous prototype for 'arch_get_vdso_data' [-Werror=missing-prototypes] Move the prototype out of the #ifdef block to reliably turn off that warning. Signed-off-by: Arnd Bergmann Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230517131102.934196-15-arnd@kernel.org --- include/linux/time_namespace.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index bb9d3f5542f8..03d9c5ac01d1 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -44,7 +44,6 @@ struct time_namespace *copy_time_ns(unsigned long flags, struct time_namespace *old_ns); void free_time_ns(struct time_namespace *ns); void timens_on_fork(struct nsproxy *nsproxy, struct task_struct *tsk); -struct vdso_data *arch_get_vdso_data(void *vvar_page); struct page *find_timens_vvar_page(struct vm_area_struct *vma); static inline void put_time_ns(struct time_namespace *ns) @@ -163,4 +162,6 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim) } #endif +struct vdso_data *arch_get_vdso_data(void *vvar_page); + #endif /* _LINUX_TIMENS_H */ -- cgit v1.2.3 From 2f804aca48322f02a8f44cca540663845ee80fb1 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 30 May 2023 01:14:45 +0300 Subject: gpiolib: Kill unused GPIOF_EXPORT and Co There is no use of the GPIOF_EXPORT in the kernel. Kill it for good. Signed-off-by: Andy Shevchenko Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- include/linux/gpio.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gpio.h b/include/linux/gpio.h index 8528353e073b..86963a00b018 100644 --- a/include/linux/gpio.h +++ b/include/linux/gpio.h @@ -38,11 +38,6 @@ struct device; /* Gpio pin is open source */ #define GPIOF_OPEN_SOURCE (1 << 4) -#define GPIOF_EXPORT (1 << 5) -#define GPIOF_EXPORT_CHANGEABLE (1 << 6) -#define GPIOF_EXPORT_DIR_FIXED (GPIOF_EXPORT) -#define GPIOF_EXPORT_DIR_CHANGEABLE (GPIOF_EXPORT | GPIOF_EXPORT_CHANGEABLE) - /** * struct gpio - a structure describing a GPIO with configuration * @gpio: the GPIO number -- cgit v1.2.3 From 9df8c63c2b814be8c40d44d21dabaf074058c98b Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Tue, 30 May 2023 01:14:46 +0300 Subject: gpiolib: Kill unused GPIOF_OPEN_* There is no use of the GPIOF_OPEN_* in the kernel. Kill it for good. Signed-off-by: Andy Shevchenko Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- include/linux/gpio.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gpio.h b/include/linux/gpio.h index 86963a00b018..88efac969754 100644 --- a/include/linux/gpio.h +++ b/include/linux/gpio.h @@ -32,12 +32,6 @@ struct device; /* Gpio pin is active-low */ #define GPIOF_ACTIVE_LOW (1 << 2) -/* Gpio pin is open drain */ -#define GPIOF_OPEN_DRAIN (1 << 3) - -/* Gpio pin is open source */ -#define GPIOF_OPEN_SOURCE (1 << 4) - /** * struct gpio - a structure describing a GPIO with configuration * @gpio: the GPIO number -- cgit v1.2.3 From f1061fa641b8b15c7815d58c20a6c29f2f1f5337 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Tue, 30 May 2023 17:12:17 +0200 Subject: fbdev: Add initializer macros for struct fb_ops For framebuffers in I/O and system memory, add macros that set struct fb_ops to the respective callback functions. For deferred I/O, add macros that generate callback functions with damage handling. Add initializer macros that set struct fb_ops to the generated callbacks. These macros can remove a lot boilerplate code from fbdev drivers. The drivers are supposed to use the macro that is required for its framebuffer. Each macro is split into smaller helpers, so that drivers with non-standard callbacks can pick and customize callbacks as needed. There are individual helper macros for read/write, mmap and drawing. v5: * fix whitespace errors (Jingfeng) Signed-off-by: Thomas Zimmermann Reviewed-by: Sam Ravnborg Link: https://patchwork.freedesktop.org/patch/msgid/20230530151228.22979-3-tzimmermann@suse.de --- include/linux/fb.h | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index 2cf8efcb9e32..ce6823e157e6 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -538,9 +538,31 @@ extern ssize_t fb_io_read(struct fb_info *info, char __user *buf, extern ssize_t fb_io_write(struct fb_info *info, const char __user *buf, size_t count, loff_t *ppos); +/* + * Initializes struct fb_ops for framebuffers in I/O memory. + */ + +#define __FB_DEFAULT_IO_OPS_RDWR \ + .fb_read = fb_io_read, \ + .fb_write = fb_io_write + +#define __FB_DEFAULT_IO_OPS_DRAW \ + .fb_fillrect = cfb_fillrect, \ + .fb_copyarea = cfb_copyarea, \ + .fb_imageblit = cfb_imageblit + +#define __FB_DEFAULT_IO_OPS_MMAP \ + .fb_mmap = NULL // default implementation + +#define FB_DEFAULT_IO_OPS \ + __FB_DEFAULT_IO_OPS_RDWR, \ + __FB_DEFAULT_IO_OPS_DRAW, \ + __FB_DEFAULT_IO_OPS_MMAP + /* * Drawing operations where framebuffer is in system RAM */ + extern void sys_fillrect(struct fb_info *info, const struct fb_fillrect *rect); extern void sys_copyarea(struct fb_info *info, const struct fb_copyarea *area); extern void sys_imageblit(struct fb_info *info, const struct fb_image *image); @@ -549,6 +571,27 @@ extern ssize_t fb_sys_read(struct fb_info *info, char __user *buf, extern ssize_t fb_sys_write(struct fb_info *info, const char __user *buf, size_t count, loff_t *ppos); +/* + * Initializes struct fb_ops for framebuffers in system memory. + */ + +#define __FB_DEFAULT_SYS_OPS_RDWR \ + .fb_read = fb_sys_read, \ + .fb_write = fb_sys_write + +#define __FB_DEFAULT_SYS_OPS_DRAW \ + .fb_fillrect = sys_fillrect, \ + .fb_copyarea = sys_copyarea, \ + .fb_imageblit = sys_imageblit + +#define __FB_DEFAULT_SYS_OPS_MMAP \ + .fb_mmap = NULL // default implementation + +#define FB_DEFAULT_SYS_OPS \ + __FB_DEFAULT_SYS_OPS_RDWR, \ + __FB_DEFAULT_SYS_OPS_DRAW, \ + __FB_DEFAULT_SYS_OPS_MMAP + /* drivers/video/fbmem.c */ extern int register_framebuffer(struct fb_info *fb_info); extern void unregister_framebuffer(struct fb_info *fb_info); @@ -604,6 +647,75 @@ extern void fb_deferred_io_cleanup(struct fb_info *info); extern int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasync); +/* + * Generate callbacks for deferred I/O + */ + +#define __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, __mode) \ + static ssize_t __prefix ## _defio_read(struct fb_info *info, char __user *buf, \ + size_t count, loff_t *ppos) \ + { \ + return fb_ ## __mode ## _read(info, buf, count, ppos); \ + } \ + static ssize_t __prefix ## _defio_write(struct fb_info *info, const char __user *buf, \ + size_t count, loff_t *ppos) \ + { \ + unsigned long offset = *ppos; \ + ssize_t ret = fb_ ## __mode ## _write(info, buf, count, ppos); \ + if (ret > 0) \ + __damage_range(info, offset, ret); \ + return ret; \ + } + +#define __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, __mode) \ + static void __prefix ## _defio_fillrect(struct fb_info *info, \ + const struct fb_fillrect *rect) \ + { \ + __mode ## _fillrect(info, rect); \ + __damage_area(info, rect->dx, rect->dy, rect->width, rect->height); \ + } \ + static void __prefix ## _defio_copyarea(struct fb_info *info, \ + const struct fb_copyarea *area) \ + { \ + __mode ## _copyarea(info, area); \ + __damage_area(info, area->dx, area->dy, area->width, area->height); \ + } \ + static void __prefix ## _defio_imageblit(struct fb_info *info, \ + const struct fb_image *image) \ + { \ + __mode ## _imageblit(info, image); \ + __damage_area(info, image->dx, image->dy, image->width, image->height); \ + } + +#define FB_GEN_DEFAULT_DEFERRED_IO_OPS(__prefix, __damage_range, __damage_area) \ + __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, io) \ + __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, cfb) + +#define FB_GEN_DEFAULT_DEFERRED_SYS_OPS(__prefix, __damage_range, __damage_area) \ + __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, sys) \ + __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, sys) + +/* + * Initializes struct fb_ops for deferred I/O. + */ + +#define __FB_DEFAULT_DEFERRED_OPS_RDWR(__prefix) \ + .fb_read = __prefix ## _defio_read, \ + .fb_write = __prefix ## _defio_write + +#define __FB_DEFAULT_DEFERRED_OPS_DRAW(__prefix) \ + .fb_fillrect = __prefix ## _defio_fillrect, \ + .fb_copyarea = __prefix ## _defio_copyarea, \ + .fb_imageblit = __prefix ## _defio_imageblit + +#define __FB_DEFAULT_DEFERRED_OPS_MMAP(__prefix) \ + .fb_mmap = fb_deferred_io_mmap + +#define FB_DEFAULT_DEFERRED_OPS(__prefix) \ + __FB_DEFAULT_DEFERRED_OPS_RDWR(__prefix), \ + __FB_DEFAULT_DEFERRED_OPS_DRAW(__prefix), \ + __FB_DEFAULT_DEFERRED_OPS_MMAP(__prefix) + static inline bool fb_be_math(struct fb_info *info) { #ifdef CONFIG_FB_FOREIGN_ENDIAN -- cgit v1.2.3 From 83f2caaaf9cb25fe74775a59bf2662f184bfaa08 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 31 May 2023 04:50:40 -0700 Subject: block: mark bio_add_page as __must_check Now that all users of bio_add_page check for the return value, mark bio_add_page as __must_check. Reviewed-by: Damien Le Moal Signed-off-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/7ae4a902e08fe2e90c012ee07aeb35d4aae28373.1685532726.git.johannes.thumshirn@wdc.com Signed-off-by: Jens Axboe --- include/linux/bio.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index 8588bcfbc6ef..d63f0bb47c65 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -465,7 +465,8 @@ extern void bio_uninit(struct bio *); void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf); void bio_chain(struct bio *, struct bio *); -int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off); +int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len, + unsigned off); bool bio_add_folio(struct bio *, struct folio *, size_t len, size_t off); extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int); -- cgit v1.2.3 From 7a150f1ed19b709837e98571f49ab1ff2625ca89 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 31 May 2023 04:50:41 -0700 Subject: block: add bio_add_folio_nofail Just like for bio_add_pages() add a no-fail variant for bio_add_folio(). Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/924dff4077812804398ef84128fb920507fa4be1.1685532726.git.johannes.thumshirn@wdc.com Signed-off-by: Jens Axboe --- include/linux/bio.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index d63f0bb47c65..0002bd78e02d 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -474,6 +474,8 @@ int bio_add_zone_append_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset); void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off); +void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, + size_t off); int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter); void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); -- cgit v1.2.3 From 6c500000af037f74b66dd01b565c8ee1b501cc1b Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 31 May 2023 04:50:43 -0700 Subject: block: mark bio_add_folio as __must_check Now that all callers of bio_add_folio() check the return value, mark it as __must_check. Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/381360a45ac3684120cfbe1e07685e9c36256e47.1685532726.git.johannes.thumshirn@wdc.com Signed-off-by: Jens Axboe --- include/linux/bio.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index 0002bd78e02d..617522928964 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -467,7 +467,8 @@ void bio_chain(struct bio *, struct bio *); int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len, unsigned off); -bool bio_add_folio(struct bio *, struct folio *, size_t len, size_t off); +bool __must_check bio_add_folio(struct bio *bio, struct folio *folio, + size_t len, size_t off); extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int); int bio_add_zone_append_page(struct bio *bio, struct page *page, -- cgit v1.2.3 From e8b6f79b41840c542b7ef45c16b31dd17e1cc6e1 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Wed, 31 May 2023 16:17:29 -0700 Subject: net: phy: broadcom: Add LPI counter Add the ability to read the PHY maintained LPI counter which is in the Clause 45 vendor space, device address 7, offset 0x803F. The counter is cleared on read. Signed-off-by: Florian Fainelli Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20230531231729.1873932-1-florian.fainelli@broadcom.com Signed-off-by: Jakub Kicinski --- include/linux/brcmphy.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index e9afbfb6d7a5..251833ab271f 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -359,6 +359,8 @@ #define LPI_FEATURE_EN 0x8000 #define LPI_FEATURE_EN_DIG1000X 0x4000 +#define BRCM_CL45VEN_EEE_LPI_CNT 0x803f + /* Core register definitions*/ #define MII_BRCM_CORE_BASE12 0x12 #define MII_BRCM_CORE_BASE13 0x13 -- cgit v1.2.3 From d0bf7d5759c1d89fb013aa41cca5832e00b9632a Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Tue, 17 Jan 2023 14:40:00 +0100 Subject: mm/slab: introduce kmem_cache flag SLAB_NO_MERGE Allow API users of kmem_cache_create to specify that they don't want any slab merge or aliasing (with similar sized objects). Use this in kfence_test. The SKB (sk_buff) kmem_cache slab is critical for network performance. Network stack uses kmem_cache_{alloc,free}_bulk APIs to gain performance by amortising the alloc/free cost. For the bulk API to perform efficiently the slub fragmentation need to be low. Especially for the SLUB allocator, the efficiency of bulk free API depend on objects belonging to the same slab (page). When running different network performance microbenchmarks, I started to notice that performance was reduced (slightly) when machines had longer uptimes. I believe the cause was 'skbuff_head_cache' got aliased/merged into the general slub for 256 bytes sized objects (with my kernel config, without CONFIG_HARDENED_USERCOPY). For SKB kmem_cache network stack have reasons for not merging, but it varies depending on kernel config (e.g. CONFIG_HARDENED_USERCOPY). We want to explicitly set SLAB_NO_MERGE for this kmem_cache. Another use case for the flag has been described by David Sterba [1]: > This can be used for more fine grained control over the caches or for > debugging builds where separate slabs can verify that no objects leak. > The slab_nomerge boot option is too coarse and would need to be > enabled on all testing hosts. There are some other ways how to disable > merging, e.g. a slab constructor but this disables poisoning besides > that it adds additional overhead. Other flags are internal and may > have other semantics. > A concrete example what motivates the flag. During 'btrfs balance' > slab top reported huge increase in caches like > 1330095 1330095 100% 0.10K 34105 39 136420K Acpi-ParseExt > 1734684 1734684 100% 0.14K 61953 28 247812K pid_namespace > 8244036 6873075 83% 0.11K 229001 36 916004K khugepaged_mm_slot > which was confusing and that it's because of slab merging was not the > first idea. After rebooting with slab_nomerge all the caches were > from btrfs_ namespace as expected. [1] https://lore.kernel.org/all/20230524101748.30714-1-dsterba@suse.com/ [ vbabka@suse.cz: rename to SLAB_NO_MERGE, change the flag value to the one proposed by David so it does not collide with internal SLAB/SLUB flags, write a comment for the flag, expand changelog, drop the skbuff part to be handled spearately ] Link: https://lore.kernel.org/all/167396280045.539803.7540459812377220500.stgit@firesoul/ Reported-by: David Sterba Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Vlastimil Babka Acked-by: Jesper Dangaard Brouer Acked-by: Roman Gushchin --- include/linux/slab.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index 6b3e155b70bf..72bc906d8bc7 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -106,6 +106,18 @@ /* Avoid kmemleak tracing */ #define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) +/* + * Prevent merging with compatible kmem caches. This flag should be used + * cautiously. Valid use cases: + * + * - caches created for self-tests (e.g. kunit) + * - general caches created and used by a subsystem, only when a + * (subsystem-specific) debug option is enabled + * - performance critical caches, should be very rare and consulted with slab + * maintainers, and not used together with CONFIG_SLUB_TINY + */ +#define SLAB_NO_MERGE ((slab_flags_t __force)0x01000000U) + /* Fault injection mark */ #ifdef CONFIG_FAILSLAB # define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) -- cgit v1.2.3 From 4ec7329517027db28c5683675ab3b3842ad60324 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Thu, 1 Jun 2023 16:48:12 +0100 Subject: net: phylib: fix phy_read*_poll_timeout() Dan Carpenter reported a signedness bug in genphy_loopback(). Andrew reports that: "It is common to get this wrong in general with PHY drivers. Dan regularly posts fixes like this soon after a PHY driver patch it merged. I really wish we could somehow get the compiler to warn when the result from phy_read() is stored into a unsigned type. It would save Dan a lot of work." Let's make phy_read*_poll_timeout() immune to further issues when "val" is an unsigned type by storing the read function's result in a signed int as well as "val", and using the signed variable both to check for an error and for propagating that error to the caller. The advantage of this method is we don't change where the cast from the signed return code to the user's variable occurs - so users will see no change. Previously Heiner changed phy_read_poll_timeout() to check for an error before evaluating the user supplied condition, but didn't update phy_read_mmd_poll_timeout(). Make that change there too. Link: https://lore.kernel.org/r/d7bb312e-2428-45f6-b9b3-59ba544e8b94@kili.mountain Signed-off-by: Russell King (Oracle) Link: https://lore.kernel.org/r/E1q4kX6-00BNuM-Mx@rmk-PC.armlinux.org.uk Signed-off-by: Jakub Kicinski --- include/linux/phy.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phy.h b/include/linux/phy.h index 7addde5d14c0..11c1e91563d4 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -1206,10 +1206,12 @@ static inline int phy_read(struct phy_device *phydev, u32 regnum) #define phy_read_poll_timeout(phydev, regnum, val, cond, sleep_us, \ timeout_us, sleep_before_read) \ ({ \ - int __ret = read_poll_timeout(phy_read, val, val < 0 || (cond), \ + int __ret, __val; \ + __ret = read_poll_timeout(__val = phy_read, val, \ + __val < 0 || (cond), \ sleep_us, timeout_us, sleep_before_read, phydev, regnum); \ - if (val < 0) \ - __ret = val; \ + if (__val < 0) \ + __ret = __val; \ if (__ret) \ phydev_err(phydev, "%s failed: %d\n", __func__, __ret); \ __ret; \ @@ -1302,11 +1304,13 @@ int phy_read_mmd(struct phy_device *phydev, int devad, u32 regnum); #define phy_read_mmd_poll_timeout(phydev, devaddr, regnum, val, cond, \ sleep_us, timeout_us, sleep_before_read) \ ({ \ - int __ret = read_poll_timeout(phy_read_mmd, val, (cond) || val < 0, \ + int __ret, __val; \ + __ret = read_poll_timeout(__val = phy_read_mmd, val, \ + __val < 0 || (cond), \ sleep_us, timeout_us, sleep_before_read, \ phydev, devaddr, regnum); \ - if (val < 0) \ - __ret = val; \ + if (__val < 0) \ + __ret = __val; \ if (__ret) \ phydev_err(phydev, "%s failed: %d\n", __func__, __ret); \ __ret; \ -- cgit v1.2.3 From 224d80c584d3016cb8d83d1c33914fdd3508aa8c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 31 May 2023 15:08:35 +0200 Subject: types: Introduce [us]128 Introduce [us]128 (when available). Unlike [us]64, ensure they are always naturally aligned. This also enables 128bit wide atomics (which require natural alignment) such as cmpxchg128(). Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Arnd Bergmann Reviewed-by: Mark Rutland Acked-by: Herbert Xu Tested-by: Mark Rutland Link: https://lore.kernel.org/r/20230531132323.385005581@infradead.org --- include/linux/types.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/types.h b/include/linux/types.h index 688fb943556a..becb8cd5916f 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -10,6 +10,11 @@ #define DECLARE_BITMAP(name,bits) \ unsigned long name[BITS_TO_LONGS(bits)] +#ifdef __SIZEOF_INT128__ +typedef __s128 s128; +typedef __u128 u128; +#endif + typedef u32 __kernel_dev_t; typedef __kernel_fd_set fd_set; -- cgit v1.2.3 From 8c8b096a23d12fedf3c0f50524f30113ef97aa8c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 31 May 2023 15:08:37 +0200 Subject: instrumentation: Wire up cmpxchg128() Wire up the cmpxchg128 family in the atomic wrapper scripts. These provide the generic cmpxchg128 family of functions from the arch_ prefixed version, adding explicit instrumentation where needed. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Arnd Bergmann Reviewed-by: Mark Rutland Acked-by: Mark Rutland Tested-by: Mark Rutland Link: https://lore.kernel.org/r/20230531132323.519237070@infradead.org --- include/linux/atomic/atomic-arch-fallback.h | 95 ++++++++++++++++++++++++++++- include/linux/atomic/atomic-instrumented.h | 86 +++++++++++++++++++++++++- 2 files changed, 179 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index a6e4437c5f36..1722ddb6f17e 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -77,6 +77,29 @@ #endif /* arch_cmpxchg64_relaxed */ +#ifndef arch_cmpxchg128_relaxed +#define arch_cmpxchg128_acquire arch_cmpxchg128 +#define arch_cmpxchg128_release arch_cmpxchg128 +#define arch_cmpxchg128_relaxed arch_cmpxchg128 +#else /* arch_cmpxchg128_relaxed */ + +#ifndef arch_cmpxchg128_acquire +#define arch_cmpxchg128_acquire(...) \ + __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg128_release +#define arch_cmpxchg128_release(...) \ + __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg128 +#define arch_cmpxchg128(...) \ + __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) +#endif + +#endif /* arch_cmpxchg128_relaxed */ + #ifndef arch_try_cmpxchg_relaxed #ifdef arch_try_cmpxchg #define arch_try_cmpxchg_acquire arch_try_cmpxchg @@ -217,6 +240,76 @@ #endif /* arch_try_cmpxchg64_relaxed */ +#ifndef arch_try_cmpxchg128_relaxed +#ifdef arch_try_cmpxchg128 +#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128 +#define arch_try_cmpxchg128_release arch_try_cmpxchg128 +#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128 +#endif /* arch_try_cmpxchg128 */ + +#ifndef arch_try_cmpxchg128 +#define arch_try_cmpxchg128(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128 */ + +#ifndef arch_try_cmpxchg128_acquire +#define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_acquire */ + +#ifndef arch_try_cmpxchg128_release +#define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_release */ + +#ifndef arch_try_cmpxchg128_relaxed +#define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) +#endif /* arch_try_cmpxchg128_relaxed */ + +#else /* arch_try_cmpxchg128_relaxed */ + +#ifndef arch_try_cmpxchg128_acquire +#define arch_try_cmpxchg128_acquire(...) \ + __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg128_release +#define arch_try_cmpxchg128_release(...) \ + __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#ifndef arch_try_cmpxchg128 +#define arch_try_cmpxchg128(...) \ + __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) +#endif + +#endif /* arch_try_cmpxchg128_relaxed */ + #ifndef arch_try_cmpxchg_local #define arch_try_cmpxchg_local(_ptr, _oldp, _new) \ ({ \ @@ -2668,4 +2761,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) #endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// ad2e2b4d168dbc60a73922616047a9bfa446af36 +// 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53 diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 03a232a1fa57..858372096d5c 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -2034,6 +2034,36 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ }) +#define cmpxchg128(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kcsan_mb(); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg128(__ai_ptr, __VA_ARGS__); \ +}) + +#define cmpxchg128_acquire(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \ +}) + +#define cmpxchg128_release(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + kcsan_release(); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \ +}) + +#define cmpxchg128_relaxed(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \ +}) + #define try_cmpxchg(ptr, oldp, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -2110,6 +2140,44 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) +#define try_cmpxchg128(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + kcsan_mb(); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg128_acquire(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg128_release(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + kcsan_release(); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + +#define try_cmpxchg128_relaxed(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + #define cmpxchg_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -2124,6 +2192,13 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ }) +#define cmpxchg128_local(ptr, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + arch_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \ +}) + #define sync_cmpxchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -2150,6 +2225,15 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) +#define try_cmpxchg128_local(ptr, oldp, ...) \ +({ \ + typeof(ptr) __ai_ptr = (ptr); \ + typeof(oldp) __ai_oldp = (oldp); \ + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ + arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ +}) + #define cmpxchg_double(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ @@ -2167,4 +2251,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) }) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e +// 82d1be694fab30414527d0877c29fa75ed5a0b74 -- cgit v1.2.3 From c5c0ba953b8c969c5d51bf1c57f239866a97c47c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 31 May 2023 15:08:38 +0200 Subject: percpu: Add {raw,this}_cpu_try_cmpxchg() Add the try_cmpxchg() form to the per-cpu ops. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Mark Rutland Tested-by: Mark Rutland Link: https://lore.kernel.org/r/20230531132323.587480729@infradead.org --- include/linux/percpu-defs.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index e60727be79c4..cbbf6d189a3a 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -343,6 +343,21 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } pscr2_ret__; \ }) +#define __pcpu_size_call_return2bool(stem, variable, ...) \ +({ \ + bool pscr2_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ + case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ + case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ + case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pscr2_ret__; \ +}) + /* * Special handling for cmpxchg_double. cmpxchg_double is passed two * percpu variables. The first has to be aligned to a double word @@ -426,6 +441,8 @@ do { \ #define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval) #define raw_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) +#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval) #define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) @@ -513,6 +530,8 @@ do { \ #define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval) #define this_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) +#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \ + __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval) #define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) -- cgit v1.2.3 From b1fe7f2cda2a003afe316ce8dfe8d3645694a67e Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 31 May 2023 15:08:42 +0200 Subject: x86,intel_iommu: Replace cmpxchg_double() Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Lu Baolu Reviewed-by: Arnd Bergmann Reviewed-by: Mark Rutland Tested-by: Mark Rutland Link: https://lore.kernel.org/r/20230531132323.855976804@infradead.org --- include/linux/dmar.h | 125 +++++++++++++++++++++++++++------------------------ 1 file changed, 66 insertions(+), 59 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 725d5e6acec0..27dbd4c64860 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -202,67 +202,74 @@ static inline void detect_intel_iommu(void) struct irte { union { - /* Shared between remapped and posted mode*/ struct { - __u64 present : 1, /* 0 */ - fpd : 1, /* 1 */ - __res0 : 6, /* 2 - 6 */ - avail : 4, /* 8 - 11 */ - __res1 : 3, /* 12 - 14 */ - pst : 1, /* 15 */ - vector : 8, /* 16 - 23 */ - __res2 : 40; /* 24 - 63 */ + union { + /* Shared between remapped and posted mode*/ + struct { + __u64 present : 1, /* 0 */ + fpd : 1, /* 1 */ + __res0 : 6, /* 2 - 6 */ + avail : 4, /* 8 - 11 */ + __res1 : 3, /* 12 - 14 */ + pst : 1, /* 15 */ + vector : 8, /* 16 - 23 */ + __res2 : 40; /* 24 - 63 */ + }; + + /* Remapped mode */ + struct { + __u64 r_present : 1, /* 0 */ + r_fpd : 1, /* 1 */ + dst_mode : 1, /* 2 */ + redir_hint : 1, /* 3 */ + trigger_mode : 1, /* 4 */ + dlvry_mode : 3, /* 5 - 7 */ + r_avail : 4, /* 8 - 11 */ + r_res0 : 4, /* 12 - 15 */ + r_vector : 8, /* 16 - 23 */ + r_res1 : 8, /* 24 - 31 */ + dest_id : 32; /* 32 - 63 */ + }; + + /* Posted mode */ + struct { + __u64 p_present : 1, /* 0 */ + p_fpd : 1, /* 1 */ + p_res0 : 6, /* 2 - 7 */ + p_avail : 4, /* 8 - 11 */ + p_res1 : 2, /* 12 - 13 */ + p_urgent : 1, /* 14 */ + p_pst : 1, /* 15 */ + p_vector : 8, /* 16 - 23 */ + p_res2 : 14, /* 24 - 37 */ + pda_l : 26; /* 38 - 63 */ + }; + __u64 low; + }; + + union { + /* Shared between remapped and posted mode*/ + struct { + __u64 sid : 16, /* 64 - 79 */ + sq : 2, /* 80 - 81 */ + svt : 2, /* 82 - 83 */ + __res3 : 44; /* 84 - 127 */ + }; + + /* Posted mode*/ + struct { + __u64 p_sid : 16, /* 64 - 79 */ + p_sq : 2, /* 80 - 81 */ + p_svt : 2, /* 82 - 83 */ + p_res3 : 12, /* 84 - 95 */ + pda_h : 32; /* 96 - 127 */ + }; + __u64 high; + }; }; - - /* Remapped mode */ - struct { - __u64 r_present : 1, /* 0 */ - r_fpd : 1, /* 1 */ - dst_mode : 1, /* 2 */ - redir_hint : 1, /* 3 */ - trigger_mode : 1, /* 4 */ - dlvry_mode : 3, /* 5 - 7 */ - r_avail : 4, /* 8 - 11 */ - r_res0 : 4, /* 12 - 15 */ - r_vector : 8, /* 16 - 23 */ - r_res1 : 8, /* 24 - 31 */ - dest_id : 32; /* 32 - 63 */ - }; - - /* Posted mode */ - struct { - __u64 p_present : 1, /* 0 */ - p_fpd : 1, /* 1 */ - p_res0 : 6, /* 2 - 7 */ - p_avail : 4, /* 8 - 11 */ - p_res1 : 2, /* 12 - 13 */ - p_urgent : 1, /* 14 */ - p_pst : 1, /* 15 */ - p_vector : 8, /* 16 - 23 */ - p_res2 : 14, /* 24 - 37 */ - pda_l : 26; /* 38 - 63 */ - }; - __u64 low; - }; - - union { - /* Shared between remapped and posted mode*/ - struct { - __u64 sid : 16, /* 64 - 79 */ - sq : 2, /* 80 - 81 */ - svt : 2, /* 82 - 83 */ - __res3 : 44; /* 84 - 127 */ - }; - - /* Posted mode*/ - struct { - __u64 p_sid : 16, /* 64 - 79 */ - p_sq : 2, /* 80 - 81 */ - p_svt : 2, /* 82 - 83 */ - p_res3 : 12, /* 84 - 95 */ - pda_h : 32; /* 96 - 127 */ - }; - __u64 high; +#ifdef CONFIG_IRQ_REMAP + __u128 irte; +#endif }; }; -- cgit v1.2.3 From 6801be4f2653e5fdddca73b527cf0728284ba8a3 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 31 May 2023 15:08:43 +0200 Subject: slub: Replace cmpxchg_double() Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Arnd Bergmann Reviewed-by: Mark Rutland Acked-by: Vlastimil Babka Acked-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Tested-by: Mark Rutland Link: https://lore.kernel.org/r/20230531132323.924677086@infradead.org --- include/linux/slub_def.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index f6df03f934e5..deb90cf4bffb 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -39,7 +39,8 @@ enum stat_item { CPU_PARTIAL_FREE, /* Refill cpu partial on free */ CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */ CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */ - NR_SLUB_STAT_ITEMS }; + NR_SLUB_STAT_ITEMS +}; #ifndef CONFIG_SLUB_TINY /* @@ -47,8 +48,13 @@ enum stat_item { * with this_cpu_cmpxchg_double() alignment requirements. */ struct kmem_cache_cpu { - void **freelist; /* Pointer to next available object */ - unsigned long tid; /* Globally unique transaction id */ + union { + struct { + void **freelist; /* Pointer to next available object */ + unsigned long tid; /* Globally unique transaction id */ + }; + freelist_aba_t freelist_tid; + }; struct slab *slab; /* The slab from which we are allocating */ #ifdef CONFIG_SLUB_CPU_PARTIAL struct slab *partial; /* Partially allocated frozen slabs */ -- cgit v1.2.3 From febe950dbfb464799beb0339cc6fb10699f4a5da Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 31 May 2023 15:08:44 +0200 Subject: arch: Remove cmpxchg_double No moar users, remove the monster. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Arnd Bergmann Reviewed-by: Mark Rutland Acked-by: Heiko Carstens Tested-by: Mark Rutland Link: https://lore.kernel.org/r/20230531132323.991907085@infradead.org --- include/linux/atomic/atomic-instrumented.h | 17 +------------ include/linux/percpu-defs.h | 38 ------------------------------ 2 files changed, 1 insertion(+), 54 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 858372096d5c..a55b5b70a3e1 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -2234,21 +2234,6 @@ atomic_long_dec_if_positive(atomic_long_t *v) arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) -#define cmpxchg_double(ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - kcsan_mb(); \ - instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ - arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ -}) - - -#define cmpxchg_double_local(ptr, ...) \ -({ \ - typeof(ptr) __ai_ptr = (ptr); \ - instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ - arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ -}) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 82d1be694fab30414527d0877c29fa75ed5a0b74 +// 3611991b015450e119bcd7417a9431af7f3ba13c diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index cbbf6d189a3a..ec3573119923 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -358,33 +358,6 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { } pscr2_ret__; \ }) -/* - * Special handling for cmpxchg_double. cmpxchg_double is passed two - * percpu variables. The first has to be aligned to a double word - * boundary and the second has to follow directly thereafter. - * We enforce this on all architectures even if they don't support - * a double cmpxchg instruction, since it's a cheap requirement, and it - * avoids breaking the requirement for architectures with the instruction. - */ -#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \ -({ \ - bool pdcrb_ret__; \ - __verify_pcpu_ptr(&(pcp1)); \ - BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ - VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \ - VM_BUG_ON((unsigned long)(&(pcp2)) != \ - (unsigned long)(&(pcp1)) + sizeof(pcp1)); \ - switch(sizeof(pcp1)) { \ - case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ - case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ - case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ - case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ - default: \ - __bad_size_call_parameter(); break; \ - } \ - pdcrb_ret__; \ -}) - #define __pcpu_size_call(stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \ @@ -443,9 +416,6 @@ do { \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) #define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \ __pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval) -#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) - #define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val)) #define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1) #define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1) @@ -505,11 +475,6 @@ do { \ raw_cpu_cmpxchg(pcp, oval, nval); \ }) -#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ -({ __this_cpu_preempt_check("cmpxchg_double"); \ - raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \ -}) - #define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val)) #define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1) #define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1) @@ -532,9 +497,6 @@ do { \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) #define this_cpu_try_cmpxchg(pcp, ovalp, nval) \ __pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval) -#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) - #define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val)) #define this_cpu_inc(pcp) this_cpu_add(pcp, 1) #define this_cpu_dec(pcp) this_cpu_sub(pcp, 1) -- cgit v1.2.3 From 14d72d4b6f0e88b5f683c1a5b7a876a55055852d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:00:59 +0100 Subject: locking/atomic: remove fallback comments Currently a subset of the fallback templates have kerneldoc comments, resulting in a haphazard set of generated kerneldoc comments as only some operations have fallback templates to begin with. We'd like to generate more consistent kerneldoc comments, and to do so we'll need to restructure the way the fallback code is generated. To minimize churn and to make it easier to restructure the fallback code, this patch removes the existing kerneldoc comments from the fallback templates. We can add new kerneldoc comments in subsequent patches. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-3-mark.rutland@arm.com --- include/linux/atomic/atomic-arch-fallback.h | 166 +--------------------------- 1 file changed, 1 insertion(+), 165 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 1722ddb6f17e..3ce4cb5e790c 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -1272,15 +1272,6 @@ arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) #endif /* arch_atomic_try_cmpxchg_relaxed */ #ifndef arch_atomic_sub_and_test -/** - * arch_atomic_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v) { @@ -1290,14 +1281,6 @@ arch_atomic_sub_and_test(int i, atomic_t *v) #endif #ifndef arch_atomic_dec_and_test -/** - * arch_atomic_dec_and_test - decrement and test - * @v: pointer of type atomic_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ static __always_inline bool arch_atomic_dec_and_test(atomic_t *v) { @@ -1307,14 +1290,6 @@ arch_atomic_dec_and_test(atomic_t *v) #endif #ifndef arch_atomic_inc_and_test -/** - * arch_atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ static __always_inline bool arch_atomic_inc_and_test(atomic_t *v) { @@ -1331,14 +1306,6 @@ arch_atomic_inc_and_test(atomic_t *v) #endif /* arch_atomic_add_negative */ #ifndef arch_atomic_add_negative -/** - * arch_atomic_add_negative - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v) { @@ -1348,14 +1315,6 @@ arch_atomic_add_negative(int i, atomic_t *v) #endif #ifndef arch_atomic_add_negative_acquire -/** - * arch_atomic_add_negative_acquire - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic_add_negative_acquire(int i, atomic_t *v) { @@ -1365,14 +1324,6 @@ arch_atomic_add_negative_acquire(int i, atomic_t *v) #endif #ifndef arch_atomic_add_negative_release -/** - * arch_atomic_add_negative_release - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic_add_negative_release(int i, atomic_t *v) { @@ -1382,14 +1333,6 @@ arch_atomic_add_negative_release(int i, atomic_t *v) #endif #ifndef arch_atomic_add_negative_relaxed -/** - * arch_atomic_add_negative_relaxed - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic_add_negative_relaxed(int i, atomic_t *v) { @@ -1437,15 +1380,6 @@ arch_atomic_add_negative(int i, atomic_t *v) #endif /* arch_atomic_add_negative_relaxed */ #ifndef arch_atomic_fetch_add_unless -/** - * arch_atomic_fetch_add_unless - add unless the number is already a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as @v was not already @u. - * Returns original value of @v - */ static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { @@ -1462,15 +1396,6 @@ arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) #endif #ifndef arch_atomic_add_unless -/** - * arch_atomic_add_unless - add unless the number is already a given value - * @v: pointer of type atomic_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, if @v was not already @u. - * Returns true if the addition was done. - */ static __always_inline bool arch_atomic_add_unless(atomic_t *v, int a, int u) { @@ -1480,13 +1405,6 @@ arch_atomic_add_unless(atomic_t *v, int a, int u) #endif #ifndef arch_atomic_inc_not_zero -/** - * arch_atomic_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1, if @v is non-zero. - * Returns true if the increment was done. - */ static __always_inline bool arch_atomic_inc_not_zero(atomic_t *v) { @@ -2488,15 +2406,6 @@ arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) #endif /* arch_atomic64_try_cmpxchg_relaxed */ #ifndef arch_atomic64_sub_and_test -/** - * arch_atomic64_sub_and_test - subtract value from variable and test result - * @i: integer value to subtract - * @v: pointer of type atomic64_t - * - * Atomically subtracts @i from @v and returns - * true if the result is zero, or false for all - * other cases. - */ static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v) { @@ -2506,14 +2415,6 @@ arch_atomic64_sub_and_test(s64 i, atomic64_t *v) #endif #ifndef arch_atomic64_dec_and_test -/** - * arch_atomic64_dec_and_test - decrement and test - * @v: pointer of type atomic64_t - * - * Atomically decrements @v by 1 and - * returns true if the result is 0, or false for all other - * cases. - */ static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v) { @@ -2523,14 +2424,6 @@ arch_atomic64_dec_and_test(atomic64_t *v) #endif #ifndef arch_atomic64_inc_and_test -/** - * arch_atomic64_inc_and_test - increment and test - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v) { @@ -2547,14 +2440,6 @@ arch_atomic64_inc_and_test(atomic64_t *v) #endif /* arch_atomic64_add_negative */ #ifndef arch_atomic64_add_negative -/** - * arch_atomic64_add_negative - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic64_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v) { @@ -2564,14 +2449,6 @@ arch_atomic64_add_negative(s64 i, atomic64_t *v) #endif #ifndef arch_atomic64_add_negative_acquire -/** - * arch_atomic64_add_negative_acquire - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic64_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) { @@ -2581,14 +2458,6 @@ arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) #endif #ifndef arch_atomic64_add_negative_release -/** - * arch_atomic64_add_negative_release - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic64_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic64_add_negative_release(s64 i, atomic64_t *v) { @@ -2598,14 +2467,6 @@ arch_atomic64_add_negative_release(s64 i, atomic64_t *v) #endif #ifndef arch_atomic64_add_negative_relaxed -/** - * arch_atomic64_add_negative_relaxed - Add and test if negative - * @i: integer value to add - * @v: pointer of type atomic64_t - * - * Atomically adds @i to @v and returns true if the result is negative, - * or false when the result is greater than or equal to zero. - */ static __always_inline bool arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) { @@ -2653,15 +2514,6 @@ arch_atomic64_add_negative(s64 i, atomic64_t *v) #endif /* arch_atomic64_add_negative_relaxed */ #ifndef arch_atomic64_fetch_add_unless -/** - * arch_atomic64_fetch_add_unless - add unless the number is already a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, so long as @v was not already @u. - * Returns original value of @v - */ static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { @@ -2678,15 +2530,6 @@ arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) #endif #ifndef arch_atomic64_add_unless -/** - * arch_atomic64_add_unless - add unless the number is already a given value - * @v: pointer of type atomic64_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @v, if @v was not already @u. - * Returns true if the addition was done. - */ static __always_inline bool arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { @@ -2696,13 +2539,6 @@ arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) #endif #ifndef arch_atomic64_inc_not_zero -/** - * arch_atomic64_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic64_t - * - * Atomically increments @v by 1, if @v is non-zero. - * Returns true if the increment was done. - */ static __always_inline bool arch_atomic64_inc_not_zero(atomic64_t *v) { @@ -2761,4 +2597,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) #endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53 +// 9f0fd6ed53267c6ec64e36cd18e6fd8df57ea277 -- cgit v1.2.3 From d12157efc8e083c77d054675fcdd594f54cc7e2b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:01 +0100 Subject: locking/atomic: make atomic*_{cmp,}xchg optional Most architectures define the atomic/atomic64 xchg and cmpxchg operations in terms of arch_xchg and arch_cmpxchg respectfully. Add fallbacks for these cases and remove the trivial cases from arch code. On some architectures the existing definitions are kept as these are used to build other arch_atomic*() operations. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-5-mark.rutland@arm.com --- include/linux/atomic/atomic-arch-fallback.h | 158 +++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 3ce4cb5e790c..1a2d81dbc2e4 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -1091,9 +1091,48 @@ arch_atomic_fetch_xor(int i, atomic_t *v) #endif /* arch_atomic_fetch_xor_relaxed */ #ifndef arch_atomic_xchg_relaxed +#ifdef arch_atomic_xchg #define arch_atomic_xchg_acquire arch_atomic_xchg #define arch_atomic_xchg_release arch_atomic_xchg #define arch_atomic_xchg_relaxed arch_atomic_xchg +#endif /* arch_atomic_xchg */ + +#ifndef arch_atomic_xchg +static __always_inline int +arch_atomic_xchg(atomic_t *v, int new) +{ + return arch_xchg(&v->counter, new); +} +#define arch_atomic_xchg arch_atomic_xchg +#endif + +#ifndef arch_atomic_xchg_acquire +static __always_inline int +arch_atomic_xchg_acquire(atomic_t *v, int new) +{ + return arch_xchg_acquire(&v->counter, new); +} +#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire +#endif + +#ifndef arch_atomic_xchg_release +static __always_inline int +arch_atomic_xchg_release(atomic_t *v, int new) +{ + return arch_xchg_release(&v->counter, new); +} +#define arch_atomic_xchg_release arch_atomic_xchg_release +#endif + +#ifndef arch_atomic_xchg_relaxed +static __always_inline int +arch_atomic_xchg_relaxed(atomic_t *v, int new) +{ + return arch_xchg_relaxed(&v->counter, new); +} +#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed +#endif + #else /* arch_atomic_xchg_relaxed */ #ifndef arch_atomic_xchg_acquire @@ -1133,9 +1172,48 @@ arch_atomic_xchg(atomic_t *v, int i) #endif /* arch_atomic_xchg_relaxed */ #ifndef arch_atomic_cmpxchg_relaxed +#ifdef arch_atomic_cmpxchg #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg +#endif /* arch_atomic_cmpxchg */ + +#ifndef arch_atomic_cmpxchg +static __always_inline int +arch_atomic_cmpxchg(atomic_t *v, int old, int new) +{ + return arch_cmpxchg(&v->counter, old, new); +} +#define arch_atomic_cmpxchg arch_atomic_cmpxchg +#endif + +#ifndef arch_atomic_cmpxchg_acquire +static __always_inline int +arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +{ + return arch_cmpxchg_acquire(&v->counter, old, new); +} +#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire +#endif + +#ifndef arch_atomic_cmpxchg_release +static __always_inline int +arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) +{ + return arch_cmpxchg_release(&v->counter, old, new); +} +#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release +#endif + +#ifndef arch_atomic_cmpxchg_relaxed +static __always_inline int +arch_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) +{ + return arch_cmpxchg_relaxed(&v->counter, old, new); +} +#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed +#endif + #else /* arch_atomic_cmpxchg_relaxed */ #ifndef arch_atomic_cmpxchg_acquire @@ -2225,9 +2303,48 @@ arch_atomic64_fetch_xor(s64 i, atomic64_t *v) #endif /* arch_atomic64_fetch_xor_relaxed */ #ifndef arch_atomic64_xchg_relaxed +#ifdef arch_atomic64_xchg #define arch_atomic64_xchg_acquire arch_atomic64_xchg #define arch_atomic64_xchg_release arch_atomic64_xchg #define arch_atomic64_xchg_relaxed arch_atomic64_xchg +#endif /* arch_atomic64_xchg */ + +#ifndef arch_atomic64_xchg +static __always_inline s64 +arch_atomic64_xchg(atomic64_t *v, s64 new) +{ + return arch_xchg(&v->counter, new); +} +#define arch_atomic64_xchg arch_atomic64_xchg +#endif + +#ifndef arch_atomic64_xchg_acquire +static __always_inline s64 +arch_atomic64_xchg_acquire(atomic64_t *v, s64 new) +{ + return arch_xchg_acquire(&v->counter, new); +} +#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire +#endif + +#ifndef arch_atomic64_xchg_release +static __always_inline s64 +arch_atomic64_xchg_release(atomic64_t *v, s64 new) +{ + return arch_xchg_release(&v->counter, new); +} +#define arch_atomic64_xchg_release arch_atomic64_xchg_release +#endif + +#ifndef arch_atomic64_xchg_relaxed +static __always_inline s64 +arch_atomic64_xchg_relaxed(atomic64_t *v, s64 new) +{ + return arch_xchg_relaxed(&v->counter, new); +} +#define arch_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed +#endif + #else /* arch_atomic64_xchg_relaxed */ #ifndef arch_atomic64_xchg_acquire @@ -2267,9 +2384,48 @@ arch_atomic64_xchg(atomic64_t *v, s64 i) #endif /* arch_atomic64_xchg_relaxed */ #ifndef arch_atomic64_cmpxchg_relaxed +#ifdef arch_atomic64_cmpxchg #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg +#endif /* arch_atomic64_cmpxchg */ + +#ifndef arch_atomic64_cmpxchg +static __always_inline s64 +arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +{ + return arch_cmpxchg(&v->counter, old, new); +} +#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg +#endif + +#ifndef arch_atomic64_cmpxchg_acquire +static __always_inline s64 +arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +{ + return arch_cmpxchg_acquire(&v->counter, old, new); +} +#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire +#endif + +#ifndef arch_atomic64_cmpxchg_release +static __always_inline s64 +arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +{ + return arch_cmpxchg_release(&v->counter, old, new); +} +#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release +#endif + +#ifndef arch_atomic64_cmpxchg_relaxed +static __always_inline s64 +arch_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +{ + return arch_cmpxchg_relaxed(&v->counter, old, new); +} +#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed +#endif + #else /* arch_atomic64_cmpxchg_relaxed */ #ifndef arch_atomic64_cmpxchg_acquire @@ -2597,4 +2753,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v) #endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 9f0fd6ed53267c6ec64e36cd18e6fd8df57ea277 +// e1cee558cc61cae887890db30fcdf93baca9f498 -- cgit v1.2.3 From c9268ac615f9f6dded7801df5993374598934377 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:14 +0100 Subject: locking/atomic: scripts: add trivial raw_atomic*_() Currently a number of arch_atomic*_() functions are optional, and where an arch does not provide a given arch_atomic*_() we will define an implementation of arch_atomic*_() in atomic-arch-fallback.h. Filling in the missing ops requires special care as we want to select the optimal definition of each op (e.g. preferentially defining ops in terms of their relaxed form rather than their fully-ordered form). The ifdeffery necessary for this requires us to group ordering variants together, which can be a bit painful to read, and is painful for kerneldoc generation. It would be easier to handle this if we generated ops into a separate namespace, as this would remove the need to take special care with the ifdeffery, and allow each ordering variant to be generated separately. This patch adds a new set of raw_atomic_() definitions, which are currently trivial wrappers of their arch_atomic_() equivalent. This will allow us to move treewide users of arch_atomic_() over to raw atomic op before we rework the fallback generation to generate raw_atomic_ directly. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-18-mark.rutland@arm.com --- include/linux/atomic.h | 1 + include/linux/atomic/atomic-instrumented.h | 595 +++++----- include/linux/atomic/atomic-raw.h | 1645 ++++++++++++++++++++++++++++ 3 files changed, 1941 insertions(+), 300 deletions(-) create mode 100644 include/linux/atomic/atomic-raw.h (limited to 'include/linux') diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 8dd57c3a99e9..127f5dc63a7d 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -79,6 +79,7 @@ #include #include +#include #include #endif /* _LINUX_ATOMIC_H */ diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index a55b5b70a3e1..90ee2f55af77 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -4,15 +4,10 @@ // DO NOT MODIFY THIS FILE DIRECTLY /* - * This file provides wrappers with KASAN instrumentation for atomic operations. - * To use this functionality an arch's atomic.h file needs to define all - * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include - * this file at the end. This file provides atomic_read() that forwards to - * arch_atomic_read() for actual atomic operation. - * Note: if an arch atomic operation is implemented by means of other atomic - * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use - * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid - * double instrumentation. + * This file provoides atomic operations with explicit instrumentation (e.g. + * KASAN, KCSAN), which should be used unless it is necessary to avoid + * instrumentation. Where it is necessary to aovid instrumenation, the + * raw_atomic*() operations should be used. */ #ifndef _LINUX_ATOMIC_INSTRUMENTED_H #define _LINUX_ATOMIC_INSTRUMENTED_H @@ -25,21 +20,21 @@ static __always_inline int atomic_read(const atomic_t *v) { instrument_atomic_read(v, sizeof(*v)); - return arch_atomic_read(v); + return raw_atomic_read(v); } static __always_inline int atomic_read_acquire(const atomic_t *v) { instrument_atomic_read(v, sizeof(*v)); - return arch_atomic_read_acquire(v); + return raw_atomic_read_acquire(v); } static __always_inline void atomic_set(atomic_t *v, int i) { instrument_atomic_write(v, sizeof(*v)); - arch_atomic_set(v, i); + raw_atomic_set(v, i); } static __always_inline void @@ -47,14 +42,14 @@ atomic_set_release(atomic_t *v, int i) { kcsan_release(); instrument_atomic_write(v, sizeof(*v)); - arch_atomic_set_release(v, i); + raw_atomic_set_release(v, i); } static __always_inline void atomic_add(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_add(i, v); + raw_atomic_add(i, v); } static __always_inline int @@ -62,14 +57,14 @@ atomic_add_return(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_return(i, v); + return raw_atomic_add_return(i, v); } static __always_inline int atomic_add_return_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_return_acquire(i, v); + return raw_atomic_add_return_acquire(i, v); } static __always_inline int @@ -77,14 +72,14 @@ atomic_add_return_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_return_release(i, v); + return raw_atomic_add_return_release(i, v); } static __always_inline int atomic_add_return_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_return_relaxed(i, v); + return raw_atomic_add_return_relaxed(i, v); } static __always_inline int @@ -92,14 +87,14 @@ atomic_fetch_add(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_add(i, v); + return raw_atomic_fetch_add(i, v); } static __always_inline int atomic_fetch_add_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_add_acquire(i, v); + return raw_atomic_fetch_add_acquire(i, v); } static __always_inline int @@ -107,21 +102,21 @@ atomic_fetch_add_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_add_release(i, v); + return raw_atomic_fetch_add_release(i, v); } static __always_inline int atomic_fetch_add_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_add_relaxed(i, v); + return raw_atomic_fetch_add_relaxed(i, v); } static __always_inline void atomic_sub(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_sub(i, v); + raw_atomic_sub(i, v); } static __always_inline int @@ -129,14 +124,14 @@ atomic_sub_return(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_sub_return(i, v); + return raw_atomic_sub_return(i, v); } static __always_inline int atomic_sub_return_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_sub_return_acquire(i, v); + return raw_atomic_sub_return_acquire(i, v); } static __always_inline int @@ -144,14 +139,14 @@ atomic_sub_return_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_sub_return_release(i, v); + return raw_atomic_sub_return_release(i, v); } static __always_inline int atomic_sub_return_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_sub_return_relaxed(i, v); + return raw_atomic_sub_return_relaxed(i, v); } static __always_inline int @@ -159,14 +154,14 @@ atomic_fetch_sub(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_sub(i, v); + return raw_atomic_fetch_sub(i, v); } static __always_inline int atomic_fetch_sub_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_sub_acquire(i, v); + return raw_atomic_fetch_sub_acquire(i, v); } static __always_inline int @@ -174,21 +169,21 @@ atomic_fetch_sub_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_sub_release(i, v); + return raw_atomic_fetch_sub_release(i, v); } static __always_inline int atomic_fetch_sub_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_sub_relaxed(i, v); + return raw_atomic_fetch_sub_relaxed(i, v); } static __always_inline void atomic_inc(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_inc(v); + raw_atomic_inc(v); } static __always_inline int @@ -196,14 +191,14 @@ atomic_inc_return(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_return(v); + return raw_atomic_inc_return(v); } static __always_inline int atomic_inc_return_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_return_acquire(v); + return raw_atomic_inc_return_acquire(v); } static __always_inline int @@ -211,14 +206,14 @@ atomic_inc_return_release(atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_return_release(v); + return raw_atomic_inc_return_release(v); } static __always_inline int atomic_inc_return_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_return_relaxed(v); + return raw_atomic_inc_return_relaxed(v); } static __always_inline int @@ -226,14 +221,14 @@ atomic_fetch_inc(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_inc(v); + return raw_atomic_fetch_inc(v); } static __always_inline int atomic_fetch_inc_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_inc_acquire(v); + return raw_atomic_fetch_inc_acquire(v); } static __always_inline int @@ -241,21 +236,21 @@ atomic_fetch_inc_release(atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_inc_release(v); + return raw_atomic_fetch_inc_release(v); } static __always_inline int atomic_fetch_inc_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_inc_relaxed(v); + return raw_atomic_fetch_inc_relaxed(v); } static __always_inline void atomic_dec(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_dec(v); + raw_atomic_dec(v); } static __always_inline int @@ -263,14 +258,14 @@ atomic_dec_return(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_return(v); + return raw_atomic_dec_return(v); } static __always_inline int atomic_dec_return_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_return_acquire(v); + return raw_atomic_dec_return_acquire(v); } static __always_inline int @@ -278,14 +273,14 @@ atomic_dec_return_release(atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_return_release(v); + return raw_atomic_dec_return_release(v); } static __always_inline int atomic_dec_return_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_return_relaxed(v); + return raw_atomic_dec_return_relaxed(v); } static __always_inline int @@ -293,14 +288,14 @@ atomic_fetch_dec(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_dec(v); + return raw_atomic_fetch_dec(v); } static __always_inline int atomic_fetch_dec_acquire(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_dec_acquire(v); + return raw_atomic_fetch_dec_acquire(v); } static __always_inline int @@ -308,21 +303,21 @@ atomic_fetch_dec_release(atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_dec_release(v); + return raw_atomic_fetch_dec_release(v); } static __always_inline int atomic_fetch_dec_relaxed(atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_dec_relaxed(v); + return raw_atomic_fetch_dec_relaxed(v); } static __always_inline void atomic_and(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_and(i, v); + raw_atomic_and(i, v); } static __always_inline int @@ -330,14 +325,14 @@ atomic_fetch_and(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_and(i, v); + return raw_atomic_fetch_and(i, v); } static __always_inline int atomic_fetch_and_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_and_acquire(i, v); + return raw_atomic_fetch_and_acquire(i, v); } static __always_inline int @@ -345,21 +340,21 @@ atomic_fetch_and_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_and_release(i, v); + return raw_atomic_fetch_and_release(i, v); } static __always_inline int atomic_fetch_and_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_and_relaxed(i, v); + return raw_atomic_fetch_and_relaxed(i, v); } static __always_inline void atomic_andnot(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_andnot(i, v); + raw_atomic_andnot(i, v); } static __always_inline int @@ -367,14 +362,14 @@ atomic_fetch_andnot(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_andnot(i, v); + return raw_atomic_fetch_andnot(i, v); } static __always_inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_andnot_acquire(i, v); + return raw_atomic_fetch_andnot_acquire(i, v); } static __always_inline int @@ -382,21 +377,21 @@ atomic_fetch_andnot_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_andnot_release(i, v); + return raw_atomic_fetch_andnot_release(i, v); } static __always_inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_andnot_relaxed(i, v); + return raw_atomic_fetch_andnot_relaxed(i, v); } static __always_inline void atomic_or(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_or(i, v); + raw_atomic_or(i, v); } static __always_inline int @@ -404,14 +399,14 @@ atomic_fetch_or(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_or(i, v); + return raw_atomic_fetch_or(i, v); } static __always_inline int atomic_fetch_or_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_or_acquire(i, v); + return raw_atomic_fetch_or_acquire(i, v); } static __always_inline int @@ -419,21 +414,21 @@ atomic_fetch_or_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_or_release(i, v); + return raw_atomic_fetch_or_release(i, v); } static __always_inline int atomic_fetch_or_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_or_relaxed(i, v); + return raw_atomic_fetch_or_relaxed(i, v); } static __always_inline void atomic_xor(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_xor(i, v); + raw_atomic_xor(i, v); } static __always_inline int @@ -441,14 +436,14 @@ atomic_fetch_xor(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_xor(i, v); + return raw_atomic_fetch_xor(i, v); } static __always_inline int atomic_fetch_xor_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_xor_acquire(i, v); + return raw_atomic_fetch_xor_acquire(i, v); } static __always_inline int @@ -456,14 +451,14 @@ atomic_fetch_xor_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_xor_release(i, v); + return raw_atomic_fetch_xor_release(i, v); } static __always_inline int atomic_fetch_xor_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_xor_relaxed(i, v); + return raw_atomic_fetch_xor_relaxed(i, v); } static __always_inline int @@ -471,14 +466,14 @@ atomic_xchg(atomic_t *v, int i) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_xchg(v, i); + return raw_atomic_xchg(v, i); } static __always_inline int atomic_xchg_acquire(atomic_t *v, int i) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_xchg_acquire(v, i); + return raw_atomic_xchg_acquire(v, i); } static __always_inline int @@ -486,14 +481,14 @@ atomic_xchg_release(atomic_t *v, int i) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_xchg_release(v, i); + return raw_atomic_xchg_release(v, i); } static __always_inline int atomic_xchg_relaxed(atomic_t *v, int i) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_xchg_relaxed(v, i); + return raw_atomic_xchg_relaxed(v, i); } static __always_inline int @@ -501,14 +496,14 @@ atomic_cmpxchg(atomic_t *v, int old, int new) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_cmpxchg(v, old, new); + return raw_atomic_cmpxchg(v, old, new); } static __always_inline int atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_cmpxchg_acquire(v, old, new); + return raw_atomic_cmpxchg_acquire(v, old, new); } static __always_inline int @@ -516,14 +511,14 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_cmpxchg_release(v, old, new); + return raw_atomic_cmpxchg_release(v, old, new); } static __always_inline int atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_cmpxchg_relaxed(v, old, new); + return raw_atomic_cmpxchg_relaxed(v, old, new); } static __always_inline bool @@ -532,7 +527,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_try_cmpxchg(v, old, new); + return raw_atomic_try_cmpxchg(v, old, new); } static __always_inline bool @@ -540,7 +535,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_try_cmpxchg_acquire(v, old, new); + return raw_atomic_try_cmpxchg_acquire(v, old, new); } static __always_inline bool @@ -549,7 +544,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_try_cmpxchg_release(v, old, new); + return raw_atomic_try_cmpxchg_release(v, old, new); } static __always_inline bool @@ -557,7 +552,7 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_try_cmpxchg_relaxed(v, old, new); + return raw_atomic_try_cmpxchg_relaxed(v, old, new); } static __always_inline bool @@ -565,7 +560,7 @@ atomic_sub_and_test(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_sub_and_test(i, v); + return raw_atomic_sub_and_test(i, v); } static __always_inline bool @@ -573,7 +568,7 @@ atomic_dec_and_test(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_and_test(v); + return raw_atomic_dec_and_test(v); } static __always_inline bool @@ -581,7 +576,7 @@ atomic_inc_and_test(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_and_test(v); + return raw_atomic_inc_and_test(v); } static __always_inline bool @@ -589,14 +584,14 @@ atomic_add_negative(int i, atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_negative(i, v); + return raw_atomic_add_negative(i, v); } static __always_inline bool atomic_add_negative_acquire(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_negative_acquire(i, v); + return raw_atomic_add_negative_acquire(i, v); } static __always_inline bool @@ -604,14 +599,14 @@ atomic_add_negative_release(int i, atomic_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_negative_release(i, v); + return raw_atomic_add_negative_release(i, v); } static __always_inline bool atomic_add_negative_relaxed(int i, atomic_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_negative_relaxed(i, v); + return raw_atomic_add_negative_relaxed(i, v); } static __always_inline int @@ -619,7 +614,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_fetch_add_unless(v, a, u); + return raw_atomic_fetch_add_unless(v, a, u); } static __always_inline bool @@ -627,7 +622,7 @@ atomic_add_unless(atomic_t *v, int a, int u) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_add_unless(v, a, u); + return raw_atomic_add_unless(v, a, u); } static __always_inline bool @@ -635,7 +630,7 @@ atomic_inc_not_zero(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_not_zero(v); + return raw_atomic_inc_not_zero(v); } static __always_inline bool @@ -643,7 +638,7 @@ atomic_inc_unless_negative(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_inc_unless_negative(v); + return raw_atomic_inc_unless_negative(v); } static __always_inline bool @@ -651,7 +646,7 @@ atomic_dec_unless_positive(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_unless_positive(v); + return raw_atomic_dec_unless_positive(v); } static __always_inline int @@ -659,28 +654,28 @@ atomic_dec_if_positive(atomic_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_dec_if_positive(v); + return raw_atomic_dec_if_positive(v); } static __always_inline s64 atomic64_read(const atomic64_t *v) { instrument_atomic_read(v, sizeof(*v)); - return arch_atomic64_read(v); + return raw_atomic64_read(v); } static __always_inline s64 atomic64_read_acquire(const atomic64_t *v) { instrument_atomic_read(v, sizeof(*v)); - return arch_atomic64_read_acquire(v); + return raw_atomic64_read_acquire(v); } static __always_inline void atomic64_set(atomic64_t *v, s64 i) { instrument_atomic_write(v, sizeof(*v)); - arch_atomic64_set(v, i); + raw_atomic64_set(v, i); } static __always_inline void @@ -688,14 +683,14 @@ atomic64_set_release(atomic64_t *v, s64 i) { kcsan_release(); instrument_atomic_write(v, sizeof(*v)); - arch_atomic64_set_release(v, i); + raw_atomic64_set_release(v, i); } static __always_inline void atomic64_add(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_add(i, v); + raw_atomic64_add(i, v); } static __always_inline s64 @@ -703,14 +698,14 @@ atomic64_add_return(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_return(i, v); + return raw_atomic64_add_return(i, v); } static __always_inline s64 atomic64_add_return_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_return_acquire(i, v); + return raw_atomic64_add_return_acquire(i, v); } static __always_inline s64 @@ -718,14 +713,14 @@ atomic64_add_return_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_return_release(i, v); + return raw_atomic64_add_return_release(i, v); } static __always_inline s64 atomic64_add_return_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_return_relaxed(i, v); + return raw_atomic64_add_return_relaxed(i, v); } static __always_inline s64 @@ -733,14 +728,14 @@ atomic64_fetch_add(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_add(i, v); + return raw_atomic64_fetch_add(i, v); } static __always_inline s64 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_add_acquire(i, v); + return raw_atomic64_fetch_add_acquire(i, v); } static __always_inline s64 @@ -748,21 +743,21 @@ atomic64_fetch_add_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_add_release(i, v); + return raw_atomic64_fetch_add_release(i, v); } static __always_inline s64 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_add_relaxed(i, v); + return raw_atomic64_fetch_add_relaxed(i, v); } static __always_inline void atomic64_sub(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_sub(i, v); + raw_atomic64_sub(i, v); } static __always_inline s64 @@ -770,14 +765,14 @@ atomic64_sub_return(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_sub_return(i, v); + return raw_atomic64_sub_return(i, v); } static __always_inline s64 atomic64_sub_return_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_sub_return_acquire(i, v); + return raw_atomic64_sub_return_acquire(i, v); } static __always_inline s64 @@ -785,14 +780,14 @@ atomic64_sub_return_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_sub_return_release(i, v); + return raw_atomic64_sub_return_release(i, v); } static __always_inline s64 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_sub_return_relaxed(i, v); + return raw_atomic64_sub_return_relaxed(i, v); } static __always_inline s64 @@ -800,14 +795,14 @@ atomic64_fetch_sub(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_sub(i, v); + return raw_atomic64_fetch_sub(i, v); } static __always_inline s64 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_sub_acquire(i, v); + return raw_atomic64_fetch_sub_acquire(i, v); } static __always_inline s64 @@ -815,21 +810,21 @@ atomic64_fetch_sub_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_sub_release(i, v); + return raw_atomic64_fetch_sub_release(i, v); } static __always_inline s64 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_sub_relaxed(i, v); + return raw_atomic64_fetch_sub_relaxed(i, v); } static __always_inline void atomic64_inc(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_inc(v); + raw_atomic64_inc(v); } static __always_inline s64 @@ -837,14 +832,14 @@ atomic64_inc_return(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_return(v); + return raw_atomic64_inc_return(v); } static __always_inline s64 atomic64_inc_return_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_return_acquire(v); + return raw_atomic64_inc_return_acquire(v); } static __always_inline s64 @@ -852,14 +847,14 @@ atomic64_inc_return_release(atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_return_release(v); + return raw_atomic64_inc_return_release(v); } static __always_inline s64 atomic64_inc_return_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_return_relaxed(v); + return raw_atomic64_inc_return_relaxed(v); } static __always_inline s64 @@ -867,14 +862,14 @@ atomic64_fetch_inc(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_inc(v); + return raw_atomic64_fetch_inc(v); } static __always_inline s64 atomic64_fetch_inc_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_inc_acquire(v); + return raw_atomic64_fetch_inc_acquire(v); } static __always_inline s64 @@ -882,21 +877,21 @@ atomic64_fetch_inc_release(atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_inc_release(v); + return raw_atomic64_fetch_inc_release(v); } static __always_inline s64 atomic64_fetch_inc_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_inc_relaxed(v); + return raw_atomic64_fetch_inc_relaxed(v); } static __always_inline void atomic64_dec(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_dec(v); + raw_atomic64_dec(v); } static __always_inline s64 @@ -904,14 +899,14 @@ atomic64_dec_return(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_return(v); + return raw_atomic64_dec_return(v); } static __always_inline s64 atomic64_dec_return_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_return_acquire(v); + return raw_atomic64_dec_return_acquire(v); } static __always_inline s64 @@ -919,14 +914,14 @@ atomic64_dec_return_release(atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_return_release(v); + return raw_atomic64_dec_return_release(v); } static __always_inline s64 atomic64_dec_return_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_return_relaxed(v); + return raw_atomic64_dec_return_relaxed(v); } static __always_inline s64 @@ -934,14 +929,14 @@ atomic64_fetch_dec(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_dec(v); + return raw_atomic64_fetch_dec(v); } static __always_inline s64 atomic64_fetch_dec_acquire(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_dec_acquire(v); + return raw_atomic64_fetch_dec_acquire(v); } static __always_inline s64 @@ -949,21 +944,21 @@ atomic64_fetch_dec_release(atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_dec_release(v); + return raw_atomic64_fetch_dec_release(v); } static __always_inline s64 atomic64_fetch_dec_relaxed(atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_dec_relaxed(v); + return raw_atomic64_fetch_dec_relaxed(v); } static __always_inline void atomic64_and(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_and(i, v); + raw_atomic64_and(i, v); } static __always_inline s64 @@ -971,14 +966,14 @@ atomic64_fetch_and(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_and(i, v); + return raw_atomic64_fetch_and(i, v); } static __always_inline s64 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_and_acquire(i, v); + return raw_atomic64_fetch_and_acquire(i, v); } static __always_inline s64 @@ -986,21 +981,21 @@ atomic64_fetch_and_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_and_release(i, v); + return raw_atomic64_fetch_and_release(i, v); } static __always_inline s64 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_and_relaxed(i, v); + return raw_atomic64_fetch_and_relaxed(i, v); } static __always_inline void atomic64_andnot(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_andnot(i, v); + raw_atomic64_andnot(i, v); } static __always_inline s64 @@ -1008,14 +1003,14 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_andnot(i, v); + return raw_atomic64_fetch_andnot(i, v); } static __always_inline s64 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_andnot_acquire(i, v); + return raw_atomic64_fetch_andnot_acquire(i, v); } static __always_inline s64 @@ -1023,21 +1018,21 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_andnot_release(i, v); + return raw_atomic64_fetch_andnot_release(i, v); } static __always_inline s64 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_andnot_relaxed(i, v); + return raw_atomic64_fetch_andnot_relaxed(i, v); } static __always_inline void atomic64_or(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_or(i, v); + raw_atomic64_or(i, v); } static __always_inline s64 @@ -1045,14 +1040,14 @@ atomic64_fetch_or(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_or(i, v); + return raw_atomic64_fetch_or(i, v); } static __always_inline s64 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_or_acquire(i, v); + return raw_atomic64_fetch_or_acquire(i, v); } static __always_inline s64 @@ -1060,21 +1055,21 @@ atomic64_fetch_or_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_or_release(i, v); + return raw_atomic64_fetch_or_release(i, v); } static __always_inline s64 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_or_relaxed(i, v); + return raw_atomic64_fetch_or_relaxed(i, v); } static __always_inline void atomic64_xor(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic64_xor(i, v); + raw_atomic64_xor(i, v); } static __always_inline s64 @@ -1082,14 +1077,14 @@ atomic64_fetch_xor(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_xor(i, v); + return raw_atomic64_fetch_xor(i, v); } static __always_inline s64 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_xor_acquire(i, v); + return raw_atomic64_fetch_xor_acquire(i, v); } static __always_inline s64 @@ -1097,14 +1092,14 @@ atomic64_fetch_xor_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_xor_release(i, v); + return raw_atomic64_fetch_xor_release(i, v); } static __always_inline s64 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_xor_relaxed(i, v); + return raw_atomic64_fetch_xor_relaxed(i, v); } static __always_inline s64 @@ -1112,14 +1107,14 @@ atomic64_xchg(atomic64_t *v, s64 i) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_xchg(v, i); + return raw_atomic64_xchg(v, i); } static __always_inline s64 atomic64_xchg_acquire(atomic64_t *v, s64 i) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_xchg_acquire(v, i); + return raw_atomic64_xchg_acquire(v, i); } static __always_inline s64 @@ -1127,14 +1122,14 @@ atomic64_xchg_release(atomic64_t *v, s64 i) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_xchg_release(v, i); + return raw_atomic64_xchg_release(v, i); } static __always_inline s64 atomic64_xchg_relaxed(atomic64_t *v, s64 i) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_xchg_relaxed(v, i); + return raw_atomic64_xchg_relaxed(v, i); } static __always_inline s64 @@ -1142,14 +1137,14 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_cmpxchg(v, old, new); + return raw_atomic64_cmpxchg(v, old, new); } static __always_inline s64 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_cmpxchg_acquire(v, old, new); + return raw_atomic64_cmpxchg_acquire(v, old, new); } static __always_inline s64 @@ -1157,14 +1152,14 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_cmpxchg_release(v, old, new); + return raw_atomic64_cmpxchg_release(v, old, new); } static __always_inline s64 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_cmpxchg_relaxed(v, old, new); + return raw_atomic64_cmpxchg_relaxed(v, old, new); } static __always_inline bool @@ -1173,7 +1168,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic64_try_cmpxchg(v, old, new); + return raw_atomic64_try_cmpxchg(v, old, new); } static __always_inline bool @@ -1181,7 +1176,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic64_try_cmpxchg_acquire(v, old, new); + return raw_atomic64_try_cmpxchg_acquire(v, old, new); } static __always_inline bool @@ -1190,7 +1185,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic64_try_cmpxchg_release(v, old, new); + return raw_atomic64_try_cmpxchg_release(v, old, new); } static __always_inline bool @@ -1198,7 +1193,7 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic64_try_cmpxchg_relaxed(v, old, new); + return raw_atomic64_try_cmpxchg_relaxed(v, old, new); } static __always_inline bool @@ -1206,7 +1201,7 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_sub_and_test(i, v); + return raw_atomic64_sub_and_test(i, v); } static __always_inline bool @@ -1214,7 +1209,7 @@ atomic64_dec_and_test(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_and_test(v); + return raw_atomic64_dec_and_test(v); } static __always_inline bool @@ -1222,7 +1217,7 @@ atomic64_inc_and_test(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_and_test(v); + return raw_atomic64_inc_and_test(v); } static __always_inline bool @@ -1230,14 +1225,14 @@ atomic64_add_negative(s64 i, atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_negative(i, v); + return raw_atomic64_add_negative(i, v); } static __always_inline bool atomic64_add_negative_acquire(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_negative_acquire(i, v); + return raw_atomic64_add_negative_acquire(i, v); } static __always_inline bool @@ -1245,14 +1240,14 @@ atomic64_add_negative_release(s64 i, atomic64_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_negative_release(i, v); + return raw_atomic64_add_negative_release(i, v); } static __always_inline bool atomic64_add_negative_relaxed(s64 i, atomic64_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_negative_relaxed(i, v); + return raw_atomic64_add_negative_relaxed(i, v); } static __always_inline s64 @@ -1260,7 +1255,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_fetch_add_unless(v, a, u); + return raw_atomic64_fetch_add_unless(v, a, u); } static __always_inline bool @@ -1268,7 +1263,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_add_unless(v, a, u); + return raw_atomic64_add_unless(v, a, u); } static __always_inline bool @@ -1276,7 +1271,7 @@ atomic64_inc_not_zero(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_not_zero(v); + return raw_atomic64_inc_not_zero(v); } static __always_inline bool @@ -1284,7 +1279,7 @@ atomic64_inc_unless_negative(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_inc_unless_negative(v); + return raw_atomic64_inc_unless_negative(v); } static __always_inline bool @@ -1292,7 +1287,7 @@ atomic64_dec_unless_positive(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_unless_positive(v); + return raw_atomic64_dec_unless_positive(v); } static __always_inline s64 @@ -1300,28 +1295,28 @@ atomic64_dec_if_positive(atomic64_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic64_dec_if_positive(v); + return raw_atomic64_dec_if_positive(v); } static __always_inline long atomic_long_read(const atomic_long_t *v) { instrument_atomic_read(v, sizeof(*v)); - return arch_atomic_long_read(v); + return raw_atomic_long_read(v); } static __always_inline long atomic_long_read_acquire(const atomic_long_t *v) { instrument_atomic_read(v, sizeof(*v)); - return arch_atomic_long_read_acquire(v); + return raw_atomic_long_read_acquire(v); } static __always_inline void atomic_long_set(atomic_long_t *v, long i) { instrument_atomic_write(v, sizeof(*v)); - arch_atomic_long_set(v, i); + raw_atomic_long_set(v, i); } static __always_inline void @@ -1329,14 +1324,14 @@ atomic_long_set_release(atomic_long_t *v, long i) { kcsan_release(); instrument_atomic_write(v, sizeof(*v)); - arch_atomic_long_set_release(v, i); + raw_atomic_long_set_release(v, i); } static __always_inline void atomic_long_add(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_add(i, v); + raw_atomic_long_add(i, v); } static __always_inline long @@ -1344,14 +1339,14 @@ atomic_long_add_return(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_return(i, v); + return raw_atomic_long_add_return(i, v); } static __always_inline long atomic_long_add_return_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_return_acquire(i, v); + return raw_atomic_long_add_return_acquire(i, v); } static __always_inline long @@ -1359,14 +1354,14 @@ atomic_long_add_return_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_return_release(i, v); + return raw_atomic_long_add_return_release(i, v); } static __always_inline long atomic_long_add_return_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_return_relaxed(i, v); + return raw_atomic_long_add_return_relaxed(i, v); } static __always_inline long @@ -1374,14 +1369,14 @@ atomic_long_fetch_add(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_add(i, v); + return raw_atomic_long_fetch_add(i, v); } static __always_inline long atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_add_acquire(i, v); + return raw_atomic_long_fetch_add_acquire(i, v); } static __always_inline long @@ -1389,21 +1384,21 @@ atomic_long_fetch_add_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_add_release(i, v); + return raw_atomic_long_fetch_add_release(i, v); } static __always_inline long atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_add_relaxed(i, v); + return raw_atomic_long_fetch_add_relaxed(i, v); } static __always_inline void atomic_long_sub(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_sub(i, v); + raw_atomic_long_sub(i, v); } static __always_inline long @@ -1411,14 +1406,14 @@ atomic_long_sub_return(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_sub_return(i, v); + return raw_atomic_long_sub_return(i, v); } static __always_inline long atomic_long_sub_return_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_sub_return_acquire(i, v); + return raw_atomic_long_sub_return_acquire(i, v); } static __always_inline long @@ -1426,14 +1421,14 @@ atomic_long_sub_return_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_sub_return_release(i, v); + return raw_atomic_long_sub_return_release(i, v); } static __always_inline long atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_sub_return_relaxed(i, v); + return raw_atomic_long_sub_return_relaxed(i, v); } static __always_inline long @@ -1441,14 +1436,14 @@ atomic_long_fetch_sub(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_sub(i, v); + return raw_atomic_long_fetch_sub(i, v); } static __always_inline long atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_sub_acquire(i, v); + return raw_atomic_long_fetch_sub_acquire(i, v); } static __always_inline long @@ -1456,21 +1451,21 @@ atomic_long_fetch_sub_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_sub_release(i, v); + return raw_atomic_long_fetch_sub_release(i, v); } static __always_inline long atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_sub_relaxed(i, v); + return raw_atomic_long_fetch_sub_relaxed(i, v); } static __always_inline void atomic_long_inc(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_inc(v); + raw_atomic_long_inc(v); } static __always_inline long @@ -1478,14 +1473,14 @@ atomic_long_inc_return(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_return(v); + return raw_atomic_long_inc_return(v); } static __always_inline long atomic_long_inc_return_acquire(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_return_acquire(v); + return raw_atomic_long_inc_return_acquire(v); } static __always_inline long @@ -1493,14 +1488,14 @@ atomic_long_inc_return_release(atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_return_release(v); + return raw_atomic_long_inc_return_release(v); } static __always_inline long atomic_long_inc_return_relaxed(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_return_relaxed(v); + return raw_atomic_long_inc_return_relaxed(v); } static __always_inline long @@ -1508,14 +1503,14 @@ atomic_long_fetch_inc(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_inc(v); + return raw_atomic_long_fetch_inc(v); } static __always_inline long atomic_long_fetch_inc_acquire(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_inc_acquire(v); + return raw_atomic_long_fetch_inc_acquire(v); } static __always_inline long @@ -1523,21 +1518,21 @@ atomic_long_fetch_inc_release(atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_inc_release(v); + return raw_atomic_long_fetch_inc_release(v); } static __always_inline long atomic_long_fetch_inc_relaxed(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_inc_relaxed(v); + return raw_atomic_long_fetch_inc_relaxed(v); } static __always_inline void atomic_long_dec(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_dec(v); + raw_atomic_long_dec(v); } static __always_inline long @@ -1545,14 +1540,14 @@ atomic_long_dec_return(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_return(v); + return raw_atomic_long_dec_return(v); } static __always_inline long atomic_long_dec_return_acquire(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_return_acquire(v); + return raw_atomic_long_dec_return_acquire(v); } static __always_inline long @@ -1560,14 +1555,14 @@ atomic_long_dec_return_release(atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_return_release(v); + return raw_atomic_long_dec_return_release(v); } static __always_inline long atomic_long_dec_return_relaxed(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_return_relaxed(v); + return raw_atomic_long_dec_return_relaxed(v); } static __always_inline long @@ -1575,14 +1570,14 @@ atomic_long_fetch_dec(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_dec(v); + return raw_atomic_long_fetch_dec(v); } static __always_inline long atomic_long_fetch_dec_acquire(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_dec_acquire(v); + return raw_atomic_long_fetch_dec_acquire(v); } static __always_inline long @@ -1590,21 +1585,21 @@ atomic_long_fetch_dec_release(atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_dec_release(v); + return raw_atomic_long_fetch_dec_release(v); } static __always_inline long atomic_long_fetch_dec_relaxed(atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_dec_relaxed(v); + return raw_atomic_long_fetch_dec_relaxed(v); } static __always_inline void atomic_long_and(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_and(i, v); + raw_atomic_long_and(i, v); } static __always_inline long @@ -1612,14 +1607,14 @@ atomic_long_fetch_and(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_and(i, v); + return raw_atomic_long_fetch_and(i, v); } static __always_inline long atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_and_acquire(i, v); + return raw_atomic_long_fetch_and_acquire(i, v); } static __always_inline long @@ -1627,21 +1622,21 @@ atomic_long_fetch_and_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_and_release(i, v); + return raw_atomic_long_fetch_and_release(i, v); } static __always_inline long atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_and_relaxed(i, v); + return raw_atomic_long_fetch_and_relaxed(i, v); } static __always_inline void atomic_long_andnot(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_andnot(i, v); + raw_atomic_long_andnot(i, v); } static __always_inline long @@ -1649,14 +1644,14 @@ atomic_long_fetch_andnot(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_andnot(i, v); + return raw_atomic_long_fetch_andnot(i, v); } static __always_inline long atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_andnot_acquire(i, v); + return raw_atomic_long_fetch_andnot_acquire(i, v); } static __always_inline long @@ -1664,21 +1659,21 @@ atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_andnot_release(i, v); + return raw_atomic_long_fetch_andnot_release(i, v); } static __always_inline long atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_andnot_relaxed(i, v); + return raw_atomic_long_fetch_andnot_relaxed(i, v); } static __always_inline void atomic_long_or(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_or(i, v); + raw_atomic_long_or(i, v); } static __always_inline long @@ -1686,14 +1681,14 @@ atomic_long_fetch_or(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_or(i, v); + return raw_atomic_long_fetch_or(i, v); } static __always_inline long atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_or_acquire(i, v); + return raw_atomic_long_fetch_or_acquire(i, v); } static __always_inline long @@ -1701,21 +1696,21 @@ atomic_long_fetch_or_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_or_release(i, v); + return raw_atomic_long_fetch_or_release(i, v); } static __always_inline long atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_or_relaxed(i, v); + return raw_atomic_long_fetch_or_relaxed(i, v); } static __always_inline void atomic_long_xor(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - arch_atomic_long_xor(i, v); + raw_atomic_long_xor(i, v); } static __always_inline long @@ -1723,14 +1718,14 @@ atomic_long_fetch_xor(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_xor(i, v); + return raw_atomic_long_fetch_xor(i, v); } static __always_inline long atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_xor_acquire(i, v); + return raw_atomic_long_fetch_xor_acquire(i, v); } static __always_inline long @@ -1738,14 +1733,14 @@ atomic_long_fetch_xor_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_xor_release(i, v); + return raw_atomic_long_fetch_xor_release(i, v); } static __always_inline long atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_xor_relaxed(i, v); + return raw_atomic_long_fetch_xor_relaxed(i, v); } static __always_inline long @@ -1753,14 +1748,14 @@ atomic_long_xchg(atomic_long_t *v, long i) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_xchg(v, i); + return raw_atomic_long_xchg(v, i); } static __always_inline long atomic_long_xchg_acquire(atomic_long_t *v, long i) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_xchg_acquire(v, i); + return raw_atomic_long_xchg_acquire(v, i); } static __always_inline long @@ -1768,14 +1763,14 @@ atomic_long_xchg_release(atomic_long_t *v, long i) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_xchg_release(v, i); + return raw_atomic_long_xchg_release(v, i); } static __always_inline long atomic_long_xchg_relaxed(atomic_long_t *v, long i) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_xchg_relaxed(v, i); + return raw_atomic_long_xchg_relaxed(v, i); } static __always_inline long @@ -1783,14 +1778,14 @@ atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_cmpxchg(v, old, new); + return raw_atomic_long_cmpxchg(v, old, new); } static __always_inline long atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_cmpxchg_acquire(v, old, new); + return raw_atomic_long_cmpxchg_acquire(v, old, new); } static __always_inline long @@ -1798,14 +1793,14 @@ atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_cmpxchg_release(v, old, new); + return raw_atomic_long_cmpxchg_release(v, old, new); } static __always_inline long atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_cmpxchg_relaxed(v, old, new); + return raw_atomic_long_cmpxchg_relaxed(v, old, new); } static __always_inline bool @@ -1814,7 +1809,7 @@ atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_long_try_cmpxchg(v, old, new); + return raw_atomic_long_try_cmpxchg(v, old, new); } static __always_inline bool @@ -1822,7 +1817,7 @@ atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_long_try_cmpxchg_acquire(v, old, new); + return raw_atomic_long_try_cmpxchg_acquire(v, old, new); } static __always_inline bool @@ -1831,7 +1826,7 @@ atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_long_try_cmpxchg_release(v, old, new); + return raw_atomic_long_try_cmpxchg_release(v, old, new); } static __always_inline bool @@ -1839,7 +1834,7 @@ atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { instrument_atomic_read_write(v, sizeof(*v)); instrument_atomic_read_write(old, sizeof(*old)); - return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); + return raw_atomic_long_try_cmpxchg_relaxed(v, old, new); } static __always_inline bool @@ -1847,7 +1842,7 @@ atomic_long_sub_and_test(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_sub_and_test(i, v); + return raw_atomic_long_sub_and_test(i, v); } static __always_inline bool @@ -1855,7 +1850,7 @@ atomic_long_dec_and_test(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_and_test(v); + return raw_atomic_long_dec_and_test(v); } static __always_inline bool @@ -1863,7 +1858,7 @@ atomic_long_inc_and_test(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_and_test(v); + return raw_atomic_long_inc_and_test(v); } static __always_inline bool @@ -1871,14 +1866,14 @@ atomic_long_add_negative(long i, atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_negative(i, v); + return raw_atomic_long_add_negative(i, v); } static __always_inline bool atomic_long_add_negative_acquire(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_negative_acquire(i, v); + return raw_atomic_long_add_negative_acquire(i, v); } static __always_inline bool @@ -1886,14 +1881,14 @@ atomic_long_add_negative_release(long i, atomic_long_t *v) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_negative_release(i, v); + return raw_atomic_long_add_negative_release(i, v); } static __always_inline bool atomic_long_add_negative_relaxed(long i, atomic_long_t *v) { instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_negative_relaxed(i, v); + return raw_atomic_long_add_negative_relaxed(i, v); } static __always_inline long @@ -1901,7 +1896,7 @@ atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_fetch_add_unless(v, a, u); + return raw_atomic_long_fetch_add_unless(v, a, u); } static __always_inline bool @@ -1909,7 +1904,7 @@ atomic_long_add_unless(atomic_long_t *v, long a, long u) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_add_unless(v, a, u); + return raw_atomic_long_add_unless(v, a, u); } static __always_inline bool @@ -1917,7 +1912,7 @@ atomic_long_inc_not_zero(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_not_zero(v); + return raw_atomic_long_inc_not_zero(v); } static __always_inline bool @@ -1925,7 +1920,7 @@ atomic_long_inc_unless_negative(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_inc_unless_negative(v); + return raw_atomic_long_inc_unless_negative(v); } static __always_inline bool @@ -1933,7 +1928,7 @@ atomic_long_dec_unless_positive(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_unless_positive(v); + return raw_atomic_long_dec_unless_positive(v); } static __always_inline long @@ -1941,7 +1936,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return arch_atomic_long_dec_if_positive(v); + return raw_atomic_long_dec_if_positive(v); } #define xchg(ptr, ...) \ @@ -1949,14 +1944,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg(__ai_ptr, __VA_ARGS__); \ + raw_xchg(__ai_ptr, __VA_ARGS__); \ }) #define xchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ + raw_xchg_acquire(__ai_ptr, __VA_ARGS__); \ }) #define xchg_release(ptr, ...) \ @@ -1964,14 +1959,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_release(__ai_ptr, __VA_ARGS__); \ + raw_xchg_release(__ai_ptr, __VA_ARGS__); \ }) #define xchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ + raw_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg(ptr, ...) \ @@ -1979,14 +1974,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg_release(ptr, ...) \ @@ -1994,14 +1989,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64(ptr, ...) \ @@ -2009,14 +2004,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg64(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_release(ptr, ...) \ @@ -2024,14 +2019,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg128(ptr, ...) \ @@ -2039,14 +2034,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg128(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg128(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg128_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg128_release(ptr, ...) \ @@ -2054,14 +2049,14 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg128_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \ }) #define try_cmpxchg(ptr, oldp, ...) \ @@ -2071,7 +2066,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg_acquire(ptr, oldp, ...) \ @@ -2080,7 +2075,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg_release(ptr, oldp, ...) \ @@ -2090,7 +2085,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg_relaxed(ptr, oldp, ...) \ @@ -2099,7 +2094,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg64(ptr, oldp, ...) \ @@ -2109,7 +2104,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg64_acquire(ptr, oldp, ...) \ @@ -2118,7 +2113,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg64_release(ptr, oldp, ...) \ @@ -2128,7 +2123,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg64_relaxed(ptr, oldp, ...) \ @@ -2137,7 +2132,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg128(ptr, oldp, ...) \ @@ -2147,7 +2142,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg128_acquire(ptr, oldp, ...) \ @@ -2156,7 +2151,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg128_release(ptr, oldp, ...) \ @@ -2166,7 +2161,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) kcsan_release(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg128_relaxed(ptr, oldp, ...) \ @@ -2175,28 +2170,28 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define cmpxchg_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg128_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \ + raw_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \ }) #define sync_cmpxchg(ptr, ...) \ @@ -2204,7 +2199,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(ptr) __ai_ptr = (ptr); \ kcsan_mb(); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ - arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ + raw_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) #define try_cmpxchg_local(ptr, oldp, ...) \ @@ -2213,7 +2208,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg64_local(ptr, oldp, ...) \ @@ -2222,7 +2217,7 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #define try_cmpxchg128_local(ptr, oldp, ...) \ @@ -2231,9 +2226,9 @@ atomic_long_dec_if_positive(atomic_long_t *v) typeof(oldp) __ai_oldp = (oldp); \ instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \ instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \ - arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ + raw_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \ }) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 3611991b015450e119bcd7417a9431af7f3ba13c +// f6502977180430e61c1a7c4e5e665f04f501fb8d diff --git a/include/linux/atomic/atomic-raw.h b/include/linux/atomic/atomic-raw.h new file mode 100644 index 000000000000..83ff0269657e --- /dev/null +++ b/include/linux/atomic/atomic-raw.h @@ -0,0 +1,1645 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Generated by scripts/atomic/gen-atomic-raw.sh +// DO NOT MODIFY THIS FILE DIRECTLY + +#ifndef _LINUX_ATOMIC_RAW_H +#define _LINUX_ATOMIC_RAW_H + +static __always_inline int +raw_atomic_read(const atomic_t *v) +{ + return arch_atomic_read(v); +} + +static __always_inline int +raw_atomic_read_acquire(const atomic_t *v) +{ + return arch_atomic_read_acquire(v); +} + +static __always_inline void +raw_atomic_set(atomic_t *v, int i) +{ + arch_atomic_set(v, i); +} + +static __always_inline void +raw_atomic_set_release(atomic_t *v, int i) +{ + arch_atomic_set_release(v, i); +} + +static __always_inline void +raw_atomic_add(int i, atomic_t *v) +{ + arch_atomic_add(i, v); +} + +static __always_inline int +raw_atomic_add_return(int i, atomic_t *v) +{ + return arch_atomic_add_return(i, v); +} + +static __always_inline int +raw_atomic_add_return_acquire(int i, atomic_t *v) +{ + return arch_atomic_add_return_acquire(i, v); +} + +static __always_inline int +raw_atomic_add_return_release(int i, atomic_t *v) +{ + return arch_atomic_add_return_release(i, v); +} + +static __always_inline int +raw_atomic_add_return_relaxed(int i, atomic_t *v) +{ + return arch_atomic_add_return_relaxed(i, v); +} + +static __always_inline int +raw_atomic_fetch_add(int i, atomic_t *v) +{ + return arch_atomic_fetch_add(i, v); +} + +static __always_inline int +raw_atomic_fetch_add_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_add_acquire(i, v); +} + +static __always_inline int +raw_atomic_fetch_add_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_add_release(i, v); +} + +static __always_inline int +raw_atomic_fetch_add_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_add_relaxed(i, v); +} + +static __always_inline void +raw_atomic_sub(int i, atomic_t *v) +{ + arch_atomic_sub(i, v); +} + +static __always_inline int +raw_atomic_sub_return(int i, atomic_t *v) +{ + return arch_atomic_sub_return(i, v); +} + +static __always_inline int +raw_atomic_sub_return_acquire(int i, atomic_t *v) +{ + return arch_atomic_sub_return_acquire(i, v); +} + +static __always_inline int +raw_atomic_sub_return_release(int i, atomic_t *v) +{ + return arch_atomic_sub_return_release(i, v); +} + +static __always_inline int +raw_atomic_sub_return_relaxed(int i, atomic_t *v) +{ + return arch_atomic_sub_return_relaxed(i, v); +} + +static __always_inline int +raw_atomic_fetch_sub(int i, atomic_t *v) +{ + return arch_atomic_fetch_sub(i, v); +} + +static __always_inline int +raw_atomic_fetch_sub_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_sub_acquire(i, v); +} + +static __always_inline int +raw_atomic_fetch_sub_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_sub_release(i, v); +} + +static __always_inline int +raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_sub_relaxed(i, v); +} + +static __always_inline void +raw_atomic_inc(atomic_t *v) +{ + arch_atomic_inc(v); +} + +static __always_inline int +raw_atomic_inc_return(atomic_t *v) +{ + return arch_atomic_inc_return(v); +} + +static __always_inline int +raw_atomic_inc_return_acquire(atomic_t *v) +{ + return arch_atomic_inc_return_acquire(v); +} + +static __always_inline int +raw_atomic_inc_return_release(atomic_t *v) +{ + return arch_atomic_inc_return_release(v); +} + +static __always_inline int +raw_atomic_inc_return_relaxed(atomic_t *v) +{ + return arch_atomic_inc_return_relaxed(v); +} + +static __always_inline int +raw_atomic_fetch_inc(atomic_t *v) +{ + return arch_atomic_fetch_inc(v); +} + +static __always_inline int +raw_atomic_fetch_inc_acquire(atomic_t *v) +{ + return arch_atomic_fetch_inc_acquire(v); +} + +static __always_inline int +raw_atomic_fetch_inc_release(atomic_t *v) +{ + return arch_atomic_fetch_inc_release(v); +} + +static __always_inline int +raw_atomic_fetch_inc_relaxed(atomic_t *v) +{ + return arch_atomic_fetch_inc_relaxed(v); +} + +static __always_inline void +raw_atomic_dec(atomic_t *v) +{ + arch_atomic_dec(v); +} + +static __always_inline int +raw_atomic_dec_return(atomic_t *v) +{ + return arch_atomic_dec_return(v); +} + +static __always_inline int +raw_atomic_dec_return_acquire(atomic_t *v) +{ + return arch_atomic_dec_return_acquire(v); +} + +static __always_inline int +raw_atomic_dec_return_release(atomic_t *v) +{ + return arch_atomic_dec_return_release(v); +} + +static __always_inline int +raw_atomic_dec_return_relaxed(atomic_t *v) +{ + return arch_atomic_dec_return_relaxed(v); +} + +static __always_inline int +raw_atomic_fetch_dec(atomic_t *v) +{ + return arch_atomic_fetch_dec(v); +} + +static __always_inline int +raw_atomic_fetch_dec_acquire(atomic_t *v) +{ + return arch_atomic_fetch_dec_acquire(v); +} + +static __always_inline int +raw_atomic_fetch_dec_release(atomic_t *v) +{ + return arch_atomic_fetch_dec_release(v); +} + +static __always_inline int +raw_atomic_fetch_dec_relaxed(atomic_t *v) +{ + return arch_atomic_fetch_dec_relaxed(v); +} + +static __always_inline void +raw_atomic_and(int i, atomic_t *v) +{ + arch_atomic_and(i, v); +} + +static __always_inline int +raw_atomic_fetch_and(int i, atomic_t *v) +{ + return arch_atomic_fetch_and(i, v); +} + +static __always_inline int +raw_atomic_fetch_and_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_and_acquire(i, v); +} + +static __always_inline int +raw_atomic_fetch_and_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_and_release(i, v); +} + +static __always_inline int +raw_atomic_fetch_and_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_and_relaxed(i, v); +} + +static __always_inline void +raw_atomic_andnot(int i, atomic_t *v) +{ + arch_atomic_andnot(i, v); +} + +static __always_inline int +raw_atomic_fetch_andnot(int i, atomic_t *v) +{ + return arch_atomic_fetch_andnot(i, v); +} + +static __always_inline int +raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_andnot_acquire(i, v); +} + +static __always_inline int +raw_atomic_fetch_andnot_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_andnot_release(i, v); +} + +static __always_inline int +raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_andnot_relaxed(i, v); +} + +static __always_inline void +raw_atomic_or(int i, atomic_t *v) +{ + arch_atomic_or(i, v); +} + +static __always_inline int +raw_atomic_fetch_or(int i, atomic_t *v) +{ + return arch_atomic_fetch_or(i, v); +} + +static __always_inline int +raw_atomic_fetch_or_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_or_acquire(i, v); +} + +static __always_inline int +raw_atomic_fetch_or_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_or_release(i, v); +} + +static __always_inline int +raw_atomic_fetch_or_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_or_relaxed(i, v); +} + +static __always_inline void +raw_atomic_xor(int i, atomic_t *v) +{ + arch_atomic_xor(i, v); +} + +static __always_inline int +raw_atomic_fetch_xor(int i, atomic_t *v) +{ + return arch_atomic_fetch_xor(i, v); +} + +static __always_inline int +raw_atomic_fetch_xor_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_xor_acquire(i, v); +} + +static __always_inline int +raw_atomic_fetch_xor_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_xor_release(i, v); +} + +static __always_inline int +raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_xor_relaxed(i, v); +} + +static __always_inline int +raw_atomic_xchg(atomic_t *v, int i) +{ + return arch_atomic_xchg(v, i); +} + +static __always_inline int +raw_atomic_xchg_acquire(atomic_t *v, int i) +{ + return arch_atomic_xchg_acquire(v, i); +} + +static __always_inline int +raw_atomic_xchg_release(atomic_t *v, int i) +{ + return arch_atomic_xchg_release(v, i); +} + +static __always_inline int +raw_atomic_xchg_relaxed(atomic_t *v, int i) +{ + return arch_atomic_xchg_relaxed(v, i); +} + +static __always_inline int +raw_atomic_cmpxchg(atomic_t *v, int old, int new) +{ + return arch_atomic_cmpxchg(v, old, new); +} + +static __always_inline int +raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +{ + return arch_atomic_cmpxchg_acquire(v, old, new); +} + +static __always_inline int +raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) +{ + return arch_atomic_cmpxchg_release(v, old, new); +} + +static __always_inline int +raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) +{ + return arch_atomic_cmpxchg_relaxed(v, old, new); +} + +static __always_inline bool +raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ + return arch_atomic_try_cmpxchg(v, old, new); +} + +static __always_inline bool +raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + return arch_atomic_try_cmpxchg_acquire(v, old, new); +} + +static __always_inline bool +raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + return arch_atomic_try_cmpxchg_release(v, old, new); +} + +static __always_inline bool +raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) +{ + return arch_atomic_try_cmpxchg_relaxed(v, old, new); +} + +static __always_inline bool +raw_atomic_sub_and_test(int i, atomic_t *v) +{ + return arch_atomic_sub_and_test(i, v); +} + +static __always_inline bool +raw_atomic_dec_and_test(atomic_t *v) +{ + return arch_atomic_dec_and_test(v); +} + +static __always_inline bool +raw_atomic_inc_and_test(atomic_t *v) +{ + return arch_atomic_inc_and_test(v); +} + +static __always_inline bool +raw_atomic_add_negative(int i, atomic_t *v) +{ + return arch_atomic_add_negative(i, v); +} + +static __always_inline bool +raw_atomic_add_negative_acquire(int i, atomic_t *v) +{ + return arch_atomic_add_negative_acquire(i, v); +} + +static __always_inline bool +raw_atomic_add_negative_release(int i, atomic_t *v) +{ + return arch_atomic_add_negative_release(i, v); +} + +static __always_inline bool +raw_atomic_add_negative_relaxed(int i, atomic_t *v) +{ + return arch_atomic_add_negative_relaxed(i, v); +} + +static __always_inline int +raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + return arch_atomic_fetch_add_unless(v, a, u); +} + +static __always_inline bool +raw_atomic_add_unless(atomic_t *v, int a, int u) +{ + return arch_atomic_add_unless(v, a, u); +} + +static __always_inline bool +raw_atomic_inc_not_zero(atomic_t *v) +{ + return arch_atomic_inc_not_zero(v); +} + +static __always_inline bool +raw_atomic_inc_unless_negative(atomic_t *v) +{ + return arch_atomic_inc_unless_negative(v); +} + +static __always_inline bool +raw_atomic_dec_unless_positive(atomic_t *v) +{ + return arch_atomic_dec_unless_positive(v); +} + +static __always_inline int +raw_atomic_dec_if_positive(atomic_t *v) +{ + return arch_atomic_dec_if_positive(v); +} + +static __always_inline s64 +raw_atomic64_read(const atomic64_t *v) +{ + return arch_atomic64_read(v); +} + +static __always_inline s64 +raw_atomic64_read_acquire(const atomic64_t *v) +{ + return arch_atomic64_read_acquire(v); +} + +static __always_inline void +raw_atomic64_set(atomic64_t *v, s64 i) +{ + arch_atomic64_set(v, i); +} + +static __always_inline void +raw_atomic64_set_release(atomic64_t *v, s64 i) +{ + arch_atomic64_set_release(v, i); +} + +static __always_inline void +raw_atomic64_add(s64 i, atomic64_t *v) +{ + arch_atomic64_add(i, v); +} + +static __always_inline s64 +raw_atomic64_add_return(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_return(i, v); +} + +static __always_inline s64 +raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_return_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_add_return_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_return_release(i, v); +} + +static __always_inline s64 +raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_return_relaxed(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_add(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_add(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_add_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_add_release(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_add_relaxed(i, v); +} + +static __always_inline void +raw_atomic64_sub(s64 i, atomic64_t *v) +{ + arch_atomic64_sub(i, v); +} + +static __always_inline s64 +raw_atomic64_sub_return(s64 i, atomic64_t *v) +{ + return arch_atomic64_sub_return(i, v); +} + +static __always_inline s64 +raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_sub_return_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_sub_return_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_sub_return_release(i, v); +} + +static __always_inline s64 +raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_sub_return_relaxed(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_sub(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_sub(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_sub_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_sub_release(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_sub_relaxed(i, v); +} + +static __always_inline void +raw_atomic64_inc(atomic64_t *v) +{ + arch_atomic64_inc(v); +} + +static __always_inline s64 +raw_atomic64_inc_return(atomic64_t *v) +{ + return arch_atomic64_inc_return(v); +} + +static __always_inline s64 +raw_atomic64_inc_return_acquire(atomic64_t *v) +{ + return arch_atomic64_inc_return_acquire(v); +} + +static __always_inline s64 +raw_atomic64_inc_return_release(atomic64_t *v) +{ + return arch_atomic64_inc_return_release(v); +} + +static __always_inline s64 +raw_atomic64_inc_return_relaxed(atomic64_t *v) +{ + return arch_atomic64_inc_return_relaxed(v); +} + +static __always_inline s64 +raw_atomic64_fetch_inc(atomic64_t *v) +{ + return arch_atomic64_fetch_inc(v); +} + +static __always_inline s64 +raw_atomic64_fetch_inc_acquire(atomic64_t *v) +{ + return arch_atomic64_fetch_inc_acquire(v); +} + +static __always_inline s64 +raw_atomic64_fetch_inc_release(atomic64_t *v) +{ + return arch_atomic64_fetch_inc_release(v); +} + +static __always_inline s64 +raw_atomic64_fetch_inc_relaxed(atomic64_t *v) +{ + return arch_atomic64_fetch_inc_relaxed(v); +} + +static __always_inline void +raw_atomic64_dec(atomic64_t *v) +{ + arch_atomic64_dec(v); +} + +static __always_inline s64 +raw_atomic64_dec_return(atomic64_t *v) +{ + return arch_atomic64_dec_return(v); +} + +static __always_inline s64 +raw_atomic64_dec_return_acquire(atomic64_t *v) +{ + return arch_atomic64_dec_return_acquire(v); +} + +static __always_inline s64 +raw_atomic64_dec_return_release(atomic64_t *v) +{ + return arch_atomic64_dec_return_release(v); +} + +static __always_inline s64 +raw_atomic64_dec_return_relaxed(atomic64_t *v) +{ + return arch_atomic64_dec_return_relaxed(v); +} + +static __always_inline s64 +raw_atomic64_fetch_dec(atomic64_t *v) +{ + return arch_atomic64_fetch_dec(v); +} + +static __always_inline s64 +raw_atomic64_fetch_dec_acquire(atomic64_t *v) +{ + return arch_atomic64_fetch_dec_acquire(v); +} + +static __always_inline s64 +raw_atomic64_fetch_dec_release(atomic64_t *v) +{ + return arch_atomic64_fetch_dec_release(v); +} + +static __always_inline s64 +raw_atomic64_fetch_dec_relaxed(atomic64_t *v) +{ + return arch_atomic64_fetch_dec_relaxed(v); +} + +static __always_inline void +raw_atomic64_and(s64 i, atomic64_t *v) +{ + arch_atomic64_and(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_and(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and_release(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and_relaxed(i, v); +} + +static __always_inline void +raw_atomic64_andnot(s64 i, atomic64_t *v) +{ + arch_atomic64_andnot(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_andnot(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_andnot_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_andnot_release(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_andnot_relaxed(i, v); +} + +static __always_inline void +raw_atomic64_or(s64 i, atomic64_t *v) +{ + arch_atomic64_or(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_or(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_or(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_or_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_or_release(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_or_relaxed(i, v); +} + +static __always_inline void +raw_atomic64_xor(s64 i, atomic64_t *v) +{ + arch_atomic64_xor(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_xor(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_xor(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_xor_acquire(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_xor_release(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_xor_relaxed(i, v); +} + +static __always_inline s64 +raw_atomic64_xchg(atomic64_t *v, s64 i) +{ + return arch_atomic64_xchg(v, i); +} + +static __always_inline s64 +raw_atomic64_xchg_acquire(atomic64_t *v, s64 i) +{ + return arch_atomic64_xchg_acquire(v, i); +} + +static __always_inline s64 +raw_atomic64_xchg_release(atomic64_t *v, s64 i) +{ + return arch_atomic64_xchg_release(v, i); +} + +static __always_inline s64 +raw_atomic64_xchg_relaxed(atomic64_t *v, s64 i) +{ + return arch_atomic64_xchg_relaxed(v, i); +} + +static __always_inline s64 +raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +{ + return arch_atomic64_cmpxchg(v, old, new); +} + +static __always_inline s64 +raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +{ + return arch_atomic64_cmpxchg_acquire(v, old, new); +} + +static __always_inline s64 +raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +{ + return arch_atomic64_cmpxchg_release(v, old, new); +} + +static __always_inline s64 +raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +{ + return arch_atomic64_cmpxchg_relaxed(v, old, new); +} + +static __always_inline bool +raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +{ + return arch_atomic64_try_cmpxchg(v, old, new); +} + +static __always_inline bool +raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + return arch_atomic64_try_cmpxchg_acquire(v, old, new); +} + +static __always_inline bool +raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + return arch_atomic64_try_cmpxchg_release(v, old, new); +} + +static __always_inline bool +raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) +{ + return arch_atomic64_try_cmpxchg_relaxed(v, old, new); +} + +static __always_inline bool +raw_atomic64_sub_and_test(s64 i, atomic64_t *v) +{ + return arch_atomic64_sub_and_test(i, v); +} + +static __always_inline bool +raw_atomic64_dec_and_test(atomic64_t *v) +{ + return arch_atomic64_dec_and_test(v); +} + +static __always_inline bool +raw_atomic64_inc_and_test(atomic64_t *v) +{ + return arch_atomic64_inc_and_test(v); +} + +static __always_inline bool +raw_atomic64_add_negative(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_negative(i, v); +} + +static __always_inline bool +raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_negative_acquire(i, v); +} + +static __always_inline bool +raw_atomic64_add_negative_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_negative_release(i, v); +} + +static __always_inline bool +raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_negative_relaxed(i, v); +} + +static __always_inline s64 +raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return arch_atomic64_fetch_add_unless(v, a, u); +} + +static __always_inline bool +raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return arch_atomic64_add_unless(v, a, u); +} + +static __always_inline bool +raw_atomic64_inc_not_zero(atomic64_t *v) +{ + return arch_atomic64_inc_not_zero(v); +} + +static __always_inline bool +raw_atomic64_inc_unless_negative(atomic64_t *v) +{ + return arch_atomic64_inc_unless_negative(v); +} + +static __always_inline bool +raw_atomic64_dec_unless_positive(atomic64_t *v) +{ + return arch_atomic64_dec_unless_positive(v); +} + +static __always_inline s64 +raw_atomic64_dec_if_positive(atomic64_t *v) +{ + return arch_atomic64_dec_if_positive(v); +} + +static __always_inline long +raw_atomic_long_read(const atomic_long_t *v) +{ + return arch_atomic_long_read(v); +} + +static __always_inline long +raw_atomic_long_read_acquire(const atomic_long_t *v) +{ + return arch_atomic_long_read_acquire(v); +} + +static __always_inline void +raw_atomic_long_set(atomic_long_t *v, long i) +{ + arch_atomic_long_set(v, i); +} + +static __always_inline void +raw_atomic_long_set_release(atomic_long_t *v, long i) +{ + arch_atomic_long_set_release(v, i); +} + +static __always_inline void +raw_atomic_long_add(long i, atomic_long_t *v) +{ + arch_atomic_long_add(i, v); +} + +static __always_inline long +raw_atomic_long_add_return(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_return(i, v); +} + +static __always_inline long +raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_return_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_add_return_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_return_release(i, v); +} + +static __always_inline long +raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_return_relaxed(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_add(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_add(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_add_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_add_release(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_add_relaxed(i, v); +} + +static __always_inline void +raw_atomic_long_sub(long i, atomic_long_t *v) +{ + arch_atomic_long_sub(i, v); +} + +static __always_inline long +raw_atomic_long_sub_return(long i, atomic_long_t *v) +{ + return arch_atomic_long_sub_return(i, v); +} + +static __always_inline long +raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_sub_return_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_sub_return_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_sub_return_release(i, v); +} + +static __always_inline long +raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_sub_return_relaxed(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_sub(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_sub(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_sub_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_sub_release(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_sub_relaxed(i, v); +} + +static __always_inline void +raw_atomic_long_inc(atomic_long_t *v) +{ + arch_atomic_long_inc(v); +} + +static __always_inline long +raw_atomic_long_inc_return(atomic_long_t *v) +{ + return arch_atomic_long_inc_return(v); +} + +static __always_inline long +raw_atomic_long_inc_return_acquire(atomic_long_t *v) +{ + return arch_atomic_long_inc_return_acquire(v); +} + +static __always_inline long +raw_atomic_long_inc_return_release(atomic_long_t *v) +{ + return arch_atomic_long_inc_return_release(v); +} + +static __always_inline long +raw_atomic_long_inc_return_relaxed(atomic_long_t *v) +{ + return arch_atomic_long_inc_return_relaxed(v); +} + +static __always_inline long +raw_atomic_long_fetch_inc(atomic_long_t *v) +{ + return arch_atomic_long_fetch_inc(v); +} + +static __always_inline long +raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) +{ + return arch_atomic_long_fetch_inc_acquire(v); +} + +static __always_inline long +raw_atomic_long_fetch_inc_release(atomic_long_t *v) +{ + return arch_atomic_long_fetch_inc_release(v); +} + +static __always_inline long +raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) +{ + return arch_atomic_long_fetch_inc_relaxed(v); +} + +static __always_inline void +raw_atomic_long_dec(atomic_long_t *v) +{ + arch_atomic_long_dec(v); +} + +static __always_inline long +raw_atomic_long_dec_return(atomic_long_t *v) +{ + return arch_atomic_long_dec_return(v); +} + +static __always_inline long +raw_atomic_long_dec_return_acquire(atomic_long_t *v) +{ + return arch_atomic_long_dec_return_acquire(v); +} + +static __always_inline long +raw_atomic_long_dec_return_release(atomic_long_t *v) +{ + return arch_atomic_long_dec_return_release(v); +} + +static __always_inline long +raw_atomic_long_dec_return_relaxed(atomic_long_t *v) +{ + return arch_atomic_long_dec_return_relaxed(v); +} + +static __always_inline long +raw_atomic_long_fetch_dec(atomic_long_t *v) +{ + return arch_atomic_long_fetch_dec(v); +} + +static __always_inline long +raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) +{ + return arch_atomic_long_fetch_dec_acquire(v); +} + +static __always_inline long +raw_atomic_long_fetch_dec_release(atomic_long_t *v) +{ + return arch_atomic_long_fetch_dec_release(v); +} + +static __always_inline long +raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) +{ + return arch_atomic_long_fetch_dec_relaxed(v); +} + +static __always_inline void +raw_atomic_long_and(long i, atomic_long_t *v) +{ + arch_atomic_long_and(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_and(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_and(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_and_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_and_release(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_and_relaxed(i, v); +} + +static __always_inline void +raw_atomic_long_andnot(long i, atomic_long_t *v) +{ + arch_atomic_long_andnot(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_andnot(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_andnot_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_andnot_release(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_andnot_relaxed(i, v); +} + +static __always_inline void +raw_atomic_long_or(long i, atomic_long_t *v) +{ + arch_atomic_long_or(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_or(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_or(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_or_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_or_release(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_or_relaxed(i, v); +} + +static __always_inline void +raw_atomic_long_xor(long i, atomic_long_t *v) +{ + arch_atomic_long_xor(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_xor(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_xor(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_xor_acquire(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_xor_release(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_fetch_xor_relaxed(i, v); +} + +static __always_inline long +raw_atomic_long_xchg(atomic_long_t *v, long i) +{ + return arch_atomic_long_xchg(v, i); +} + +static __always_inline long +raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) +{ + return arch_atomic_long_xchg_acquire(v, i); +} + +static __always_inline long +raw_atomic_long_xchg_release(atomic_long_t *v, long i) +{ + return arch_atomic_long_xchg_release(v, i); +} + +static __always_inline long +raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) +{ + return arch_atomic_long_xchg_relaxed(v, i); +} + +static __always_inline long +raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) +{ + return arch_atomic_long_cmpxchg(v, old, new); +} + +static __always_inline long +raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) +{ + return arch_atomic_long_cmpxchg_acquire(v, old, new); +} + +static __always_inline long +raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) +{ + return arch_atomic_long_cmpxchg_release(v, old, new); +} + +static __always_inline long +raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) +{ + return arch_atomic_long_cmpxchg_relaxed(v, old, new); +} + +static __always_inline bool +raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) +{ + return arch_atomic_long_try_cmpxchg(v, old, new); +} + +static __always_inline bool +raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) +{ + return arch_atomic_long_try_cmpxchg_acquire(v, old, new); +} + +static __always_inline bool +raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) +{ + return arch_atomic_long_try_cmpxchg_release(v, old, new); +} + +static __always_inline bool +raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) +{ + return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); +} + +static __always_inline bool +raw_atomic_long_sub_and_test(long i, atomic_long_t *v) +{ + return arch_atomic_long_sub_and_test(i, v); +} + +static __always_inline bool +raw_atomic_long_dec_and_test(atomic_long_t *v) +{ + return arch_atomic_long_dec_and_test(v); +} + +static __always_inline bool +raw_atomic_long_inc_and_test(atomic_long_t *v) +{ + return arch_atomic_long_inc_and_test(v); +} + +static __always_inline bool +raw_atomic_long_add_negative(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_negative(i, v); +} + +static __always_inline bool +raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_negative_acquire(i, v); +} + +static __always_inline bool +raw_atomic_long_add_negative_release(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_negative_release(i, v); +} + +static __always_inline bool +raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) +{ + return arch_atomic_long_add_negative_relaxed(i, v); +} + +static __always_inline long +raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) +{ + return arch_atomic_long_fetch_add_unless(v, a, u); +} + +static __always_inline bool +raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) +{ + return arch_atomic_long_add_unless(v, a, u); +} + +static __always_inline bool +raw_atomic_long_inc_not_zero(atomic_long_t *v) +{ + return arch_atomic_long_inc_not_zero(v); +} + +static __always_inline bool +raw_atomic_long_inc_unless_negative(atomic_long_t *v) +{ + return arch_atomic_long_inc_unless_negative(v); +} + +static __always_inline bool +raw_atomic_long_dec_unless_positive(atomic_long_t *v) +{ + return arch_atomic_long_dec_unless_positive(v); +} + +static __always_inline long +raw_atomic_long_dec_if_positive(atomic_long_t *v) +{ + return arch_atomic_long_dec_if_positive(v); +} + +#define raw_xchg(...) \ + arch_xchg(__VA_ARGS__) + +#define raw_xchg_acquire(...) \ + arch_xchg_acquire(__VA_ARGS__) + +#define raw_xchg_release(...) \ + arch_xchg_release(__VA_ARGS__) + +#define raw_xchg_relaxed(...) \ + arch_xchg_relaxed(__VA_ARGS__) + +#define raw_cmpxchg(...) \ + arch_cmpxchg(__VA_ARGS__) + +#define raw_cmpxchg_acquire(...) \ + arch_cmpxchg_acquire(__VA_ARGS__) + +#define raw_cmpxchg_release(...) \ + arch_cmpxchg_release(__VA_ARGS__) + +#define raw_cmpxchg_relaxed(...) \ + arch_cmpxchg_relaxed(__VA_ARGS__) + +#define raw_cmpxchg64(...) \ + arch_cmpxchg64(__VA_ARGS__) + +#define raw_cmpxchg64_acquire(...) \ + arch_cmpxchg64_acquire(__VA_ARGS__) + +#define raw_cmpxchg64_release(...) \ + arch_cmpxchg64_release(__VA_ARGS__) + +#define raw_cmpxchg64_relaxed(...) \ + arch_cmpxchg64_relaxed(__VA_ARGS__) + +#define raw_cmpxchg128(...) \ + arch_cmpxchg128(__VA_ARGS__) + +#define raw_cmpxchg128_acquire(...) \ + arch_cmpxchg128_acquire(__VA_ARGS__) + +#define raw_cmpxchg128_release(...) \ + arch_cmpxchg128_release(__VA_ARGS__) + +#define raw_cmpxchg128_relaxed(...) \ + arch_cmpxchg128_relaxed(__VA_ARGS__) + +#define raw_try_cmpxchg(...) \ + arch_try_cmpxchg(__VA_ARGS__) + +#define raw_try_cmpxchg_acquire(...) \ + arch_try_cmpxchg_acquire(__VA_ARGS__) + +#define raw_try_cmpxchg_release(...) \ + arch_try_cmpxchg_release(__VA_ARGS__) + +#define raw_try_cmpxchg_relaxed(...) \ + arch_try_cmpxchg_relaxed(__VA_ARGS__) + +#define raw_try_cmpxchg64(...) \ + arch_try_cmpxchg64(__VA_ARGS__) + +#define raw_try_cmpxchg64_acquire(...) \ + arch_try_cmpxchg64_acquire(__VA_ARGS__) + +#define raw_try_cmpxchg64_release(...) \ + arch_try_cmpxchg64_release(__VA_ARGS__) + +#define raw_try_cmpxchg64_relaxed(...) \ + arch_try_cmpxchg64_relaxed(__VA_ARGS__) + +#define raw_try_cmpxchg128(...) \ + arch_try_cmpxchg128(__VA_ARGS__) + +#define raw_try_cmpxchg128_acquire(...) \ + arch_try_cmpxchg128_acquire(__VA_ARGS__) + +#define raw_try_cmpxchg128_release(...) \ + arch_try_cmpxchg128_release(__VA_ARGS__) + +#define raw_try_cmpxchg128_relaxed(...) \ + arch_try_cmpxchg128_relaxed(__VA_ARGS__) + +#define raw_cmpxchg_local(...) \ + arch_cmpxchg_local(__VA_ARGS__) + +#define raw_cmpxchg64_local(...) \ + arch_cmpxchg64_local(__VA_ARGS__) + +#define raw_cmpxchg128_local(...) \ + arch_cmpxchg128_local(__VA_ARGS__) + +#define raw_sync_cmpxchg(...) \ + arch_sync_cmpxchg(__VA_ARGS__) + +#define raw_try_cmpxchg_local(...) \ + arch_try_cmpxchg_local(__VA_ARGS__) + +#define raw_try_cmpxchg64_local(...) \ + arch_try_cmpxchg64_local(__VA_ARGS__) + +#define raw_try_cmpxchg128_local(...) \ + arch_try_cmpxchg128_local(__VA_ARGS__) + +#endif /* _LINUX_ATOMIC_RAW_H */ +// 01d54200571b3857755a07c10074a4fd58cef6b1 -- cgit v1.2.3 From 0f613bfa8268a89be25f2b6b58fc6fe8ccd9a2ba Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:15 +0100 Subject: locking/atomic: treewide: use raw_atomic*_() Now that we have raw_atomic*_() definitions, there's no need to use arch_atomic*_() definitions outside of the low-level atomic definitions. Move treewide users of arch_atomic*_() over to the equivalent raw_atomic*_(). There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-19-mark.rutland@arm.com --- include/linux/context_tracking.h | 4 ++-- include/linux/context_tracking_state.h | 2 +- include/linux/cpumask.h | 2 +- include/linux/jump_label.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index d3cbb6c16bab..6e76b9dba00e 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -119,7 +119,7 @@ extern void ct_idle_exit(void); */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { - return !(arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX); + return !(raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & RCU_DYNTICKS_IDX); } /* @@ -128,7 +128,7 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) */ static __always_inline unsigned long ct_state_inc(int incby) { - return arch_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state)); + return raw_atomic_add_return(incby, this_cpu_ptr(&context_tracking.state)); } static __always_inline bool warn_rcu_enter(void) diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index fdd537ea513f..bbff5f7f8803 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -51,7 +51,7 @@ DECLARE_PER_CPU(struct context_tracking, context_tracking); #ifdef CONFIG_CONTEXT_TRACKING_USER static __always_inline int __ct_state(void) { - return arch_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; + return raw_atomic_read(this_cpu_ptr(&context_tracking.state)) & CT_STATE_MASK; } #endif diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index ca736b05ec7b..0d2e2a38b92d 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -1071,7 +1071,7 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu) */ static __always_inline unsigned int num_online_cpus(void) { - return arch_atomic_read(&__num_online_cpus); + return raw_atomic_read(&__num_online_cpus); } #define num_possible_cpus() cpumask_weight(cpu_possible_mask) #define num_present_cpus() cpumask_weight(cpu_present_mask) diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 4e968ebadce6..f0a949b7c973 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -257,7 +257,7 @@ extern enum jump_label_type jump_label_init_type(struct jump_entry *entry); static __always_inline int static_key_count(struct static_key *key) { - return arch_atomic_read(&key->enabled); + return raw_atomic_read(&key->enabled); } static __always_inline void jump_label_init(void) -- cgit v1.2.3 From 1815da1718aa4c062b94cf3fc09432f552e25768 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:16 +0100 Subject: locking/atomic: scripts: build raw_atomic_long*() directly Now that arch_atomic*() usage is limited to the atomic headers, we no longer have any users of arch_atomic_long_*(), and can generate raw_atomic_long_*() directly. Generate the raw_atomic_long_*() ops directly. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-20-mark.rutland@arm.com --- include/linux/atomic.h | 2 +- include/linux/atomic/atomic-long.h | 682 ++++++++++++++++++------------------- include/linux/atomic/atomic-raw.h | 512 +--------------------------- 3 files changed, 343 insertions(+), 853 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 127f5dc63a7d..296cfae0389f 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -78,8 +78,8 @@ }) #include -#include #include +#include #include #endif /* _LINUX_ATOMIC_H */ diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h index 2fc51ba66beb..92dc82ce1ce6 100644 --- a/include/linux/atomic/atomic-long.h +++ b/include/linux/atomic/atomic-long.h @@ -24,1027 +24,1027 @@ typedef atomic_t atomic_long_t; #ifdef CONFIG_64BIT static __always_inline long -arch_atomic_long_read(const atomic_long_t *v) +raw_atomic_long_read(const atomic_long_t *v) { - return arch_atomic64_read(v); + return raw_atomic64_read(v); } static __always_inline long -arch_atomic_long_read_acquire(const atomic_long_t *v) +raw_atomic_long_read_acquire(const atomic_long_t *v) { - return arch_atomic64_read_acquire(v); + return raw_atomic64_read_acquire(v); } static __always_inline void -arch_atomic_long_set(atomic_long_t *v, long i) +raw_atomic_long_set(atomic_long_t *v, long i) { - arch_atomic64_set(v, i); + raw_atomic64_set(v, i); } static __always_inline void -arch_atomic_long_set_release(atomic_long_t *v, long i) +raw_atomic_long_set_release(atomic_long_t *v, long i) { - arch_atomic64_set_release(v, i); + raw_atomic64_set_release(v, i); } static __always_inline void -arch_atomic_long_add(long i, atomic_long_t *v) +raw_atomic_long_add(long i, atomic_long_t *v) { - arch_atomic64_add(i, v); + raw_atomic64_add(i, v); } static __always_inline long -arch_atomic_long_add_return(long i, atomic_long_t *v) +raw_atomic_long_add_return(long i, atomic_long_t *v) { - return arch_atomic64_add_return(i, v); + return raw_atomic64_add_return(i, v); } static __always_inline long -arch_atomic_long_add_return_acquire(long i, atomic_long_t *v) +raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) { - return arch_atomic64_add_return_acquire(i, v); + return raw_atomic64_add_return_acquire(i, v); } static __always_inline long -arch_atomic_long_add_return_release(long i, atomic_long_t *v) +raw_atomic_long_add_return_release(long i, atomic_long_t *v) { - return arch_atomic64_add_return_release(i, v); + return raw_atomic64_add_return_release(i, v); } static __always_inline long -arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v) +raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_add_return_relaxed(i, v); + return raw_atomic64_add_return_relaxed(i, v); } static __always_inline long -arch_atomic_long_fetch_add(long i, atomic_long_t *v) +raw_atomic_long_fetch_add(long i, atomic_long_t *v) { - return arch_atomic64_fetch_add(i, v); + return raw_atomic64_fetch_add(i, v); } static __always_inline long -arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { - return arch_atomic64_fetch_add_acquire(i, v); + return raw_atomic64_fetch_add_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_add_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) { - return arch_atomic64_fetch_add_release(i, v); + return raw_atomic64_fetch_add_release(i, v); } static __always_inline long -arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_fetch_add_relaxed(i, v); + return raw_atomic64_fetch_add_relaxed(i, v); } static __always_inline void -arch_atomic_long_sub(long i, atomic_long_t *v) +raw_atomic_long_sub(long i, atomic_long_t *v) { - arch_atomic64_sub(i, v); + raw_atomic64_sub(i, v); } static __always_inline long -arch_atomic_long_sub_return(long i, atomic_long_t *v) +raw_atomic_long_sub_return(long i, atomic_long_t *v) { - return arch_atomic64_sub_return(i, v); + return raw_atomic64_sub_return(i, v); } static __always_inline long -arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v) +raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) { - return arch_atomic64_sub_return_acquire(i, v); + return raw_atomic64_sub_return_acquire(i, v); } static __always_inline long -arch_atomic_long_sub_return_release(long i, atomic_long_t *v) +raw_atomic_long_sub_return_release(long i, atomic_long_t *v) { - return arch_atomic64_sub_return_release(i, v); + return raw_atomic64_sub_return_release(i, v); } static __always_inline long -arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) +raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_sub_return_relaxed(i, v); + return raw_atomic64_sub_return_relaxed(i, v); } static __always_inline long -arch_atomic_long_fetch_sub(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub(long i, atomic_long_t *v) { - return arch_atomic64_fetch_sub(i, v); + return raw_atomic64_fetch_sub(i, v); } static __always_inline long -arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { - return arch_atomic64_fetch_sub_acquire(i, v); + return raw_atomic64_fetch_sub_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) { - return arch_atomic64_fetch_sub_release(i, v); + return raw_atomic64_fetch_sub_release(i, v); } static __always_inline long -arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_fetch_sub_relaxed(i, v); + return raw_atomic64_fetch_sub_relaxed(i, v); } static __always_inline void -arch_atomic_long_inc(atomic_long_t *v) +raw_atomic_long_inc(atomic_long_t *v) { - arch_atomic64_inc(v); + raw_atomic64_inc(v); } static __always_inline long -arch_atomic_long_inc_return(atomic_long_t *v) +raw_atomic_long_inc_return(atomic_long_t *v) { - return arch_atomic64_inc_return(v); + return raw_atomic64_inc_return(v); } static __always_inline long -arch_atomic_long_inc_return_acquire(atomic_long_t *v) +raw_atomic_long_inc_return_acquire(atomic_long_t *v) { - return arch_atomic64_inc_return_acquire(v); + return raw_atomic64_inc_return_acquire(v); } static __always_inline long -arch_atomic_long_inc_return_release(atomic_long_t *v) +raw_atomic_long_inc_return_release(atomic_long_t *v) { - return arch_atomic64_inc_return_release(v); + return raw_atomic64_inc_return_release(v); } static __always_inline long -arch_atomic_long_inc_return_relaxed(atomic_long_t *v) +raw_atomic_long_inc_return_relaxed(atomic_long_t *v) { - return arch_atomic64_inc_return_relaxed(v); + return raw_atomic64_inc_return_relaxed(v); } static __always_inline long -arch_atomic_long_fetch_inc(atomic_long_t *v) +raw_atomic_long_fetch_inc(atomic_long_t *v) { - return arch_atomic64_fetch_inc(v); + return raw_atomic64_fetch_inc(v); } static __always_inline long -arch_atomic_long_fetch_inc_acquire(atomic_long_t *v) +raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) { - return arch_atomic64_fetch_inc_acquire(v); + return raw_atomic64_fetch_inc_acquire(v); } static __always_inline long -arch_atomic_long_fetch_inc_release(atomic_long_t *v) +raw_atomic_long_fetch_inc_release(atomic_long_t *v) { - return arch_atomic64_fetch_inc_release(v); + return raw_atomic64_fetch_inc_release(v); } static __always_inline long -arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v) +raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) { - return arch_atomic64_fetch_inc_relaxed(v); + return raw_atomic64_fetch_inc_relaxed(v); } static __always_inline void -arch_atomic_long_dec(atomic_long_t *v) +raw_atomic_long_dec(atomic_long_t *v) { - arch_atomic64_dec(v); + raw_atomic64_dec(v); } static __always_inline long -arch_atomic_long_dec_return(atomic_long_t *v) +raw_atomic_long_dec_return(atomic_long_t *v) { - return arch_atomic64_dec_return(v); + return raw_atomic64_dec_return(v); } static __always_inline long -arch_atomic_long_dec_return_acquire(atomic_long_t *v) +raw_atomic_long_dec_return_acquire(atomic_long_t *v) { - return arch_atomic64_dec_return_acquire(v); + return raw_atomic64_dec_return_acquire(v); } static __always_inline long -arch_atomic_long_dec_return_release(atomic_long_t *v) +raw_atomic_long_dec_return_release(atomic_long_t *v) { - return arch_atomic64_dec_return_release(v); + return raw_atomic64_dec_return_release(v); } static __always_inline long -arch_atomic_long_dec_return_relaxed(atomic_long_t *v) +raw_atomic_long_dec_return_relaxed(atomic_long_t *v) { - return arch_atomic64_dec_return_relaxed(v); + return raw_atomic64_dec_return_relaxed(v); } static __always_inline long -arch_atomic_long_fetch_dec(atomic_long_t *v) +raw_atomic_long_fetch_dec(atomic_long_t *v) { - return arch_atomic64_fetch_dec(v); + return raw_atomic64_fetch_dec(v); } static __always_inline long -arch_atomic_long_fetch_dec_acquire(atomic_long_t *v) +raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) { - return arch_atomic64_fetch_dec_acquire(v); + return raw_atomic64_fetch_dec_acquire(v); } static __always_inline long -arch_atomic_long_fetch_dec_release(atomic_long_t *v) +raw_atomic_long_fetch_dec_release(atomic_long_t *v) { - return arch_atomic64_fetch_dec_release(v); + return raw_atomic64_fetch_dec_release(v); } static __always_inline long -arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v) +raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) { - return arch_atomic64_fetch_dec_relaxed(v); + return raw_atomic64_fetch_dec_relaxed(v); } static __always_inline void -arch_atomic_long_and(long i, atomic_long_t *v) +raw_atomic_long_and(long i, atomic_long_t *v) { - arch_atomic64_and(i, v); + raw_atomic64_and(i, v); } static __always_inline long -arch_atomic_long_fetch_and(long i, atomic_long_t *v) +raw_atomic_long_fetch_and(long i, atomic_long_t *v) { - return arch_atomic64_fetch_and(i, v); + return raw_atomic64_fetch_and(i, v); } static __always_inline long -arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { - return arch_atomic64_fetch_and_acquire(i, v); + return raw_atomic64_fetch_and_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_and_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) { - return arch_atomic64_fetch_and_release(i, v); + return raw_atomic64_fetch_and_release(i, v); } static __always_inline long -arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_fetch_and_relaxed(i, v); + return raw_atomic64_fetch_and_relaxed(i, v); } static __always_inline void -arch_atomic_long_andnot(long i, atomic_long_t *v) +raw_atomic_long_andnot(long i, atomic_long_t *v) { - arch_atomic64_andnot(i, v); + raw_atomic64_andnot(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) { - return arch_atomic64_fetch_andnot(i, v); + return raw_atomic64_fetch_andnot(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { - return arch_atomic64_fetch_andnot_acquire(i, v); + return raw_atomic64_fetch_andnot_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { - return arch_atomic64_fetch_andnot_release(i, v); + return raw_atomic64_fetch_andnot_release(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_fetch_andnot_relaxed(i, v); + return raw_atomic64_fetch_andnot_relaxed(i, v); } static __always_inline void -arch_atomic_long_or(long i, atomic_long_t *v) +raw_atomic_long_or(long i, atomic_long_t *v) { - arch_atomic64_or(i, v); + raw_atomic64_or(i, v); } static __always_inline long -arch_atomic_long_fetch_or(long i, atomic_long_t *v) +raw_atomic_long_fetch_or(long i, atomic_long_t *v) { - return arch_atomic64_fetch_or(i, v); + return raw_atomic64_fetch_or(i, v); } static __always_inline long -arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { - return arch_atomic64_fetch_or_acquire(i, v); + return raw_atomic64_fetch_or_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_or_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) { - return arch_atomic64_fetch_or_release(i, v); + return raw_atomic64_fetch_or_release(i, v); } static __always_inline long -arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_fetch_or_relaxed(i, v); + return raw_atomic64_fetch_or_relaxed(i, v); } static __always_inline void -arch_atomic_long_xor(long i, atomic_long_t *v) +raw_atomic_long_xor(long i, atomic_long_t *v) { - arch_atomic64_xor(i, v); + raw_atomic64_xor(i, v); } static __always_inline long -arch_atomic_long_fetch_xor(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor(long i, atomic_long_t *v) { - return arch_atomic64_fetch_xor(i, v); + return raw_atomic64_fetch_xor(i, v); } static __always_inline long -arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { - return arch_atomic64_fetch_xor_acquire(i, v); + return raw_atomic64_fetch_xor_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) { - return arch_atomic64_fetch_xor_release(i, v); + return raw_atomic64_fetch_xor_release(i, v); } static __always_inline long -arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_fetch_xor_relaxed(i, v); + return raw_atomic64_fetch_xor_relaxed(i, v); } static __always_inline long -arch_atomic_long_xchg(atomic_long_t *v, long i) +raw_atomic_long_xchg(atomic_long_t *v, long i) { - return arch_atomic64_xchg(v, i); + return raw_atomic64_xchg(v, i); } static __always_inline long -arch_atomic_long_xchg_acquire(atomic_long_t *v, long i) +raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) { - return arch_atomic64_xchg_acquire(v, i); + return raw_atomic64_xchg_acquire(v, i); } static __always_inline long -arch_atomic_long_xchg_release(atomic_long_t *v, long i) +raw_atomic_long_xchg_release(atomic_long_t *v, long i) { - return arch_atomic64_xchg_release(v, i); + return raw_atomic64_xchg_release(v, i); } static __always_inline long -arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i) +raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) { - return arch_atomic64_xchg_relaxed(v, i); + return raw_atomic64_xchg_relaxed(v, i); } static __always_inline long -arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { - return arch_atomic64_cmpxchg(v, old, new); + return raw_atomic64_cmpxchg(v, old, new); } static __always_inline long -arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { - return arch_atomic64_cmpxchg_acquire(v, old, new); + return raw_atomic64_cmpxchg_acquire(v, old, new); } static __always_inline long -arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { - return arch_atomic64_cmpxchg_release(v, old, new); + return raw_atomic64_cmpxchg_release(v, old, new); } static __always_inline long -arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { - return arch_atomic64_cmpxchg_relaxed(v, old, new); + return raw_atomic64_cmpxchg_relaxed(v, old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { - return arch_atomic64_try_cmpxchg(v, (s64 *)old, new); + return raw_atomic64_try_cmpxchg(v, (s64 *)old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { - return arch_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new); + return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { - return arch_atomic64_try_cmpxchg_release(v, (s64 *)old, new); + return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { - return arch_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new); + return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new); } static __always_inline bool -arch_atomic_long_sub_and_test(long i, atomic_long_t *v) +raw_atomic_long_sub_and_test(long i, atomic_long_t *v) { - return arch_atomic64_sub_and_test(i, v); + return raw_atomic64_sub_and_test(i, v); } static __always_inline bool -arch_atomic_long_dec_and_test(atomic_long_t *v) +raw_atomic_long_dec_and_test(atomic_long_t *v) { - return arch_atomic64_dec_and_test(v); + return raw_atomic64_dec_and_test(v); } static __always_inline bool -arch_atomic_long_inc_and_test(atomic_long_t *v) +raw_atomic_long_inc_and_test(atomic_long_t *v) { - return arch_atomic64_inc_and_test(v); + return raw_atomic64_inc_and_test(v); } static __always_inline bool -arch_atomic_long_add_negative(long i, atomic_long_t *v) +raw_atomic_long_add_negative(long i, atomic_long_t *v) { - return arch_atomic64_add_negative(i, v); + return raw_atomic64_add_negative(i, v); } static __always_inline bool -arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v) +raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) { - return arch_atomic64_add_negative_acquire(i, v); + return raw_atomic64_add_negative_acquire(i, v); } static __always_inline bool -arch_atomic_long_add_negative_release(long i, atomic_long_t *v) +raw_atomic_long_add_negative_release(long i, atomic_long_t *v) { - return arch_atomic64_add_negative_release(i, v); + return raw_atomic64_add_negative_release(i, v); } static __always_inline bool -arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) +raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) { - return arch_atomic64_add_negative_relaxed(i, v); + return raw_atomic64_add_negative_relaxed(i, v); } static __always_inline long -arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) +raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { - return arch_atomic64_fetch_add_unless(v, a, u); + return raw_atomic64_fetch_add_unless(v, a, u); } static __always_inline bool -arch_atomic_long_add_unless(atomic_long_t *v, long a, long u) +raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) { - return arch_atomic64_add_unless(v, a, u); + return raw_atomic64_add_unless(v, a, u); } static __always_inline bool -arch_atomic_long_inc_not_zero(atomic_long_t *v) +raw_atomic_long_inc_not_zero(atomic_long_t *v) { - return arch_atomic64_inc_not_zero(v); + return raw_atomic64_inc_not_zero(v); } static __always_inline bool -arch_atomic_long_inc_unless_negative(atomic_long_t *v) +raw_atomic_long_inc_unless_negative(atomic_long_t *v) { - return arch_atomic64_inc_unless_negative(v); + return raw_atomic64_inc_unless_negative(v); } static __always_inline bool -arch_atomic_long_dec_unless_positive(atomic_long_t *v) +raw_atomic_long_dec_unless_positive(atomic_long_t *v) { - return arch_atomic64_dec_unless_positive(v); + return raw_atomic64_dec_unless_positive(v); } static __always_inline long -arch_atomic_long_dec_if_positive(atomic_long_t *v) +raw_atomic_long_dec_if_positive(atomic_long_t *v) { - return arch_atomic64_dec_if_positive(v); + return raw_atomic64_dec_if_positive(v); } #else /* CONFIG_64BIT */ static __always_inline long -arch_atomic_long_read(const atomic_long_t *v) +raw_atomic_long_read(const atomic_long_t *v) { - return arch_atomic_read(v); + return raw_atomic_read(v); } static __always_inline long -arch_atomic_long_read_acquire(const atomic_long_t *v) +raw_atomic_long_read_acquire(const atomic_long_t *v) { - return arch_atomic_read_acquire(v); + return raw_atomic_read_acquire(v); } static __always_inline void -arch_atomic_long_set(atomic_long_t *v, long i) +raw_atomic_long_set(atomic_long_t *v, long i) { - arch_atomic_set(v, i); + raw_atomic_set(v, i); } static __always_inline void -arch_atomic_long_set_release(atomic_long_t *v, long i) +raw_atomic_long_set_release(atomic_long_t *v, long i) { - arch_atomic_set_release(v, i); + raw_atomic_set_release(v, i); } static __always_inline void -arch_atomic_long_add(long i, atomic_long_t *v) +raw_atomic_long_add(long i, atomic_long_t *v) { - arch_atomic_add(i, v); + raw_atomic_add(i, v); } static __always_inline long -arch_atomic_long_add_return(long i, atomic_long_t *v) +raw_atomic_long_add_return(long i, atomic_long_t *v) { - return arch_atomic_add_return(i, v); + return raw_atomic_add_return(i, v); } static __always_inline long -arch_atomic_long_add_return_acquire(long i, atomic_long_t *v) +raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) { - return arch_atomic_add_return_acquire(i, v); + return raw_atomic_add_return_acquire(i, v); } static __always_inline long -arch_atomic_long_add_return_release(long i, atomic_long_t *v) +raw_atomic_long_add_return_release(long i, atomic_long_t *v) { - return arch_atomic_add_return_release(i, v); + return raw_atomic_add_return_release(i, v); } static __always_inline long -arch_atomic_long_add_return_relaxed(long i, atomic_long_t *v) +raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) { - return arch_atomic_add_return_relaxed(i, v); + return raw_atomic_add_return_relaxed(i, v); } static __always_inline long -arch_atomic_long_fetch_add(long i, atomic_long_t *v) +raw_atomic_long_fetch_add(long i, atomic_long_t *v) { - return arch_atomic_fetch_add(i, v); + return raw_atomic_fetch_add(i, v); } static __always_inline long -arch_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { - return arch_atomic_fetch_add_acquire(i, v); + return raw_atomic_fetch_add_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_add_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) { - return arch_atomic_fetch_add_release(i, v); + return raw_atomic_fetch_add_release(i, v); } static __always_inline long -arch_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { - return arch_atomic_fetch_add_relaxed(i, v); + return raw_atomic_fetch_add_relaxed(i, v); } static __always_inline void -arch_atomic_long_sub(long i, atomic_long_t *v) +raw_atomic_long_sub(long i, atomic_long_t *v) { - arch_atomic_sub(i, v); + raw_atomic_sub(i, v); } static __always_inline long -arch_atomic_long_sub_return(long i, atomic_long_t *v) +raw_atomic_long_sub_return(long i, atomic_long_t *v) { - return arch_atomic_sub_return(i, v); + return raw_atomic_sub_return(i, v); } static __always_inline long -arch_atomic_long_sub_return_acquire(long i, atomic_long_t *v) +raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) { - return arch_atomic_sub_return_acquire(i, v); + return raw_atomic_sub_return_acquire(i, v); } static __always_inline long -arch_atomic_long_sub_return_release(long i, atomic_long_t *v) +raw_atomic_long_sub_return_release(long i, atomic_long_t *v) { - return arch_atomic_sub_return_release(i, v); + return raw_atomic_sub_return_release(i, v); } static __always_inline long -arch_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) +raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { - return arch_atomic_sub_return_relaxed(i, v); + return raw_atomic_sub_return_relaxed(i, v); } static __always_inline long -arch_atomic_long_fetch_sub(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub(long i, atomic_long_t *v) { - return arch_atomic_fetch_sub(i, v); + return raw_atomic_fetch_sub(i, v); } static __always_inline long -arch_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { - return arch_atomic_fetch_sub_acquire(i, v); + return raw_atomic_fetch_sub_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_sub_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) { - return arch_atomic_fetch_sub_release(i, v); + return raw_atomic_fetch_sub_release(i, v); } static __always_inline long -arch_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { - return arch_atomic_fetch_sub_relaxed(i, v); + return raw_atomic_fetch_sub_relaxed(i, v); } static __always_inline void -arch_atomic_long_inc(atomic_long_t *v) +raw_atomic_long_inc(atomic_long_t *v) { - arch_atomic_inc(v); + raw_atomic_inc(v); } static __always_inline long -arch_atomic_long_inc_return(atomic_long_t *v) +raw_atomic_long_inc_return(atomic_long_t *v) { - return arch_atomic_inc_return(v); + return raw_atomic_inc_return(v); } static __always_inline long -arch_atomic_long_inc_return_acquire(atomic_long_t *v) +raw_atomic_long_inc_return_acquire(atomic_long_t *v) { - return arch_atomic_inc_return_acquire(v); + return raw_atomic_inc_return_acquire(v); } static __always_inline long -arch_atomic_long_inc_return_release(atomic_long_t *v) +raw_atomic_long_inc_return_release(atomic_long_t *v) { - return arch_atomic_inc_return_release(v); + return raw_atomic_inc_return_release(v); } static __always_inline long -arch_atomic_long_inc_return_relaxed(atomic_long_t *v) +raw_atomic_long_inc_return_relaxed(atomic_long_t *v) { - return arch_atomic_inc_return_relaxed(v); + return raw_atomic_inc_return_relaxed(v); } static __always_inline long -arch_atomic_long_fetch_inc(atomic_long_t *v) +raw_atomic_long_fetch_inc(atomic_long_t *v) { - return arch_atomic_fetch_inc(v); + return raw_atomic_fetch_inc(v); } static __always_inline long -arch_atomic_long_fetch_inc_acquire(atomic_long_t *v) +raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) { - return arch_atomic_fetch_inc_acquire(v); + return raw_atomic_fetch_inc_acquire(v); } static __always_inline long -arch_atomic_long_fetch_inc_release(atomic_long_t *v) +raw_atomic_long_fetch_inc_release(atomic_long_t *v) { - return arch_atomic_fetch_inc_release(v); + return raw_atomic_fetch_inc_release(v); } static __always_inline long -arch_atomic_long_fetch_inc_relaxed(atomic_long_t *v) +raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) { - return arch_atomic_fetch_inc_relaxed(v); + return raw_atomic_fetch_inc_relaxed(v); } static __always_inline void -arch_atomic_long_dec(atomic_long_t *v) +raw_atomic_long_dec(atomic_long_t *v) { - arch_atomic_dec(v); + raw_atomic_dec(v); } static __always_inline long -arch_atomic_long_dec_return(atomic_long_t *v) +raw_atomic_long_dec_return(atomic_long_t *v) { - return arch_atomic_dec_return(v); + return raw_atomic_dec_return(v); } static __always_inline long -arch_atomic_long_dec_return_acquire(atomic_long_t *v) +raw_atomic_long_dec_return_acquire(atomic_long_t *v) { - return arch_atomic_dec_return_acquire(v); + return raw_atomic_dec_return_acquire(v); } static __always_inline long -arch_atomic_long_dec_return_release(atomic_long_t *v) +raw_atomic_long_dec_return_release(atomic_long_t *v) { - return arch_atomic_dec_return_release(v); + return raw_atomic_dec_return_release(v); } static __always_inline long -arch_atomic_long_dec_return_relaxed(atomic_long_t *v) +raw_atomic_long_dec_return_relaxed(atomic_long_t *v) { - return arch_atomic_dec_return_relaxed(v); + return raw_atomic_dec_return_relaxed(v); } static __always_inline long -arch_atomic_long_fetch_dec(atomic_long_t *v) +raw_atomic_long_fetch_dec(atomic_long_t *v) { - return arch_atomic_fetch_dec(v); + return raw_atomic_fetch_dec(v); } static __always_inline long -arch_atomic_long_fetch_dec_acquire(atomic_long_t *v) +raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) { - return arch_atomic_fetch_dec_acquire(v); + return raw_atomic_fetch_dec_acquire(v); } static __always_inline long -arch_atomic_long_fetch_dec_release(atomic_long_t *v) +raw_atomic_long_fetch_dec_release(atomic_long_t *v) { - return arch_atomic_fetch_dec_release(v); + return raw_atomic_fetch_dec_release(v); } static __always_inline long -arch_atomic_long_fetch_dec_relaxed(atomic_long_t *v) +raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) { - return arch_atomic_fetch_dec_relaxed(v); + return raw_atomic_fetch_dec_relaxed(v); } static __always_inline void -arch_atomic_long_and(long i, atomic_long_t *v) +raw_atomic_long_and(long i, atomic_long_t *v) { - arch_atomic_and(i, v); + raw_atomic_and(i, v); } static __always_inline long -arch_atomic_long_fetch_and(long i, atomic_long_t *v) +raw_atomic_long_fetch_and(long i, atomic_long_t *v) { - return arch_atomic_fetch_and(i, v); + return raw_atomic_fetch_and(i, v); } static __always_inline long -arch_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { - return arch_atomic_fetch_and_acquire(i, v); + return raw_atomic_fetch_and_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_and_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) { - return arch_atomic_fetch_and_release(i, v); + return raw_atomic_fetch_and_release(i, v); } static __always_inline long -arch_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { - return arch_atomic_fetch_and_relaxed(i, v); + return raw_atomic_fetch_and_relaxed(i, v); } static __always_inline void -arch_atomic_long_andnot(long i, atomic_long_t *v) +raw_atomic_long_andnot(long i, atomic_long_t *v) { - arch_atomic_andnot(i, v); + raw_atomic_andnot(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) { - return arch_atomic_fetch_andnot(i, v); + return raw_atomic_fetch_andnot(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { - return arch_atomic_fetch_andnot_acquire(i, v); + return raw_atomic_fetch_andnot_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { - return arch_atomic_fetch_andnot_release(i, v); + return raw_atomic_fetch_andnot_release(i, v); } static __always_inline long -arch_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { - return arch_atomic_fetch_andnot_relaxed(i, v); + return raw_atomic_fetch_andnot_relaxed(i, v); } static __always_inline void -arch_atomic_long_or(long i, atomic_long_t *v) +raw_atomic_long_or(long i, atomic_long_t *v) { - arch_atomic_or(i, v); + raw_atomic_or(i, v); } static __always_inline long -arch_atomic_long_fetch_or(long i, atomic_long_t *v) +raw_atomic_long_fetch_or(long i, atomic_long_t *v) { - return arch_atomic_fetch_or(i, v); + return raw_atomic_fetch_or(i, v); } static __always_inline long -arch_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { - return arch_atomic_fetch_or_acquire(i, v); + return raw_atomic_fetch_or_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_or_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) { - return arch_atomic_fetch_or_release(i, v); + return raw_atomic_fetch_or_release(i, v); } static __always_inline long -arch_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { - return arch_atomic_fetch_or_relaxed(i, v); + return raw_atomic_fetch_or_relaxed(i, v); } static __always_inline void -arch_atomic_long_xor(long i, atomic_long_t *v) +raw_atomic_long_xor(long i, atomic_long_t *v) { - arch_atomic_xor(i, v); + raw_atomic_xor(i, v); } static __always_inline long -arch_atomic_long_fetch_xor(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor(long i, atomic_long_t *v) { - return arch_atomic_fetch_xor(i, v); + return raw_atomic_fetch_xor(i, v); } static __always_inline long -arch_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { - return arch_atomic_fetch_xor_acquire(i, v); + return raw_atomic_fetch_xor_acquire(i, v); } static __always_inline long -arch_atomic_long_fetch_xor_release(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) { - return arch_atomic_fetch_xor_release(i, v); + return raw_atomic_fetch_xor_release(i, v); } static __always_inline long -arch_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) +raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { - return arch_atomic_fetch_xor_relaxed(i, v); + return raw_atomic_fetch_xor_relaxed(i, v); } static __always_inline long -arch_atomic_long_xchg(atomic_long_t *v, long i) +raw_atomic_long_xchg(atomic_long_t *v, long i) { - return arch_atomic_xchg(v, i); + return raw_atomic_xchg(v, i); } static __always_inline long -arch_atomic_long_xchg_acquire(atomic_long_t *v, long i) +raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) { - return arch_atomic_xchg_acquire(v, i); + return raw_atomic_xchg_acquire(v, i); } static __always_inline long -arch_atomic_long_xchg_release(atomic_long_t *v, long i) +raw_atomic_long_xchg_release(atomic_long_t *v, long i) { - return arch_atomic_xchg_release(v, i); + return raw_atomic_xchg_release(v, i); } static __always_inline long -arch_atomic_long_xchg_relaxed(atomic_long_t *v, long i) +raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) { - return arch_atomic_xchg_relaxed(v, i); + return raw_atomic_xchg_relaxed(v, i); } static __always_inline long -arch_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { - return arch_atomic_cmpxchg(v, old, new); + return raw_atomic_cmpxchg(v, old, new); } static __always_inline long -arch_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { - return arch_atomic_cmpxchg_acquire(v, old, new); + return raw_atomic_cmpxchg_acquire(v, old, new); } static __always_inline long -arch_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { - return arch_atomic_cmpxchg_release(v, old, new); + return raw_atomic_cmpxchg_release(v, old, new); } static __always_inline long -arch_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) +raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { - return arch_atomic_cmpxchg_relaxed(v, old, new); + return raw_atomic_cmpxchg_relaxed(v, old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { - return arch_atomic_try_cmpxchg(v, (int *)old, new); + return raw_atomic_try_cmpxchg(v, (int *)old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { - return arch_atomic_try_cmpxchg_acquire(v, (int *)old, new); + return raw_atomic_try_cmpxchg_acquire(v, (int *)old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { - return arch_atomic_try_cmpxchg_release(v, (int *)old, new); + return raw_atomic_try_cmpxchg_release(v, (int *)old, new); } static __always_inline bool -arch_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) +raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { - return arch_atomic_try_cmpxchg_relaxed(v, (int *)old, new); + return raw_atomic_try_cmpxchg_relaxed(v, (int *)old, new); } static __always_inline bool -arch_atomic_long_sub_and_test(long i, atomic_long_t *v) +raw_atomic_long_sub_and_test(long i, atomic_long_t *v) { - return arch_atomic_sub_and_test(i, v); + return raw_atomic_sub_and_test(i, v); } static __always_inline bool -arch_atomic_long_dec_and_test(atomic_long_t *v) +raw_atomic_long_dec_and_test(atomic_long_t *v) { - return arch_atomic_dec_and_test(v); + return raw_atomic_dec_and_test(v); } static __always_inline bool -arch_atomic_long_inc_and_test(atomic_long_t *v) +raw_atomic_long_inc_and_test(atomic_long_t *v) { - return arch_atomic_inc_and_test(v); + return raw_atomic_inc_and_test(v); } static __always_inline bool -arch_atomic_long_add_negative(long i, atomic_long_t *v) +raw_atomic_long_add_negative(long i, atomic_long_t *v) { - return arch_atomic_add_negative(i, v); + return raw_atomic_add_negative(i, v); } static __always_inline bool -arch_atomic_long_add_negative_acquire(long i, atomic_long_t *v) +raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) { - return arch_atomic_add_negative_acquire(i, v); + return raw_atomic_add_negative_acquire(i, v); } static __always_inline bool -arch_atomic_long_add_negative_release(long i, atomic_long_t *v) +raw_atomic_long_add_negative_release(long i, atomic_long_t *v) { - return arch_atomic_add_negative_release(i, v); + return raw_atomic_add_negative_release(i, v); } static __always_inline bool -arch_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) +raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) { - return arch_atomic_add_negative_relaxed(i, v); + return raw_atomic_add_negative_relaxed(i, v); } static __always_inline long -arch_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) +raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { - return arch_atomic_fetch_add_unless(v, a, u); + return raw_atomic_fetch_add_unless(v, a, u); } static __always_inline bool -arch_atomic_long_add_unless(atomic_long_t *v, long a, long u) +raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) { - return arch_atomic_add_unless(v, a, u); + return raw_atomic_add_unless(v, a, u); } static __always_inline bool -arch_atomic_long_inc_not_zero(atomic_long_t *v) +raw_atomic_long_inc_not_zero(atomic_long_t *v) { - return arch_atomic_inc_not_zero(v); + return raw_atomic_inc_not_zero(v); } static __always_inline bool -arch_atomic_long_inc_unless_negative(atomic_long_t *v) +raw_atomic_long_inc_unless_negative(atomic_long_t *v) { - return arch_atomic_inc_unless_negative(v); + return raw_atomic_inc_unless_negative(v); } static __always_inline bool -arch_atomic_long_dec_unless_positive(atomic_long_t *v) +raw_atomic_long_dec_unless_positive(atomic_long_t *v) { - return arch_atomic_dec_unless_positive(v); + return raw_atomic_dec_unless_positive(v); } static __always_inline long -arch_atomic_long_dec_if_positive(atomic_long_t *v) +raw_atomic_long_dec_if_positive(atomic_long_t *v) { - return arch_atomic_dec_if_positive(v); + return raw_atomic_dec_if_positive(v); } #endif /* CONFIG_64BIT */ #endif /* _LINUX_ATOMIC_LONG_H */ -// a194c07d7d2f4b0e178d3c118c919775d5d65f50 +// 108784846d3bbbb201b8dabe621c5dc30b216206 diff --git a/include/linux/atomic/atomic-raw.h b/include/linux/atomic/atomic-raw.h index 83ff0269657e..8b2fc04cf8c5 100644 --- a/include/linux/atomic/atomic-raw.h +++ b/include/linux/atomic/atomic-raw.h @@ -1026,516 +1026,6 @@ raw_atomic64_dec_if_positive(atomic64_t *v) return arch_atomic64_dec_if_positive(v); } -static __always_inline long -raw_atomic_long_read(const atomic_long_t *v) -{ - return arch_atomic_long_read(v); -} - -static __always_inline long -raw_atomic_long_read_acquire(const atomic_long_t *v) -{ - return arch_atomic_long_read_acquire(v); -} - -static __always_inline void -raw_atomic_long_set(atomic_long_t *v, long i) -{ - arch_atomic_long_set(v, i); -} - -static __always_inline void -raw_atomic_long_set_release(atomic_long_t *v, long i) -{ - arch_atomic_long_set_release(v, i); -} - -static __always_inline void -raw_atomic_long_add(long i, atomic_long_t *v) -{ - arch_atomic_long_add(i, v); -} - -static __always_inline long -raw_atomic_long_add_return(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_return(i, v); -} - -static __always_inline long -raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_return_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_add_return_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_return_release(i, v); -} - -static __always_inline long -raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_return_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_add(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_add_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_add_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_add_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_sub(long i, atomic_long_t *v) -{ - arch_atomic_long_sub(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return(long i, atomic_long_t *v) -{ - return arch_atomic_long_sub_return(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_sub_return_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_sub_return_release(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_sub_return_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_sub(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_sub_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_sub_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_sub_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_inc(atomic_long_t *v) -{ - arch_atomic_long_inc(v); -} - -static __always_inline long -raw_atomic_long_inc_return(atomic_long_t *v) -{ - return arch_atomic_long_inc_return(v); -} - -static __always_inline long -raw_atomic_long_inc_return_acquire(atomic_long_t *v) -{ - return arch_atomic_long_inc_return_acquire(v); -} - -static __always_inline long -raw_atomic_long_inc_return_release(atomic_long_t *v) -{ - return arch_atomic_long_inc_return_release(v); -} - -static __always_inline long -raw_atomic_long_inc_return_relaxed(atomic_long_t *v) -{ - return arch_atomic_long_inc_return_relaxed(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc(atomic_long_t *v) -{ - return arch_atomic_long_fetch_inc(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) -{ - return arch_atomic_long_fetch_inc_acquire(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc_release(atomic_long_t *v) -{ - return arch_atomic_long_fetch_inc_release(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) -{ - return arch_atomic_long_fetch_inc_relaxed(v); -} - -static __always_inline void -raw_atomic_long_dec(atomic_long_t *v) -{ - arch_atomic_long_dec(v); -} - -static __always_inline long -raw_atomic_long_dec_return(atomic_long_t *v) -{ - return arch_atomic_long_dec_return(v); -} - -static __always_inline long -raw_atomic_long_dec_return_acquire(atomic_long_t *v) -{ - return arch_atomic_long_dec_return_acquire(v); -} - -static __always_inline long -raw_atomic_long_dec_return_release(atomic_long_t *v) -{ - return arch_atomic_long_dec_return_release(v); -} - -static __always_inline long -raw_atomic_long_dec_return_relaxed(atomic_long_t *v) -{ - return arch_atomic_long_dec_return_relaxed(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec(atomic_long_t *v) -{ - return arch_atomic_long_fetch_dec(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) -{ - return arch_atomic_long_fetch_dec_acquire(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec_release(atomic_long_t *v) -{ - return arch_atomic_long_fetch_dec_release(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) -{ - return arch_atomic_long_fetch_dec_relaxed(v); -} - -static __always_inline void -raw_atomic_long_and(long i, atomic_long_t *v) -{ - arch_atomic_long_and(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_and(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_and_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_and_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_and_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_andnot(long i, atomic_long_t *v) -{ - arch_atomic_long_andnot(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_andnot(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_andnot_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_andnot_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_andnot_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_or(long i, atomic_long_t *v) -{ - arch_atomic_long_or(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_or(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_or_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_or_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_or_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_xor(long i, atomic_long_t *v) -{ - arch_atomic_long_xor(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_xor(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_xor_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_xor_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_fetch_xor_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_xchg(atomic_long_t *v, long i) -{ - return arch_atomic_long_xchg(v, i); -} - -static __always_inline long -raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) -{ - return arch_atomic_long_xchg_acquire(v, i); -} - -static __always_inline long -raw_atomic_long_xchg_release(atomic_long_t *v, long i) -{ - return arch_atomic_long_xchg_release(v, i); -} - -static __always_inline long -raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) -{ - return arch_atomic_long_xchg_relaxed(v, i); -} - -static __always_inline long -raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) -{ - return arch_atomic_long_cmpxchg(v, old, new); -} - -static __always_inline long -raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) -{ - return arch_atomic_long_cmpxchg_acquire(v, old, new); -} - -static __always_inline long -raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) -{ - return arch_atomic_long_cmpxchg_release(v, old, new); -} - -static __always_inline long -raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) -{ - return arch_atomic_long_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) -{ - return arch_atomic_long_try_cmpxchg(v, old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) -{ - return arch_atomic_long_try_cmpxchg_acquire(v, old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) -{ - return arch_atomic_long_try_cmpxchg_release(v, old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) -{ - return arch_atomic_long_try_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic_long_sub_and_test(long i, atomic_long_t *v) -{ - return arch_atomic_long_sub_and_test(i, v); -} - -static __always_inline bool -raw_atomic_long_dec_and_test(atomic_long_t *v) -{ - return arch_atomic_long_dec_and_test(v); -} - -static __always_inline bool -raw_atomic_long_inc_and_test(atomic_long_t *v) -{ - return arch_atomic_long_inc_and_test(v); -} - -static __always_inline bool -raw_atomic_long_add_negative(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_negative(i, v); -} - -static __always_inline bool -raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_negative_acquire(i, v); -} - -static __always_inline bool -raw_atomic_long_add_negative_release(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_negative_release(i, v); -} - -static __always_inline bool -raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) -{ - return arch_atomic_long_add_negative_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) -{ - return arch_atomic_long_fetch_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) -{ - return arch_atomic_long_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic_long_inc_not_zero(atomic_long_t *v) -{ - return arch_atomic_long_inc_not_zero(v); -} - -static __always_inline bool -raw_atomic_long_inc_unless_negative(atomic_long_t *v) -{ - return arch_atomic_long_inc_unless_negative(v); -} - -static __always_inline bool -raw_atomic_long_dec_unless_positive(atomic_long_t *v) -{ - return arch_atomic_long_dec_unless_positive(v); -} - -static __always_inline long -raw_atomic_long_dec_if_positive(atomic_long_t *v) -{ - return arch_atomic_long_dec_if_positive(v); -} - #define raw_xchg(...) \ arch_xchg(__VA_ARGS__) @@ -1642,4 +1132,4 @@ raw_atomic_long_dec_if_positive(atomic_long_t *v) arch_try_cmpxchg128_local(__VA_ARGS__) #endif /* _LINUX_ATOMIC_RAW_H */ -// 01d54200571b3857755a07c10074a4fd58cef6b1 +// b23ed4424e85200e200ded094522e1d743b3a5b1 -- cgit v1.2.3 From 9257959a6e5b4fca6fc8e985790bff62c2046f20 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:17 +0100 Subject: locking/atomic: scripts: restructure fallback ifdeffery Currently the various ordering variants of an atomic operation are defined in groups of full/acquire/release/relaxed ordering variants with some shared ifdeffery and several potential definitions of each ordering variant in different branches of the shared ifdeffery. As an ordering variant can have several potential definitions down different branches of the shared ifdeffery, it can be painful for a human to find a relevant definition, and we don't have a good location to place anything common to all definitions of an ordering variant (e.g. kerneldoc). Historically the grouping of full/acquire/release/relaxed ordering variants was necessary as we filled in the missing atomics in the same namespace as the architecture used. It would be easy to accidentally define one ordering fallback in terms of another ordering fallback with redundant barriers, and avoiding that would otherwise require a lot of baroque ifdeffery. With recent changes we no longer need to fill in the missing atomics in the arch_atomic*_() namespace, and only need to fill in the raw_atomic*_() namespace. Due to this, there's no risk of a namespace collision, and we can define each raw_atomic*_ ordering variant with its own ifdeffery checking for the arch_atomic*_ ordering variants. Restructure the fallbacks in this way, with each ordering variant having its own ifdeffery of the form: | #if defined(arch_atomic_fetch_andnot_acquire) | #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire | #elif defined(arch_atomic_fetch_andnot_relaxed) | static __always_inline int | raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) | { | int ret = arch_atomic_fetch_andnot_relaxed(i, v); | __atomic_acquire_fence(); | return ret; | } | #elif defined(arch_atomic_fetch_andnot) | #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot | #else | static __always_inline int | raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) | { | return raw_atomic_fetch_and_acquire(~i, v); | } | #endif Note that where there's no relevant arch_atomic*_() ordering variant, we'll define the operation in terms of a distinct raw_atomic*_(), as this itself might have been filled in with a fallback. As we now generate the raw_atomic*_() implementations directly, we no longer need the trivial wrappers, so they are removed. This makes the ifdeffery easier to follow, and will allow for further improvements in subsequent patches. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-21-mark.rutland@arm.com --- include/linux/atomic.h | 1 - include/linux/atomic/atomic-arch-fallback.h | 3178 ++++++++++++++------------- include/linux/atomic/atomic-raw.h | 1135 ---------- 3 files changed, 1670 insertions(+), 2644 deletions(-) delete mode 100644 include/linux/atomic/atomic-raw.h (limited to 'include/linux') diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 296cfae0389f..8dd57c3a99e9 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -78,7 +78,6 @@ }) #include -#include #include #include diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 1a2d81dbc2e4..99bc1a871dc1 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -8,2749 +8,2911 @@ #include -#ifndef arch_xchg_relaxed -#define arch_xchg_acquire arch_xchg -#define arch_xchg_release arch_xchg -#define arch_xchg_relaxed arch_xchg -#else /* arch_xchg_relaxed */ - -#ifndef arch_xchg_acquire -#define arch_xchg_acquire(...) \ - __atomic_op_acquire(arch_xchg, __VA_ARGS__) +#if defined(arch_xchg) +#define raw_xchg arch_xchg +#elif defined(arch_xchg_relaxed) +#define raw_xchg(...) \ + __atomic_op_fence(arch_xchg, __VA_ARGS__) +#else +extern void raw_xchg_not_implemented(void); +#define raw_xchg(...) raw_xchg_not_implemented() #endif -#ifndef arch_xchg_release -#define arch_xchg_release(...) \ - __atomic_op_release(arch_xchg, __VA_ARGS__) +#if defined(arch_xchg_acquire) +#define raw_xchg_acquire arch_xchg_acquire +#elif defined(arch_xchg_relaxed) +#define raw_xchg_acquire(...) \ + __atomic_op_acquire(arch_xchg, __VA_ARGS__) +#elif defined(arch_xchg) +#define raw_xchg_acquire arch_xchg +#else +extern void raw_xchg_acquire_not_implemented(void); +#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented() #endif -#ifndef arch_xchg -#define arch_xchg(...) \ - __atomic_op_fence(arch_xchg, __VA_ARGS__) +#if defined(arch_xchg_release) +#define raw_xchg_release arch_xchg_release +#elif defined(arch_xchg_relaxed) +#define raw_xchg_release(...) \ + __atomic_op_release(arch_xchg, __VA_ARGS__) +#elif defined(arch_xchg) +#define raw_xchg_release arch_xchg +#else +extern void raw_xchg_release_not_implemented(void); +#define raw_xchg_release(...) raw_xchg_release_not_implemented() +#endif + +#if defined(arch_xchg_relaxed) +#define raw_xchg_relaxed arch_xchg_relaxed +#elif defined(arch_xchg) +#define raw_xchg_relaxed arch_xchg +#else +extern void raw_xchg_relaxed_not_implemented(void); +#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented() +#endif + +#if defined(arch_cmpxchg) +#define raw_cmpxchg arch_cmpxchg +#elif defined(arch_cmpxchg_relaxed) +#define raw_cmpxchg(...) \ + __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) +#else +extern void raw_cmpxchg_not_implemented(void); +#define raw_cmpxchg(...) raw_cmpxchg_not_implemented() #endif -#endif /* arch_xchg_relaxed */ - -#ifndef arch_cmpxchg_relaxed -#define arch_cmpxchg_acquire arch_cmpxchg -#define arch_cmpxchg_release arch_cmpxchg -#define arch_cmpxchg_relaxed arch_cmpxchg -#else /* arch_cmpxchg_relaxed */ - -#ifndef arch_cmpxchg_acquire -#define arch_cmpxchg_acquire(...) \ +#if defined(arch_cmpxchg_acquire) +#define raw_cmpxchg_acquire arch_cmpxchg_acquire +#elif defined(arch_cmpxchg_relaxed) +#define raw_cmpxchg_acquire(...) \ __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) +#elif defined(arch_cmpxchg) +#define raw_cmpxchg_acquire arch_cmpxchg +#else +extern void raw_cmpxchg_acquire_not_implemented(void); +#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented() #endif -#ifndef arch_cmpxchg_release -#define arch_cmpxchg_release(...) \ +#if defined(arch_cmpxchg_release) +#define raw_cmpxchg_release arch_cmpxchg_release +#elif defined(arch_cmpxchg_relaxed) +#define raw_cmpxchg_release(...) \ __atomic_op_release(arch_cmpxchg, __VA_ARGS__) +#elif defined(arch_cmpxchg) +#define raw_cmpxchg_release arch_cmpxchg +#else +extern void raw_cmpxchg_release_not_implemented(void); +#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented() +#endif + +#if defined(arch_cmpxchg_relaxed) +#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed +#elif defined(arch_cmpxchg) +#define raw_cmpxchg_relaxed arch_cmpxchg +#else +extern void raw_cmpxchg_relaxed_not_implemented(void); +#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented() +#endif + +#if defined(arch_cmpxchg64) +#define raw_cmpxchg64 arch_cmpxchg64 +#elif defined(arch_cmpxchg64_relaxed) +#define raw_cmpxchg64(...) \ + __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) +#else +extern void raw_cmpxchg64_not_implemented(void); +#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented() #endif -#ifndef arch_cmpxchg -#define arch_cmpxchg(...) \ - __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) -#endif - -#endif /* arch_cmpxchg_relaxed */ - -#ifndef arch_cmpxchg64_relaxed -#define arch_cmpxchg64_acquire arch_cmpxchg64 -#define arch_cmpxchg64_release arch_cmpxchg64 -#define arch_cmpxchg64_relaxed arch_cmpxchg64 -#else /* arch_cmpxchg64_relaxed */ - -#ifndef arch_cmpxchg64_acquire -#define arch_cmpxchg64_acquire(...) \ +#if defined(arch_cmpxchg64_acquire) +#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire +#elif defined(arch_cmpxchg64_relaxed) +#define raw_cmpxchg64_acquire(...) \ __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) +#elif defined(arch_cmpxchg64) +#define raw_cmpxchg64_acquire arch_cmpxchg64 +#else +extern void raw_cmpxchg64_acquire_not_implemented(void); +#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented() #endif -#ifndef arch_cmpxchg64_release -#define arch_cmpxchg64_release(...) \ +#if defined(arch_cmpxchg64_release) +#define raw_cmpxchg64_release arch_cmpxchg64_release +#elif defined(arch_cmpxchg64_relaxed) +#define raw_cmpxchg64_release(...) \ __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) +#elif defined(arch_cmpxchg64) +#define raw_cmpxchg64_release arch_cmpxchg64 +#else +extern void raw_cmpxchg64_release_not_implemented(void); +#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented() +#endif + +#if defined(arch_cmpxchg64_relaxed) +#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed +#elif defined(arch_cmpxchg64) +#define raw_cmpxchg64_relaxed arch_cmpxchg64 +#else +extern void raw_cmpxchg64_relaxed_not_implemented(void); +#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented() +#endif + +#if defined(arch_cmpxchg128) +#define raw_cmpxchg128 arch_cmpxchg128 +#elif defined(arch_cmpxchg128_relaxed) +#define raw_cmpxchg128(...) \ + __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) +#else +extern void raw_cmpxchg128_not_implemented(void); +#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented() #endif -#ifndef arch_cmpxchg64 -#define arch_cmpxchg64(...) \ - __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) -#endif - -#endif /* arch_cmpxchg64_relaxed */ - -#ifndef arch_cmpxchg128_relaxed -#define arch_cmpxchg128_acquire arch_cmpxchg128 -#define arch_cmpxchg128_release arch_cmpxchg128 -#define arch_cmpxchg128_relaxed arch_cmpxchg128 -#else /* arch_cmpxchg128_relaxed */ - -#ifndef arch_cmpxchg128_acquire -#define arch_cmpxchg128_acquire(...) \ +#if defined(arch_cmpxchg128_acquire) +#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire +#elif defined(arch_cmpxchg128_relaxed) +#define raw_cmpxchg128_acquire(...) \ __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) +#elif defined(arch_cmpxchg128) +#define raw_cmpxchg128_acquire arch_cmpxchg128 +#else +extern void raw_cmpxchg128_acquire_not_implemented(void); +#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented() #endif -#ifndef arch_cmpxchg128_release -#define arch_cmpxchg128_release(...) \ +#if defined(arch_cmpxchg128_release) +#define raw_cmpxchg128_release arch_cmpxchg128_release +#elif defined(arch_cmpxchg128_relaxed) +#define raw_cmpxchg128_release(...) \ __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) -#endif - -#ifndef arch_cmpxchg128 -#define arch_cmpxchg128(...) \ - __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) -#endif - -#endif /* arch_cmpxchg128_relaxed */ - -#ifndef arch_try_cmpxchg_relaxed -#ifdef arch_try_cmpxchg -#define arch_try_cmpxchg_acquire arch_try_cmpxchg -#define arch_try_cmpxchg_release arch_try_cmpxchg -#define arch_try_cmpxchg_relaxed arch_try_cmpxchg -#endif /* arch_try_cmpxchg */ - -#ifndef arch_try_cmpxchg -#define arch_try_cmpxchg(_ptr, _oldp, _new) \ +#elif defined(arch_cmpxchg128) +#define raw_cmpxchg128_release arch_cmpxchg128 +#else +extern void raw_cmpxchg128_release_not_implemented(void); +#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented() +#endif + +#if defined(arch_cmpxchg128_relaxed) +#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed +#elif defined(arch_cmpxchg128) +#define raw_cmpxchg128_relaxed arch_cmpxchg128 +#else +extern void raw_cmpxchg128_relaxed_not_implemented(void); +#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented() +#endif + +#if defined(arch_try_cmpxchg) +#define raw_try_cmpxchg arch_try_cmpxchg +#elif defined(arch_try_cmpxchg_relaxed) +#define raw_try_cmpxchg(...) \ + __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) +#else +#define raw_try_cmpxchg(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg */ +#endif -#ifndef arch_try_cmpxchg_acquire -#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg_acquire) +#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire +#elif defined(arch_try_cmpxchg_relaxed) +#define raw_try_cmpxchg_acquire(...) \ + __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) +#elif defined(arch_try_cmpxchg) +#define raw_try_cmpxchg_acquire arch_try_cmpxchg +#else +#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg_acquire */ +#endif -#ifndef arch_try_cmpxchg_release -#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg_release) +#define raw_try_cmpxchg_release arch_try_cmpxchg_release +#elif defined(arch_try_cmpxchg_relaxed) +#define raw_try_cmpxchg_release(...) \ + __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) +#elif defined(arch_try_cmpxchg) +#define raw_try_cmpxchg_release arch_try_cmpxchg +#else +#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg_release */ +#endif -#ifndef arch_try_cmpxchg_relaxed -#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg_relaxed) +#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed +#elif defined(arch_try_cmpxchg) +#define raw_try_cmpxchg_relaxed arch_try_cmpxchg +#else +#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg_relaxed */ - -#else /* arch_try_cmpxchg_relaxed */ - -#ifndef arch_try_cmpxchg_acquire -#define arch_try_cmpxchg_acquire(...) \ - __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) -#endif - -#ifndef arch_try_cmpxchg_release -#define arch_try_cmpxchg_release(...) \ - __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) #endif -#ifndef arch_try_cmpxchg -#define arch_try_cmpxchg(...) \ - __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) -#endif - -#endif /* arch_try_cmpxchg_relaxed */ - -#ifndef arch_try_cmpxchg64_relaxed -#ifdef arch_try_cmpxchg64 -#define arch_try_cmpxchg64_acquire arch_try_cmpxchg64 -#define arch_try_cmpxchg64_release arch_try_cmpxchg64 -#define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64 -#endif /* arch_try_cmpxchg64 */ - -#ifndef arch_try_cmpxchg64 -#define arch_try_cmpxchg64(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg64) +#define raw_try_cmpxchg64 arch_try_cmpxchg64 +#elif defined(arch_try_cmpxchg64_relaxed) +#define raw_try_cmpxchg64(...) \ + __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) +#else +#define raw_try_cmpxchg64(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg64 */ +#endif -#ifndef arch_try_cmpxchg64_acquire -#define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg64_acquire) +#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire +#elif defined(arch_try_cmpxchg64_relaxed) +#define raw_try_cmpxchg64_acquire(...) \ + __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) +#elif defined(arch_try_cmpxchg64) +#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64 +#else +#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg64_acquire */ +#endif -#ifndef arch_try_cmpxchg64_release -#define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg64_release) +#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release +#elif defined(arch_try_cmpxchg64_relaxed) +#define raw_try_cmpxchg64_release(...) \ + __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) +#elif defined(arch_try_cmpxchg64) +#define raw_try_cmpxchg64_release arch_try_cmpxchg64 +#else +#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg64_release */ +#endif -#ifndef arch_try_cmpxchg64_relaxed -#define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg64_relaxed) +#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed +#elif defined(arch_try_cmpxchg64) +#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64 +#else +#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg64_relaxed */ - -#else /* arch_try_cmpxchg64_relaxed */ - -#ifndef arch_try_cmpxchg64_acquire -#define arch_try_cmpxchg64_acquire(...) \ - __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) -#endif - -#ifndef arch_try_cmpxchg64_release -#define arch_try_cmpxchg64_release(...) \ - __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) -#endif - -#ifndef arch_try_cmpxchg64 -#define arch_try_cmpxchg64(...) \ - __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) #endif -#endif /* arch_try_cmpxchg64_relaxed */ - -#ifndef arch_try_cmpxchg128_relaxed -#ifdef arch_try_cmpxchg128 -#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128 -#define arch_try_cmpxchg128_release arch_try_cmpxchg128 -#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128 -#endif /* arch_try_cmpxchg128 */ - -#ifndef arch_try_cmpxchg128 -#define arch_try_cmpxchg128(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg128) +#define raw_try_cmpxchg128 arch_try_cmpxchg128 +#elif defined(arch_try_cmpxchg128_relaxed) +#define raw_try_cmpxchg128(...) \ + __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) +#else +#define raw_try_cmpxchg128(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg128 */ +#endif -#ifndef arch_try_cmpxchg128_acquire -#define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg128_acquire) +#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire +#elif defined(arch_try_cmpxchg128_relaxed) +#define raw_try_cmpxchg128_acquire(...) \ + __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) +#elif defined(arch_try_cmpxchg128) +#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128 +#else +#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg128_acquire */ +#endif -#ifndef arch_try_cmpxchg128_release -#define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg128_release) +#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release +#elif defined(arch_try_cmpxchg128_relaxed) +#define raw_try_cmpxchg128_release(...) \ + __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) +#elif defined(arch_try_cmpxchg128) +#define raw_try_cmpxchg128_release arch_try_cmpxchg128 +#else +#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg128_release */ +#endif -#ifndef arch_try_cmpxchg128_relaxed -#define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ +#if defined(arch_try_cmpxchg128_relaxed) +#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed +#elif defined(arch_try_cmpxchg128) +#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128 +#else +#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg128_relaxed */ - -#else /* arch_try_cmpxchg128_relaxed */ - -#ifndef arch_try_cmpxchg128_acquire -#define arch_try_cmpxchg128_acquire(...) \ - __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) #endif -#ifndef arch_try_cmpxchg128_release -#define arch_try_cmpxchg128_release(...) \ - __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) -#endif +#define raw_cmpxchg_local arch_cmpxchg_local -#ifndef arch_try_cmpxchg128 -#define arch_try_cmpxchg128(...) \ - __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) +#ifdef arch_try_cmpxchg_local +#define raw_try_cmpxchg_local arch_try_cmpxchg_local +#else +#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \ +({ \ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ + ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \ + if (unlikely(___r != ___o)) \ + *___op = ___r; \ + likely(___r == ___o); \ +}) #endif -#endif /* arch_try_cmpxchg128_relaxed */ +#define raw_cmpxchg64_local arch_cmpxchg64_local -#ifndef arch_try_cmpxchg_local -#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \ +#ifdef arch_try_cmpxchg64_local +#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local +#else +#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg_local */ +#endif + +#define raw_cmpxchg128_local arch_cmpxchg128_local -#ifndef arch_try_cmpxchg64_local -#define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \ +#ifdef arch_try_cmpxchg128_local +#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local +#else +#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \ ({ \ typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ - ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \ + ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \ if (unlikely(___r != ___o)) \ *___op = ___r; \ likely(___r == ___o); \ }) -#endif /* arch_try_cmpxchg64_local */ +#endif + +#define raw_sync_cmpxchg arch_sync_cmpxchg -#ifndef arch_atomic_read_acquire +#define raw_atomic_read arch_atomic_read + +#if defined(arch_atomic_read_acquire) +#define raw_atomic_read_acquire arch_atomic_read_acquire +#elif defined(arch_atomic_read) +#define raw_atomic_read_acquire arch_atomic_read +#else static __always_inline int -arch_atomic_read_acquire(const atomic_t *v) +raw_atomic_read_acquire(const atomic_t *v) { int ret; if (__native_word(atomic_t)) { ret = smp_load_acquire(&(v)->counter); } else { - ret = arch_atomic_read(v); + ret = raw_atomic_read(v); __atomic_acquire_fence(); } return ret; } -#define arch_atomic_read_acquire arch_atomic_read_acquire #endif -#ifndef arch_atomic_set_release +#define raw_atomic_set arch_atomic_set + +#if defined(arch_atomic_set_release) +#define raw_atomic_set_release arch_atomic_set_release +#elif defined(arch_atomic_set) +#define raw_atomic_set_release arch_atomic_set +#else static __always_inline void -arch_atomic_set_release(atomic_t *v, int i) +raw_atomic_set_release(atomic_t *v, int i) { if (__native_word(atomic_t)) { smp_store_release(&(v)->counter, i); } else { __atomic_release_fence(); - arch_atomic_set(v, i); + raw_atomic_set(v, i); } } -#define arch_atomic_set_release arch_atomic_set_release #endif -#ifndef arch_atomic_add_return_relaxed -#define arch_atomic_add_return_acquire arch_atomic_add_return -#define arch_atomic_add_return_release arch_atomic_add_return -#define arch_atomic_add_return_relaxed arch_atomic_add_return -#else /* arch_atomic_add_return_relaxed */ +#define raw_atomic_add arch_atomic_add + +#if defined(arch_atomic_add_return) +#define raw_atomic_add_return arch_atomic_add_return +#elif defined(arch_atomic_add_return_relaxed) +static __always_inline int +raw_atomic_add_return(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_add_return_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#else +#error "Unable to define raw_atomic_add_return" +#endif -#ifndef arch_atomic_add_return_acquire +#if defined(arch_atomic_add_return_acquire) +#define raw_atomic_add_return_acquire arch_atomic_add_return_acquire +#elif defined(arch_atomic_add_return_relaxed) static __always_inline int -arch_atomic_add_return_acquire(int i, atomic_t *v) +raw_atomic_add_return_acquire(int i, atomic_t *v) { int ret = arch_atomic_add_return_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire +#elif defined(arch_atomic_add_return) +#define raw_atomic_add_return_acquire arch_atomic_add_return +#else +#error "Unable to define raw_atomic_add_return_acquire" #endif -#ifndef arch_atomic_add_return_release +#if defined(arch_atomic_add_return_release) +#define raw_atomic_add_return_release arch_atomic_add_return_release +#elif defined(arch_atomic_add_return_relaxed) static __always_inline int -arch_atomic_add_return_release(int i, atomic_t *v) +raw_atomic_add_return_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_add_return_relaxed(i, v); } -#define arch_atomic_add_return_release arch_atomic_add_return_release +#elif defined(arch_atomic_add_return) +#define raw_atomic_add_return_release arch_atomic_add_return +#else +#error "Unable to define raw_atomic_add_return_release" #endif -#ifndef arch_atomic_add_return +#if defined(arch_atomic_add_return_relaxed) +#define raw_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#elif defined(arch_atomic_add_return) +#define raw_atomic_add_return_relaxed arch_atomic_add_return +#else +#error "Unable to define raw_atomic_add_return_relaxed" +#endif + +#if defined(arch_atomic_fetch_add) +#define raw_atomic_fetch_add arch_atomic_fetch_add +#elif defined(arch_atomic_fetch_add_relaxed) static __always_inline int -arch_atomic_add_return(int i, atomic_t *v) +raw_atomic_fetch_add(int i, atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_add_return_relaxed(i, v); + ret = arch_atomic_fetch_add_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_add_return arch_atomic_add_return +#else +#error "Unable to define raw_atomic_fetch_add" #endif -#endif /* arch_atomic_add_return_relaxed */ - -#ifndef arch_atomic_fetch_add_relaxed -#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add -#define arch_atomic_fetch_add_release arch_atomic_fetch_add -#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add -#else /* arch_atomic_fetch_add_relaxed */ - -#ifndef arch_atomic_fetch_add_acquire +#if defined(arch_atomic_fetch_add_acquire) +#define raw_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire +#elif defined(arch_atomic_fetch_add_relaxed) static __always_inline int -arch_atomic_fetch_add_acquire(int i, atomic_t *v) +raw_atomic_fetch_add_acquire(int i, atomic_t *v) { int ret = arch_atomic_fetch_add_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire +#elif defined(arch_atomic_fetch_add) +#define raw_atomic_fetch_add_acquire arch_atomic_fetch_add +#else +#error "Unable to define raw_atomic_fetch_add_acquire" #endif -#ifndef arch_atomic_fetch_add_release +#if defined(arch_atomic_fetch_add_release) +#define raw_atomic_fetch_add_release arch_atomic_fetch_add_release +#elif defined(arch_atomic_fetch_add_relaxed) static __always_inline int -arch_atomic_fetch_add_release(int i, atomic_t *v) +raw_atomic_fetch_add_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_fetch_add_relaxed(i, v); } -#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release +#elif defined(arch_atomic_fetch_add) +#define raw_atomic_fetch_add_release arch_atomic_fetch_add +#else +#error "Unable to define raw_atomic_fetch_add_release" +#endif + +#if defined(arch_atomic_fetch_add_relaxed) +#define raw_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#elif defined(arch_atomic_fetch_add) +#define raw_atomic_fetch_add_relaxed arch_atomic_fetch_add +#else +#error "Unable to define raw_atomic_fetch_add_relaxed" #endif -#ifndef arch_atomic_fetch_add +#define raw_atomic_sub arch_atomic_sub + +#if defined(arch_atomic_sub_return) +#define raw_atomic_sub_return arch_atomic_sub_return +#elif defined(arch_atomic_sub_return_relaxed) static __always_inline int -arch_atomic_fetch_add(int i, atomic_t *v) +raw_atomic_sub_return(int i, atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_fetch_add_relaxed(i, v); + ret = arch_atomic_sub_return_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_add arch_atomic_fetch_add +#else +#error "Unable to define raw_atomic_sub_return" #endif -#endif /* arch_atomic_fetch_add_relaxed */ - -#ifndef arch_atomic_sub_return_relaxed -#define arch_atomic_sub_return_acquire arch_atomic_sub_return -#define arch_atomic_sub_return_release arch_atomic_sub_return -#define arch_atomic_sub_return_relaxed arch_atomic_sub_return -#else /* arch_atomic_sub_return_relaxed */ - -#ifndef arch_atomic_sub_return_acquire +#if defined(arch_atomic_sub_return_acquire) +#define raw_atomic_sub_return_acquire arch_atomic_sub_return_acquire +#elif defined(arch_atomic_sub_return_relaxed) static __always_inline int -arch_atomic_sub_return_acquire(int i, atomic_t *v) +raw_atomic_sub_return_acquire(int i, atomic_t *v) { int ret = arch_atomic_sub_return_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire +#elif defined(arch_atomic_sub_return) +#define raw_atomic_sub_return_acquire arch_atomic_sub_return +#else +#error "Unable to define raw_atomic_sub_return_acquire" #endif -#ifndef arch_atomic_sub_return_release +#if defined(arch_atomic_sub_return_release) +#define raw_atomic_sub_return_release arch_atomic_sub_return_release +#elif defined(arch_atomic_sub_return_relaxed) static __always_inline int -arch_atomic_sub_return_release(int i, atomic_t *v) +raw_atomic_sub_return_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_sub_return_relaxed(i, v); } -#define arch_atomic_sub_return_release arch_atomic_sub_return_release +#elif defined(arch_atomic_sub_return) +#define raw_atomic_sub_return_release arch_atomic_sub_return +#else +#error "Unable to define raw_atomic_sub_return_release" +#endif + +#if defined(arch_atomic_sub_return_relaxed) +#define raw_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#elif defined(arch_atomic_sub_return) +#define raw_atomic_sub_return_relaxed arch_atomic_sub_return +#else +#error "Unable to define raw_atomic_sub_return_relaxed" #endif -#ifndef arch_atomic_sub_return +#if defined(arch_atomic_fetch_sub) +#define raw_atomic_fetch_sub arch_atomic_fetch_sub +#elif defined(arch_atomic_fetch_sub_relaxed) static __always_inline int -arch_atomic_sub_return(int i, atomic_t *v) +raw_atomic_fetch_sub(int i, atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_sub_return_relaxed(i, v); + ret = arch_atomic_fetch_sub_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_sub_return arch_atomic_sub_return +#else +#error "Unable to define raw_atomic_fetch_sub" #endif -#endif /* arch_atomic_sub_return_relaxed */ - -#ifndef arch_atomic_fetch_sub_relaxed -#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub -#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub -#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub -#else /* arch_atomic_fetch_sub_relaxed */ - -#ifndef arch_atomic_fetch_sub_acquire +#if defined(arch_atomic_fetch_sub_acquire) +#define raw_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire +#elif defined(arch_atomic_fetch_sub_relaxed) static __always_inline int -arch_atomic_fetch_sub_acquire(int i, atomic_t *v) +raw_atomic_fetch_sub_acquire(int i, atomic_t *v) { int ret = arch_atomic_fetch_sub_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire +#elif defined(arch_atomic_fetch_sub) +#define raw_atomic_fetch_sub_acquire arch_atomic_fetch_sub +#else +#error "Unable to define raw_atomic_fetch_sub_acquire" #endif -#ifndef arch_atomic_fetch_sub_release +#if defined(arch_atomic_fetch_sub_release) +#define raw_atomic_fetch_sub_release arch_atomic_fetch_sub_release +#elif defined(arch_atomic_fetch_sub_relaxed) static __always_inline int -arch_atomic_fetch_sub_release(int i, atomic_t *v) +raw_atomic_fetch_sub_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_fetch_sub_relaxed(i, v); } -#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release +#elif defined(arch_atomic_fetch_sub) +#define raw_atomic_fetch_sub_release arch_atomic_fetch_sub +#else +#error "Unable to define raw_atomic_fetch_sub_release" #endif -#ifndef arch_atomic_fetch_sub -static __always_inline int -arch_atomic_fetch_sub(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = arch_atomic_fetch_sub_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define arch_atomic_fetch_sub arch_atomic_fetch_sub +#if defined(arch_atomic_fetch_sub_relaxed) +#define raw_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed +#elif defined(arch_atomic_fetch_sub) +#define raw_atomic_fetch_sub_relaxed arch_atomic_fetch_sub +#else +#error "Unable to define raw_atomic_fetch_sub_relaxed" #endif -#endif /* arch_atomic_fetch_sub_relaxed */ - -#ifndef arch_atomic_inc +#if defined(arch_atomic_inc) +#define raw_atomic_inc arch_atomic_inc +#else static __always_inline void -arch_atomic_inc(atomic_t *v) +raw_atomic_inc(atomic_t *v) { - arch_atomic_add(1, v); + raw_atomic_add(1, v); } -#define arch_atomic_inc arch_atomic_inc #endif -#ifndef arch_atomic_inc_return_relaxed -#ifdef arch_atomic_inc_return -#define arch_atomic_inc_return_acquire arch_atomic_inc_return -#define arch_atomic_inc_return_release arch_atomic_inc_return -#define arch_atomic_inc_return_relaxed arch_atomic_inc_return -#endif /* arch_atomic_inc_return */ - -#ifndef arch_atomic_inc_return +#if defined(arch_atomic_inc_return) +#define raw_atomic_inc_return arch_atomic_inc_return +#elif defined(arch_atomic_inc_return_relaxed) +static __always_inline int +raw_atomic_inc_return(atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_inc_return_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#else static __always_inline int -arch_atomic_inc_return(atomic_t *v) +raw_atomic_inc_return(atomic_t *v) { - return arch_atomic_add_return(1, v); + return raw_atomic_add_return(1, v); } -#define arch_atomic_inc_return arch_atomic_inc_return #endif -#ifndef arch_atomic_inc_return_acquire +#if defined(arch_atomic_inc_return_acquire) +#define raw_atomic_inc_return_acquire arch_atomic_inc_return_acquire +#elif defined(arch_atomic_inc_return_relaxed) static __always_inline int -arch_atomic_inc_return_acquire(atomic_t *v) +raw_atomic_inc_return_acquire(atomic_t *v) { - return arch_atomic_add_return_acquire(1, v); + int ret = arch_atomic_inc_return_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire -#endif - -#ifndef arch_atomic_inc_return_release +#elif defined(arch_atomic_inc_return) +#define raw_atomic_inc_return_acquire arch_atomic_inc_return +#else static __always_inline int -arch_atomic_inc_return_release(atomic_t *v) +raw_atomic_inc_return_acquire(atomic_t *v) { - return arch_atomic_add_return_release(1, v); + return raw_atomic_add_return_acquire(1, v); } -#define arch_atomic_inc_return_release arch_atomic_inc_return_release #endif -#ifndef arch_atomic_inc_return_relaxed +#if defined(arch_atomic_inc_return_release) +#define raw_atomic_inc_return_release arch_atomic_inc_return_release +#elif defined(arch_atomic_inc_return_relaxed) static __always_inline int -arch_atomic_inc_return_relaxed(atomic_t *v) +raw_atomic_inc_return_release(atomic_t *v) { - return arch_atomic_add_return_relaxed(1, v); + __atomic_release_fence(); + return arch_atomic_inc_return_relaxed(v); } -#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed -#endif - -#else /* arch_atomic_inc_return_relaxed */ - -#ifndef arch_atomic_inc_return_acquire +#elif defined(arch_atomic_inc_return) +#define raw_atomic_inc_return_release arch_atomic_inc_return +#else static __always_inline int -arch_atomic_inc_return_acquire(atomic_t *v) +raw_atomic_inc_return_release(atomic_t *v) { - int ret = arch_atomic_inc_return_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic_add_return_release(1, v); } -#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire #endif -#ifndef arch_atomic_inc_return_release +#if defined(arch_atomic_inc_return_relaxed) +#define raw_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed +#elif defined(arch_atomic_inc_return) +#define raw_atomic_inc_return_relaxed arch_atomic_inc_return +#else static __always_inline int -arch_atomic_inc_return_release(atomic_t *v) +raw_atomic_inc_return_relaxed(atomic_t *v) { - __atomic_release_fence(); - return arch_atomic_inc_return_relaxed(v); + return raw_atomic_add_return_relaxed(1, v); } -#define arch_atomic_inc_return_release arch_atomic_inc_return_release #endif -#ifndef arch_atomic_inc_return +#if defined(arch_atomic_fetch_inc) +#define raw_atomic_fetch_inc arch_atomic_fetch_inc +#elif defined(arch_atomic_fetch_inc_relaxed) static __always_inline int -arch_atomic_inc_return(atomic_t *v) +raw_atomic_fetch_inc(atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_inc_return_relaxed(v); + ret = arch_atomic_fetch_inc_relaxed(v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_inc_return arch_atomic_inc_return -#endif - -#endif /* arch_atomic_inc_return_relaxed */ - -#ifndef arch_atomic_fetch_inc_relaxed -#ifdef arch_atomic_fetch_inc -#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc -#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc -#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc -#endif /* arch_atomic_fetch_inc */ - -#ifndef arch_atomic_fetch_inc +#else static __always_inline int -arch_atomic_fetch_inc(atomic_t *v) +raw_atomic_fetch_inc(atomic_t *v) { - return arch_atomic_fetch_add(1, v); + return raw_atomic_fetch_add(1, v); } -#define arch_atomic_fetch_inc arch_atomic_fetch_inc #endif -#ifndef arch_atomic_fetch_inc_acquire +#if defined(arch_atomic_fetch_inc_acquire) +#define raw_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire +#elif defined(arch_atomic_fetch_inc_relaxed) static __always_inline int -arch_atomic_fetch_inc_acquire(atomic_t *v) +raw_atomic_fetch_inc_acquire(atomic_t *v) { - return arch_atomic_fetch_add_acquire(1, v); + int ret = arch_atomic_fetch_inc_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire -#endif - -#ifndef arch_atomic_fetch_inc_release +#elif defined(arch_atomic_fetch_inc) +#define raw_atomic_fetch_inc_acquire arch_atomic_fetch_inc +#else static __always_inline int -arch_atomic_fetch_inc_release(atomic_t *v) +raw_atomic_fetch_inc_acquire(atomic_t *v) { - return arch_atomic_fetch_add_release(1, v); + return raw_atomic_fetch_add_acquire(1, v); } -#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release #endif -#ifndef arch_atomic_fetch_inc_relaxed +#if defined(arch_atomic_fetch_inc_release) +#define raw_atomic_fetch_inc_release arch_atomic_fetch_inc_release +#elif defined(arch_atomic_fetch_inc_relaxed) +static __always_inline int +raw_atomic_fetch_inc_release(atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_inc_relaxed(v); +} +#elif defined(arch_atomic_fetch_inc) +#define raw_atomic_fetch_inc_release arch_atomic_fetch_inc +#else static __always_inline int -arch_atomic_fetch_inc_relaxed(atomic_t *v) +raw_atomic_fetch_inc_release(atomic_t *v) { - return arch_atomic_fetch_add_relaxed(1, v); + return raw_atomic_fetch_add_release(1, v); } -#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed #endif -#else /* arch_atomic_fetch_inc_relaxed */ - -#ifndef arch_atomic_fetch_inc_acquire +#if defined(arch_atomic_fetch_inc_relaxed) +#define raw_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed +#elif defined(arch_atomic_fetch_inc) +#define raw_atomic_fetch_inc_relaxed arch_atomic_fetch_inc +#else static __always_inline int -arch_atomic_fetch_inc_acquire(atomic_t *v) +raw_atomic_fetch_inc_relaxed(atomic_t *v) { - int ret = arch_atomic_fetch_inc_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic_fetch_add_relaxed(1, v); } -#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire #endif -#ifndef arch_atomic_fetch_inc_release -static __always_inline int -arch_atomic_fetch_inc_release(atomic_t *v) +#if defined(arch_atomic_dec) +#define raw_atomic_dec arch_atomic_dec +#else +static __always_inline void +raw_atomic_dec(atomic_t *v) { - __atomic_release_fence(); - return arch_atomic_fetch_inc_relaxed(v); + raw_atomic_sub(1, v); } -#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release #endif -#ifndef arch_atomic_fetch_inc +#if defined(arch_atomic_dec_return) +#define raw_atomic_dec_return arch_atomic_dec_return +#elif defined(arch_atomic_dec_return_relaxed) static __always_inline int -arch_atomic_fetch_inc(atomic_t *v) +raw_atomic_dec_return(atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_fetch_inc_relaxed(v); + ret = arch_atomic_dec_return_relaxed(v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_inc arch_atomic_fetch_inc -#endif - -#endif /* arch_atomic_fetch_inc_relaxed */ - -#ifndef arch_atomic_dec -static __always_inline void -arch_atomic_dec(atomic_t *v) -{ - arch_atomic_sub(1, v); -} -#define arch_atomic_dec arch_atomic_dec -#endif - -#ifndef arch_atomic_dec_return_relaxed -#ifdef arch_atomic_dec_return -#define arch_atomic_dec_return_acquire arch_atomic_dec_return -#define arch_atomic_dec_return_release arch_atomic_dec_return -#define arch_atomic_dec_return_relaxed arch_atomic_dec_return -#endif /* arch_atomic_dec_return */ - -#ifndef arch_atomic_dec_return +#else static __always_inline int -arch_atomic_dec_return(atomic_t *v) +raw_atomic_dec_return(atomic_t *v) { - return arch_atomic_sub_return(1, v); + return raw_atomic_sub_return(1, v); } -#define arch_atomic_dec_return arch_atomic_dec_return #endif -#ifndef arch_atomic_dec_return_acquire +#if defined(arch_atomic_dec_return_acquire) +#define raw_atomic_dec_return_acquire arch_atomic_dec_return_acquire +#elif defined(arch_atomic_dec_return_relaxed) static __always_inline int -arch_atomic_dec_return_acquire(atomic_t *v) +raw_atomic_dec_return_acquire(atomic_t *v) { - return arch_atomic_sub_return_acquire(1, v); + int ret = arch_atomic_dec_return_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire -#endif - -#ifndef arch_atomic_dec_return_release +#elif defined(arch_atomic_dec_return) +#define raw_atomic_dec_return_acquire arch_atomic_dec_return +#else static __always_inline int -arch_atomic_dec_return_release(atomic_t *v) +raw_atomic_dec_return_acquire(atomic_t *v) { - return arch_atomic_sub_return_release(1, v); + return raw_atomic_sub_return_acquire(1, v); } -#define arch_atomic_dec_return_release arch_atomic_dec_return_release #endif -#ifndef arch_atomic_dec_return_relaxed +#if defined(arch_atomic_dec_return_release) +#define raw_atomic_dec_return_release arch_atomic_dec_return_release +#elif defined(arch_atomic_dec_return_relaxed) static __always_inline int -arch_atomic_dec_return_relaxed(atomic_t *v) +raw_atomic_dec_return_release(atomic_t *v) { - return arch_atomic_sub_return_relaxed(1, v); + __atomic_release_fence(); + return arch_atomic_dec_return_relaxed(v); } -#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed -#endif - -#else /* arch_atomic_dec_return_relaxed */ - -#ifndef arch_atomic_dec_return_acquire +#elif defined(arch_atomic_dec_return) +#define raw_atomic_dec_return_release arch_atomic_dec_return +#else static __always_inline int -arch_atomic_dec_return_acquire(atomic_t *v) +raw_atomic_dec_return_release(atomic_t *v) { - int ret = arch_atomic_dec_return_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic_sub_return_release(1, v); } -#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire #endif -#ifndef arch_atomic_dec_return_release +#if defined(arch_atomic_dec_return_relaxed) +#define raw_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed +#elif defined(arch_atomic_dec_return) +#define raw_atomic_dec_return_relaxed arch_atomic_dec_return +#else static __always_inline int -arch_atomic_dec_return_release(atomic_t *v) +raw_atomic_dec_return_relaxed(atomic_t *v) { - __atomic_release_fence(); - return arch_atomic_dec_return_relaxed(v); + return raw_atomic_sub_return_relaxed(1, v); } -#define arch_atomic_dec_return_release arch_atomic_dec_return_release #endif -#ifndef arch_atomic_dec_return +#if defined(arch_atomic_fetch_dec) +#define raw_atomic_fetch_dec arch_atomic_fetch_dec +#elif defined(arch_atomic_fetch_dec_relaxed) static __always_inline int -arch_atomic_dec_return(atomic_t *v) +raw_atomic_fetch_dec(atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_dec_return_relaxed(v); + ret = arch_atomic_fetch_dec_relaxed(v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_dec_return arch_atomic_dec_return -#endif - -#endif /* arch_atomic_dec_return_relaxed */ - -#ifndef arch_atomic_fetch_dec_relaxed -#ifdef arch_atomic_fetch_dec -#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec -#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec -#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec -#endif /* arch_atomic_fetch_dec */ - -#ifndef arch_atomic_fetch_dec +#else static __always_inline int -arch_atomic_fetch_dec(atomic_t *v) +raw_atomic_fetch_dec(atomic_t *v) { - return arch_atomic_fetch_sub(1, v); + return raw_atomic_fetch_sub(1, v); } -#define arch_atomic_fetch_dec arch_atomic_fetch_dec #endif -#ifndef arch_atomic_fetch_dec_acquire +#if defined(arch_atomic_fetch_dec_acquire) +#define raw_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire +#elif defined(arch_atomic_fetch_dec_relaxed) static __always_inline int -arch_atomic_fetch_dec_acquire(atomic_t *v) +raw_atomic_fetch_dec_acquire(atomic_t *v) { - return arch_atomic_fetch_sub_acquire(1, v); + int ret = arch_atomic_fetch_dec_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire -#endif - -#ifndef arch_atomic_fetch_dec_release +#elif defined(arch_atomic_fetch_dec) +#define raw_atomic_fetch_dec_acquire arch_atomic_fetch_dec +#else static __always_inline int -arch_atomic_fetch_dec_release(atomic_t *v) +raw_atomic_fetch_dec_acquire(atomic_t *v) { - return arch_atomic_fetch_sub_release(1, v); + return raw_atomic_fetch_sub_acquire(1, v); } -#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release #endif -#ifndef arch_atomic_fetch_dec_relaxed +#if defined(arch_atomic_fetch_dec_release) +#define raw_atomic_fetch_dec_release arch_atomic_fetch_dec_release +#elif defined(arch_atomic_fetch_dec_relaxed) static __always_inline int -arch_atomic_fetch_dec_relaxed(atomic_t *v) +raw_atomic_fetch_dec_release(atomic_t *v) { - return arch_atomic_fetch_sub_relaxed(1, v); + __atomic_release_fence(); + return arch_atomic_fetch_dec_relaxed(v); } -#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed -#endif - -#else /* arch_atomic_fetch_dec_relaxed */ - -#ifndef arch_atomic_fetch_dec_acquire +#elif defined(arch_atomic_fetch_dec) +#define raw_atomic_fetch_dec_release arch_atomic_fetch_dec +#else static __always_inline int -arch_atomic_fetch_dec_acquire(atomic_t *v) +raw_atomic_fetch_dec_release(atomic_t *v) { - int ret = arch_atomic_fetch_dec_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic_fetch_sub_release(1, v); } -#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire #endif -#ifndef arch_atomic_fetch_dec_release +#if defined(arch_atomic_fetch_dec_relaxed) +#define raw_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed +#elif defined(arch_atomic_fetch_dec) +#define raw_atomic_fetch_dec_relaxed arch_atomic_fetch_dec +#else static __always_inline int -arch_atomic_fetch_dec_release(atomic_t *v) +raw_atomic_fetch_dec_relaxed(atomic_t *v) { - __atomic_release_fence(); - return arch_atomic_fetch_dec_relaxed(v); + return raw_atomic_fetch_sub_relaxed(1, v); } -#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release #endif -#ifndef arch_atomic_fetch_dec +#define raw_atomic_and arch_atomic_and + +#if defined(arch_atomic_fetch_and) +#define raw_atomic_fetch_and arch_atomic_fetch_and +#elif defined(arch_atomic_fetch_and_relaxed) static __always_inline int -arch_atomic_fetch_dec(atomic_t *v) +raw_atomic_fetch_and(int i, atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_fetch_dec_relaxed(v); + ret = arch_atomic_fetch_and_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_dec arch_atomic_fetch_dec +#else +#error "Unable to define raw_atomic_fetch_and" #endif -#endif /* arch_atomic_fetch_dec_relaxed */ - -#ifndef arch_atomic_fetch_and_relaxed -#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and -#define arch_atomic_fetch_and_release arch_atomic_fetch_and -#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and -#else /* arch_atomic_fetch_and_relaxed */ - -#ifndef arch_atomic_fetch_and_acquire +#if defined(arch_atomic_fetch_and_acquire) +#define raw_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire +#elif defined(arch_atomic_fetch_and_relaxed) static __always_inline int -arch_atomic_fetch_and_acquire(int i, atomic_t *v) +raw_atomic_fetch_and_acquire(int i, atomic_t *v) { int ret = arch_atomic_fetch_and_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire +#elif defined(arch_atomic_fetch_and) +#define raw_atomic_fetch_and_acquire arch_atomic_fetch_and +#else +#error "Unable to define raw_atomic_fetch_and_acquire" #endif -#ifndef arch_atomic_fetch_and_release +#if defined(arch_atomic_fetch_and_release) +#define raw_atomic_fetch_and_release arch_atomic_fetch_and_release +#elif defined(arch_atomic_fetch_and_relaxed) static __always_inline int -arch_atomic_fetch_and_release(int i, atomic_t *v) +raw_atomic_fetch_and_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_fetch_and_relaxed(i, v); } -#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release +#elif defined(arch_atomic_fetch_and) +#define raw_atomic_fetch_and_release arch_atomic_fetch_and +#else +#error "Unable to define raw_atomic_fetch_and_release" #endif -#ifndef arch_atomic_fetch_and +#if defined(arch_atomic_fetch_and_relaxed) +#define raw_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#elif defined(arch_atomic_fetch_and) +#define raw_atomic_fetch_and_relaxed arch_atomic_fetch_and +#else +#error "Unable to define raw_atomic_fetch_and_relaxed" +#endif + +#if defined(arch_atomic_andnot) +#define raw_atomic_andnot arch_atomic_andnot +#else +static __always_inline void +raw_atomic_andnot(int i, atomic_t *v) +{ + raw_atomic_and(~i, v); +} +#endif + +#if defined(arch_atomic_fetch_andnot) +#define raw_atomic_fetch_andnot arch_atomic_fetch_andnot +#elif defined(arch_atomic_fetch_andnot_relaxed) static __always_inline int -arch_atomic_fetch_and(int i, atomic_t *v) +raw_atomic_fetch_andnot(int i, atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_fetch_and_relaxed(i, v); + ret = arch_atomic_fetch_andnot_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_and arch_atomic_fetch_and -#endif - -#endif /* arch_atomic_fetch_and_relaxed */ - -#ifndef arch_atomic_andnot -static __always_inline void -arch_atomic_andnot(int i, atomic_t *v) +#else +static __always_inline int +raw_atomic_fetch_andnot(int i, atomic_t *v) { - arch_atomic_and(~i, v); + return raw_atomic_fetch_and(~i, v); } -#define arch_atomic_andnot arch_atomic_andnot #endif -#ifndef arch_atomic_fetch_andnot_relaxed -#ifdef arch_atomic_fetch_andnot -#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot -#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot -#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot -#endif /* arch_atomic_fetch_andnot */ - -#ifndef arch_atomic_fetch_andnot +#if defined(arch_atomic_fetch_andnot_acquire) +#define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire +#elif defined(arch_atomic_fetch_andnot_relaxed) +static __always_inline int +raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_andnot_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#elif defined(arch_atomic_fetch_andnot) +#define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot +#else static __always_inline int -arch_atomic_fetch_andnot(int i, atomic_t *v) +raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) { - return arch_atomic_fetch_and(~i, v); + return raw_atomic_fetch_and_acquire(~i, v); } -#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot #endif -#ifndef arch_atomic_fetch_andnot_acquire +#if defined(arch_atomic_fetch_andnot_release) +#define raw_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release +#elif defined(arch_atomic_fetch_andnot_relaxed) static __always_inline int -arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) +raw_atomic_fetch_andnot_release(int i, atomic_t *v) { - return arch_atomic_fetch_and_acquire(~i, v); + __atomic_release_fence(); + return arch_atomic_fetch_andnot_relaxed(i, v); } -#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire -#endif - -#ifndef arch_atomic_fetch_andnot_release +#elif defined(arch_atomic_fetch_andnot) +#define raw_atomic_fetch_andnot_release arch_atomic_fetch_andnot +#else static __always_inline int -arch_atomic_fetch_andnot_release(int i, atomic_t *v) +raw_atomic_fetch_andnot_release(int i, atomic_t *v) { - return arch_atomic_fetch_and_release(~i, v); + return raw_atomic_fetch_and_release(~i, v); } -#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release #endif -#ifndef arch_atomic_fetch_andnot_relaxed +#if defined(arch_atomic_fetch_andnot_relaxed) +#define raw_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#elif defined(arch_atomic_fetch_andnot) +#define raw_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot +#else static __always_inline int -arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v) +raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) { - return arch_atomic_fetch_and_relaxed(~i, v); + return raw_atomic_fetch_and_relaxed(~i, v); } -#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed #endif -#else /* arch_atomic_fetch_andnot_relaxed */ +#define raw_atomic_or arch_atomic_or -#ifndef arch_atomic_fetch_andnot_acquire +#if defined(arch_atomic_fetch_or) +#define raw_atomic_fetch_or arch_atomic_fetch_or +#elif defined(arch_atomic_fetch_or_relaxed) static __always_inline int -arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) +raw_atomic_fetch_or(int i, atomic_t *v) { - int ret = arch_atomic_fetch_andnot_relaxed(i, v); - __atomic_acquire_fence(); + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_or_relaxed(i, v); + __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire +#else +#error "Unable to define raw_atomic_fetch_or" #endif -#ifndef arch_atomic_fetch_andnot_release +#if defined(arch_atomic_fetch_or_acquire) +#define raw_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire +#elif defined(arch_atomic_fetch_or_relaxed) static __always_inline int -arch_atomic_fetch_andnot_release(int i, atomic_t *v) -{ - __atomic_release_fence(); - return arch_atomic_fetch_andnot_relaxed(i, v); -} -#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release -#endif - -#ifndef arch_atomic_fetch_andnot -static __always_inline int -arch_atomic_fetch_andnot(int i, atomic_t *v) -{ - int ret; - __atomic_pre_full_fence(); - ret = arch_atomic_fetch_andnot_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot -#endif - -#endif /* arch_atomic_fetch_andnot_relaxed */ - -#ifndef arch_atomic_fetch_or_relaxed -#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or -#define arch_atomic_fetch_or_release arch_atomic_fetch_or -#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or -#else /* arch_atomic_fetch_or_relaxed */ - -#ifndef arch_atomic_fetch_or_acquire -static __always_inline int -arch_atomic_fetch_or_acquire(int i, atomic_t *v) +raw_atomic_fetch_or_acquire(int i, atomic_t *v) { int ret = arch_atomic_fetch_or_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire +#elif defined(arch_atomic_fetch_or) +#define raw_atomic_fetch_or_acquire arch_atomic_fetch_or +#else +#error "Unable to define raw_atomic_fetch_or_acquire" #endif -#ifndef arch_atomic_fetch_or_release +#if defined(arch_atomic_fetch_or_release) +#define raw_atomic_fetch_or_release arch_atomic_fetch_or_release +#elif defined(arch_atomic_fetch_or_relaxed) static __always_inline int -arch_atomic_fetch_or_release(int i, atomic_t *v) +raw_atomic_fetch_or_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_fetch_or_relaxed(i, v); } -#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release +#elif defined(arch_atomic_fetch_or) +#define raw_atomic_fetch_or_release arch_atomic_fetch_or +#else +#error "Unable to define raw_atomic_fetch_or_release" #endif -#ifndef arch_atomic_fetch_or +#if defined(arch_atomic_fetch_or_relaxed) +#define raw_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#elif defined(arch_atomic_fetch_or) +#define raw_atomic_fetch_or_relaxed arch_atomic_fetch_or +#else +#error "Unable to define raw_atomic_fetch_or_relaxed" +#endif + +#define raw_atomic_xor arch_atomic_xor + +#if defined(arch_atomic_fetch_xor) +#define raw_atomic_fetch_xor arch_atomic_fetch_xor +#elif defined(arch_atomic_fetch_xor_relaxed) static __always_inline int -arch_atomic_fetch_or(int i, atomic_t *v) +raw_atomic_fetch_xor(int i, atomic_t *v) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_fetch_or_relaxed(i, v); + ret = arch_atomic_fetch_xor_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_or arch_atomic_fetch_or +#else +#error "Unable to define raw_atomic_fetch_xor" #endif -#endif /* arch_atomic_fetch_or_relaxed */ - -#ifndef arch_atomic_fetch_xor_relaxed -#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor -#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor -#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor -#else /* arch_atomic_fetch_xor_relaxed */ - -#ifndef arch_atomic_fetch_xor_acquire +#if defined(arch_atomic_fetch_xor_acquire) +#define raw_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire +#elif defined(arch_atomic_fetch_xor_relaxed) static __always_inline int -arch_atomic_fetch_xor_acquire(int i, atomic_t *v) +raw_atomic_fetch_xor_acquire(int i, atomic_t *v) { int ret = arch_atomic_fetch_xor_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire +#elif defined(arch_atomic_fetch_xor) +#define raw_atomic_fetch_xor_acquire arch_atomic_fetch_xor +#else +#error "Unable to define raw_atomic_fetch_xor_acquire" #endif -#ifndef arch_atomic_fetch_xor_release +#if defined(arch_atomic_fetch_xor_release) +#define raw_atomic_fetch_xor_release arch_atomic_fetch_xor_release +#elif defined(arch_atomic_fetch_xor_relaxed) static __always_inline int -arch_atomic_fetch_xor_release(int i, atomic_t *v) +raw_atomic_fetch_xor_release(int i, atomic_t *v) { __atomic_release_fence(); return arch_atomic_fetch_xor_relaxed(i, v); } -#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release +#elif defined(arch_atomic_fetch_xor) +#define raw_atomic_fetch_xor_release arch_atomic_fetch_xor +#else +#error "Unable to define raw_atomic_fetch_xor_release" #endif -#ifndef arch_atomic_fetch_xor +#if defined(arch_atomic_fetch_xor_relaxed) +#define raw_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed +#elif defined(arch_atomic_fetch_xor) +#define raw_atomic_fetch_xor_relaxed arch_atomic_fetch_xor +#else +#error "Unable to define raw_atomic_fetch_xor_relaxed" +#endif + +#if defined(arch_atomic_xchg) +#define raw_atomic_xchg arch_atomic_xchg +#elif defined(arch_atomic_xchg_relaxed) static __always_inline int -arch_atomic_fetch_xor(int i, atomic_t *v) +raw_atomic_xchg(atomic_t *v, int i) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_fetch_xor_relaxed(i, v); + ret = arch_atomic_xchg_relaxed(v, i); __atomic_post_full_fence(); return ret; } -#define arch_atomic_fetch_xor arch_atomic_fetch_xor -#endif - -#endif /* arch_atomic_fetch_xor_relaxed */ - -#ifndef arch_atomic_xchg_relaxed -#ifdef arch_atomic_xchg -#define arch_atomic_xchg_acquire arch_atomic_xchg -#define arch_atomic_xchg_release arch_atomic_xchg -#define arch_atomic_xchg_relaxed arch_atomic_xchg -#endif /* arch_atomic_xchg */ - -#ifndef arch_atomic_xchg +#else static __always_inline int -arch_atomic_xchg(atomic_t *v, int new) +raw_atomic_xchg(atomic_t *v, int new) { - return arch_xchg(&v->counter, new); + return raw_xchg(&v->counter, new); } -#define arch_atomic_xchg arch_atomic_xchg #endif -#ifndef arch_atomic_xchg_acquire +#if defined(arch_atomic_xchg_acquire) +#define raw_atomic_xchg_acquire arch_atomic_xchg_acquire +#elif defined(arch_atomic_xchg_relaxed) static __always_inline int -arch_atomic_xchg_acquire(atomic_t *v, int new) +raw_atomic_xchg_acquire(atomic_t *v, int i) { - return arch_xchg_acquire(&v->counter, new); + int ret = arch_atomic_xchg_relaxed(v, i); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire -#endif - -#ifndef arch_atomic_xchg_release +#elif defined(arch_atomic_xchg) +#define raw_atomic_xchg_acquire arch_atomic_xchg +#else static __always_inline int -arch_atomic_xchg_release(atomic_t *v, int new) +raw_atomic_xchg_acquire(atomic_t *v, int new) { - return arch_xchg_release(&v->counter, new); + return raw_xchg_acquire(&v->counter, new); } -#define arch_atomic_xchg_release arch_atomic_xchg_release #endif -#ifndef arch_atomic_xchg_relaxed +#if defined(arch_atomic_xchg_release) +#define raw_atomic_xchg_release arch_atomic_xchg_release +#elif defined(arch_atomic_xchg_relaxed) static __always_inline int -arch_atomic_xchg_relaxed(atomic_t *v, int new) +raw_atomic_xchg_release(atomic_t *v, int i) { - return arch_xchg_relaxed(&v->counter, new); + __atomic_release_fence(); + return arch_atomic_xchg_relaxed(v, i); } -#define arch_atomic_xchg_relaxed arch_atomic_xchg_relaxed -#endif - -#else /* arch_atomic_xchg_relaxed */ - -#ifndef arch_atomic_xchg_acquire +#elif defined(arch_atomic_xchg) +#define raw_atomic_xchg_release arch_atomic_xchg +#else static __always_inline int -arch_atomic_xchg_acquire(atomic_t *v, int i) +raw_atomic_xchg_release(atomic_t *v, int new) { - int ret = arch_atomic_xchg_relaxed(v, i); - __atomic_acquire_fence(); - return ret; + return raw_xchg_release(&v->counter, new); } -#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire #endif -#ifndef arch_atomic_xchg_release +#if defined(arch_atomic_xchg_relaxed) +#define raw_atomic_xchg_relaxed arch_atomic_xchg_relaxed +#elif defined(arch_atomic_xchg) +#define raw_atomic_xchg_relaxed arch_atomic_xchg +#else static __always_inline int -arch_atomic_xchg_release(atomic_t *v, int i) +raw_atomic_xchg_relaxed(atomic_t *v, int new) { - __atomic_release_fence(); - return arch_atomic_xchg_relaxed(v, i); + return raw_xchg_relaxed(&v->counter, new); } -#define arch_atomic_xchg_release arch_atomic_xchg_release #endif -#ifndef arch_atomic_xchg +#if defined(arch_atomic_cmpxchg) +#define raw_atomic_cmpxchg arch_atomic_cmpxchg +#elif defined(arch_atomic_cmpxchg_relaxed) static __always_inline int -arch_atomic_xchg(atomic_t *v, int i) +raw_atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; __atomic_pre_full_fence(); - ret = arch_atomic_xchg_relaxed(v, i); + ret = arch_atomic_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; } -#define arch_atomic_xchg arch_atomic_xchg -#endif - -#endif /* arch_atomic_xchg_relaxed */ - -#ifndef arch_atomic_cmpxchg_relaxed -#ifdef arch_atomic_cmpxchg -#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg -#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg -#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg -#endif /* arch_atomic_cmpxchg */ - -#ifndef arch_atomic_cmpxchg +#else static __always_inline int -arch_atomic_cmpxchg(atomic_t *v, int old, int new) +raw_atomic_cmpxchg(atomic_t *v, int old, int new) { - return arch_cmpxchg(&v->counter, old, new); + return raw_cmpxchg(&v->counter, old, new); } -#define arch_atomic_cmpxchg arch_atomic_cmpxchg #endif -#ifndef arch_atomic_cmpxchg_acquire +#if defined(arch_atomic_cmpxchg_acquire) +#define raw_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire +#elif defined(arch_atomic_cmpxchg_relaxed) static __always_inline int -arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { - return arch_cmpxchg_acquire(&v->counter, old, new); + int ret = arch_atomic_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire -#endif - -#ifndef arch_atomic_cmpxchg_release +#elif defined(arch_atomic_cmpxchg) +#define raw_atomic_cmpxchg_acquire arch_atomic_cmpxchg +#else static __always_inline int -arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) +raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { - return arch_cmpxchg_release(&v->counter, old, new); + return raw_cmpxchg_acquire(&v->counter, old, new); } -#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release #endif -#ifndef arch_atomic_cmpxchg_relaxed +#if defined(arch_atomic_cmpxchg_release) +#define raw_atomic_cmpxchg_release arch_atomic_cmpxchg_release +#elif defined(arch_atomic_cmpxchg_relaxed) static __always_inline int -arch_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) +raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) { - return arch_cmpxchg_relaxed(&v->counter, old, new); + __atomic_release_fence(); + return arch_atomic_cmpxchg_relaxed(v, old, new); } -#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed -#endif - -#else /* arch_atomic_cmpxchg_relaxed */ - -#ifndef arch_atomic_cmpxchg_acquire +#elif defined(arch_atomic_cmpxchg) +#define raw_atomic_cmpxchg_release arch_atomic_cmpxchg +#else static __always_inline int -arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) { - int ret = arch_atomic_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; + return raw_cmpxchg_release(&v->counter, old, new); } -#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire #endif -#ifndef arch_atomic_cmpxchg_release +#if defined(arch_atomic_cmpxchg_relaxed) +#define raw_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed +#elif defined(arch_atomic_cmpxchg) +#define raw_atomic_cmpxchg_relaxed arch_atomic_cmpxchg +#else static __always_inline int -arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) +raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { - __atomic_release_fence(); - return arch_atomic_cmpxchg_relaxed(v, old, new); + return raw_cmpxchg_relaxed(&v->counter, old, new); } -#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release #endif -#ifndef arch_atomic_cmpxchg -static __always_inline int -arch_atomic_cmpxchg(atomic_t *v, int old, int new) +#if defined(arch_atomic_try_cmpxchg) +#define raw_atomic_try_cmpxchg arch_atomic_try_cmpxchg +#elif defined(arch_atomic_try_cmpxchg_relaxed) +static __always_inline bool +raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) { - int ret; + bool ret; __atomic_pre_full_fence(); - ret = arch_atomic_cmpxchg_relaxed(v, old, new); + ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; } -#define arch_atomic_cmpxchg arch_atomic_cmpxchg -#endif - -#endif /* arch_atomic_cmpxchg_relaxed */ - -#ifndef arch_atomic_try_cmpxchg_relaxed -#ifdef arch_atomic_try_cmpxchg -#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg -#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg -#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg -#endif /* arch_atomic_try_cmpxchg */ - -#ifndef arch_atomic_try_cmpxchg +#else static __always_inline bool -arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) +raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) { int r, o = *old; - r = arch_atomic_cmpxchg(v, o, new); + r = raw_atomic_cmpxchg(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg #endif -#ifndef arch_atomic_try_cmpxchg_acquire +#if defined(arch_atomic_try_cmpxchg_acquire) +#define raw_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire +#elif defined(arch_atomic_try_cmpxchg_relaxed) +static __always_inline bool +raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; +} +#elif defined(arch_atomic_try_cmpxchg) +#define raw_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg +#else static __always_inline bool -arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { int r, o = *old; - r = arch_atomic_cmpxchg_acquire(v, o, new); + r = raw_atomic_cmpxchg_acquire(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire #endif -#ifndef arch_atomic_try_cmpxchg_release +#if defined(arch_atomic_try_cmpxchg_release) +#define raw_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release +#elif defined(arch_atomic_try_cmpxchg_relaxed) static __always_inline bool -arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + __atomic_release_fence(); + return arch_atomic_try_cmpxchg_relaxed(v, old, new); +} +#elif defined(arch_atomic_try_cmpxchg) +#define raw_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg +#else +static __always_inline bool +raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { int r, o = *old; - r = arch_atomic_cmpxchg_release(v, o, new); + r = raw_atomic_cmpxchg_release(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release #endif -#ifndef arch_atomic_try_cmpxchg_relaxed +#if defined(arch_atomic_try_cmpxchg_relaxed) +#define raw_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed +#elif defined(arch_atomic_try_cmpxchg) +#define raw_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg +#else static __always_inline bool -arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) +raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { int r, o = *old; - r = arch_atomic_cmpxchg_relaxed(v, o, new); + r = raw_atomic_cmpxchg_relaxed(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed -#endif - -#else /* arch_atomic_try_cmpxchg_relaxed */ - -#ifndef arch_atomic_try_cmpxchg_acquire -static __always_inline bool -arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) -{ - bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; -} -#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire -#endif - -#ifndef arch_atomic_try_cmpxchg_release -static __always_inline bool -arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) -{ - __atomic_release_fence(); - return arch_atomic_try_cmpxchg_relaxed(v, old, new); -} -#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release -#endif - -#ifndef arch_atomic_try_cmpxchg -static __always_inline bool -arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) -{ - bool ret; - __atomic_pre_full_fence(); - ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); - __atomic_post_full_fence(); - return ret; -} -#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg #endif -#endif /* arch_atomic_try_cmpxchg_relaxed */ - -#ifndef arch_atomic_sub_and_test +#if defined(arch_atomic_sub_and_test) +#define raw_atomic_sub_and_test arch_atomic_sub_and_test +#else static __always_inline bool -arch_atomic_sub_and_test(int i, atomic_t *v) +raw_atomic_sub_and_test(int i, atomic_t *v) { - return arch_atomic_sub_return(i, v) == 0; + return raw_atomic_sub_return(i, v) == 0; } -#define arch_atomic_sub_and_test arch_atomic_sub_and_test #endif -#ifndef arch_atomic_dec_and_test +#if defined(arch_atomic_dec_and_test) +#define raw_atomic_dec_and_test arch_atomic_dec_and_test +#else static __always_inline bool -arch_atomic_dec_and_test(atomic_t *v) +raw_atomic_dec_and_test(atomic_t *v) { - return arch_atomic_dec_return(v) == 0; + return raw_atomic_dec_return(v) == 0; } -#define arch_atomic_dec_and_test arch_atomic_dec_and_test #endif -#ifndef arch_atomic_inc_and_test +#if defined(arch_atomic_inc_and_test) +#define raw_atomic_inc_and_test arch_atomic_inc_and_test +#else static __always_inline bool -arch_atomic_inc_and_test(atomic_t *v) +raw_atomic_inc_and_test(atomic_t *v) { - return arch_atomic_inc_return(v) == 0; + return raw_atomic_inc_return(v) == 0; } -#define arch_atomic_inc_and_test arch_atomic_inc_and_test #endif -#ifndef arch_atomic_add_negative_relaxed -#ifdef arch_atomic_add_negative -#define arch_atomic_add_negative_acquire arch_atomic_add_negative -#define arch_atomic_add_negative_release arch_atomic_add_negative -#define arch_atomic_add_negative_relaxed arch_atomic_add_negative -#endif /* arch_atomic_add_negative */ - -#ifndef arch_atomic_add_negative +#if defined(arch_atomic_add_negative) +#define raw_atomic_add_negative arch_atomic_add_negative +#elif defined(arch_atomic_add_negative_relaxed) static __always_inline bool -arch_atomic_add_negative(int i, atomic_t *v) +raw_atomic_add_negative(int i, atomic_t *v) { - return arch_atomic_add_return(i, v) < 0; + bool ret; + __atomic_pre_full_fence(); + ret = arch_atomic_add_negative_relaxed(i, v); + __atomic_post_full_fence(); + return ret; } -#define arch_atomic_add_negative arch_atomic_add_negative -#endif - -#ifndef arch_atomic_add_negative_acquire +#else static __always_inline bool -arch_atomic_add_negative_acquire(int i, atomic_t *v) +raw_atomic_add_negative(int i, atomic_t *v) { - return arch_atomic_add_return_acquire(i, v) < 0; + return raw_atomic_add_return(i, v) < 0; } -#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire #endif -#ifndef arch_atomic_add_negative_release +#if defined(arch_atomic_add_negative_acquire) +#define raw_atomic_add_negative_acquire arch_atomic_add_negative_acquire +#elif defined(arch_atomic_add_negative_relaxed) static __always_inline bool -arch_atomic_add_negative_release(int i, atomic_t *v) +raw_atomic_add_negative_acquire(int i, atomic_t *v) { - return arch_atomic_add_return_release(i, v) < 0; + bool ret = arch_atomic_add_negative_relaxed(i, v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic_add_negative_release arch_atomic_add_negative_release -#endif - -#ifndef arch_atomic_add_negative_relaxed +#elif defined(arch_atomic_add_negative) +#define raw_atomic_add_negative_acquire arch_atomic_add_negative +#else static __always_inline bool -arch_atomic_add_negative_relaxed(int i, atomic_t *v) +raw_atomic_add_negative_acquire(int i, atomic_t *v) { - return arch_atomic_add_return_relaxed(i, v) < 0; + return raw_atomic_add_return_acquire(i, v) < 0; } -#define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed #endif -#else /* arch_atomic_add_negative_relaxed */ - -#ifndef arch_atomic_add_negative_acquire +#if defined(arch_atomic_add_negative_release) +#define raw_atomic_add_negative_release arch_atomic_add_negative_release +#elif defined(arch_atomic_add_negative_relaxed) static __always_inline bool -arch_atomic_add_negative_acquire(int i, atomic_t *v) +raw_atomic_add_negative_release(int i, atomic_t *v) { - bool ret = arch_atomic_add_negative_relaxed(i, v); - __atomic_acquire_fence(); - return ret; + __atomic_release_fence(); + return arch_atomic_add_negative_relaxed(i, v); } -#define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire -#endif - -#ifndef arch_atomic_add_negative_release +#elif defined(arch_atomic_add_negative) +#define raw_atomic_add_negative_release arch_atomic_add_negative +#else static __always_inline bool -arch_atomic_add_negative_release(int i, atomic_t *v) +raw_atomic_add_negative_release(int i, atomic_t *v) { - __atomic_release_fence(); - return arch_atomic_add_negative_relaxed(i, v); + return raw_atomic_add_return_release(i, v) < 0; } -#define arch_atomic_add_negative_release arch_atomic_add_negative_release #endif -#ifndef arch_atomic_add_negative +#if defined(arch_atomic_add_negative_relaxed) +#define raw_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed +#elif defined(arch_atomic_add_negative) +#define raw_atomic_add_negative_relaxed arch_atomic_add_negative +#else static __always_inline bool -arch_atomic_add_negative(int i, atomic_t *v) +raw_atomic_add_negative_relaxed(int i, atomic_t *v) { - bool ret; - __atomic_pre_full_fence(); - ret = arch_atomic_add_negative_relaxed(i, v); - __atomic_post_full_fence(); - return ret; + return raw_atomic_add_return_relaxed(i, v) < 0; } -#define arch_atomic_add_negative arch_atomic_add_negative #endif -#endif /* arch_atomic_add_negative_relaxed */ - -#ifndef arch_atomic_fetch_add_unless +#if defined(arch_atomic_fetch_add_unless) +#define raw_atomic_fetch_add_unless arch_atomic_fetch_add_unless +#else static __always_inline int -arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) +raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) { - int c = arch_atomic_read(v); + int c = raw_atomic_read(v); do { if (unlikely(c == u)) break; - } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); + } while (!raw_atomic_try_cmpxchg(v, &c, c + a)); return c; } -#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless #endif -#ifndef arch_atomic_add_unless +#if defined(arch_atomic_add_unless) +#define raw_atomic_add_unless arch_atomic_add_unless +#else static __always_inline bool -arch_atomic_add_unless(atomic_t *v, int a, int u) +raw_atomic_add_unless(atomic_t *v, int a, int u) { - return arch_atomic_fetch_add_unless(v, a, u) != u; + return raw_atomic_fetch_add_unless(v, a, u) != u; } -#define arch_atomic_add_unless arch_atomic_add_unless #endif -#ifndef arch_atomic_inc_not_zero +#if defined(arch_atomic_inc_not_zero) +#define raw_atomic_inc_not_zero arch_atomic_inc_not_zero +#else static __always_inline bool -arch_atomic_inc_not_zero(atomic_t *v) +raw_atomic_inc_not_zero(atomic_t *v) { - return arch_atomic_add_unless(v, 1, 0); + return raw_atomic_add_unless(v, 1, 0); } -#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero #endif -#ifndef arch_atomic_inc_unless_negative +#if defined(arch_atomic_inc_unless_negative) +#define raw_atomic_inc_unless_negative arch_atomic_inc_unless_negative +#else static __always_inline bool -arch_atomic_inc_unless_negative(atomic_t *v) +raw_atomic_inc_unless_negative(atomic_t *v) { - int c = arch_atomic_read(v); + int c = raw_atomic_read(v); do { if (unlikely(c < 0)) return false; - } while (!arch_atomic_try_cmpxchg(v, &c, c + 1)); + } while (!raw_atomic_try_cmpxchg(v, &c, c + 1)); return true; } -#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative #endif -#ifndef arch_atomic_dec_unless_positive +#if defined(arch_atomic_dec_unless_positive) +#define raw_atomic_dec_unless_positive arch_atomic_dec_unless_positive +#else static __always_inline bool -arch_atomic_dec_unless_positive(atomic_t *v) +raw_atomic_dec_unless_positive(atomic_t *v) { - int c = arch_atomic_read(v); + int c = raw_atomic_read(v); do { if (unlikely(c > 0)) return false; - } while (!arch_atomic_try_cmpxchg(v, &c, c - 1)); + } while (!raw_atomic_try_cmpxchg(v, &c, c - 1)); return true; } -#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive #endif -#ifndef arch_atomic_dec_if_positive +#if defined(arch_atomic_dec_if_positive) +#define raw_atomic_dec_if_positive arch_atomic_dec_if_positive +#else static __always_inline int -arch_atomic_dec_if_positive(atomic_t *v) +raw_atomic_dec_if_positive(atomic_t *v) { - int dec, c = arch_atomic_read(v); + int dec, c = raw_atomic_read(v); do { dec = c - 1; if (unlikely(dec < 0)) break; - } while (!arch_atomic_try_cmpxchg(v, &c, dec)); + } while (!raw_atomic_try_cmpxchg(v, &c, dec)); return dec; } -#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive #endif #ifdef CONFIG_GENERIC_ATOMIC64 #include #endif -#ifndef arch_atomic64_read_acquire +#define raw_atomic64_read arch_atomic64_read + +#if defined(arch_atomic64_read_acquire) +#define raw_atomic64_read_acquire arch_atomic64_read_acquire +#elif defined(arch_atomic64_read) +#define raw_atomic64_read_acquire arch_atomic64_read +#else static __always_inline s64 -arch_atomic64_read_acquire(const atomic64_t *v) +raw_atomic64_read_acquire(const atomic64_t *v) { s64 ret; if (__native_word(atomic64_t)) { ret = smp_load_acquire(&(v)->counter); } else { - ret = arch_atomic64_read(v); + ret = raw_atomic64_read(v); __atomic_acquire_fence(); } return ret; } -#define arch_atomic64_read_acquire arch_atomic64_read_acquire #endif -#ifndef arch_atomic64_set_release +#define raw_atomic64_set arch_atomic64_set + +#if defined(arch_atomic64_set_release) +#define raw_atomic64_set_release arch_atomic64_set_release +#elif defined(arch_atomic64_set) +#define raw_atomic64_set_release arch_atomic64_set +#else static __always_inline void -arch_atomic64_set_release(atomic64_t *v, s64 i) +raw_atomic64_set_release(atomic64_t *v, s64 i) { if (__native_word(atomic64_t)) { smp_store_release(&(v)->counter, i); } else { __atomic_release_fence(); - arch_atomic64_set(v, i); + raw_atomic64_set(v, i); } } -#define arch_atomic64_set_release arch_atomic64_set_release #endif -#ifndef arch_atomic64_add_return_relaxed -#define arch_atomic64_add_return_acquire arch_atomic64_add_return -#define arch_atomic64_add_return_release arch_atomic64_add_return -#define arch_atomic64_add_return_relaxed arch_atomic64_add_return -#else /* arch_atomic64_add_return_relaxed */ +#define raw_atomic64_add arch_atomic64_add + +#if defined(arch_atomic64_add_return) +#define raw_atomic64_add_return arch_atomic64_add_return +#elif defined(arch_atomic64_add_return_relaxed) +static __always_inline s64 +raw_atomic64_add_return(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_add_return_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#else +#error "Unable to define raw_atomic64_add_return" +#endif -#ifndef arch_atomic64_add_return_acquire +#if defined(arch_atomic64_add_return_acquire) +#define raw_atomic64_add_return_acquire arch_atomic64_add_return_acquire +#elif defined(arch_atomic64_add_return_relaxed) static __always_inline s64 -arch_atomic64_add_return_acquire(s64 i, atomic64_t *v) +raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_add_return_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire +#elif defined(arch_atomic64_add_return) +#define raw_atomic64_add_return_acquire arch_atomic64_add_return +#else +#error "Unable to define raw_atomic64_add_return_acquire" #endif -#ifndef arch_atomic64_add_return_release +#if defined(arch_atomic64_add_return_release) +#define raw_atomic64_add_return_release arch_atomic64_add_return_release +#elif defined(arch_atomic64_add_return_relaxed) static __always_inline s64 -arch_atomic64_add_return_release(s64 i, atomic64_t *v) +raw_atomic64_add_return_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_add_return_relaxed(i, v); } -#define arch_atomic64_add_return_release arch_atomic64_add_return_release +#elif defined(arch_atomic64_add_return) +#define raw_atomic64_add_return_release arch_atomic64_add_return +#else +#error "Unable to define raw_atomic64_add_return_release" +#endif + +#if defined(arch_atomic64_add_return_relaxed) +#define raw_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#elif defined(arch_atomic64_add_return) +#define raw_atomic64_add_return_relaxed arch_atomic64_add_return +#else +#error "Unable to define raw_atomic64_add_return_relaxed" #endif -#ifndef arch_atomic64_add_return +#if defined(arch_atomic64_fetch_add) +#define raw_atomic64_fetch_add arch_atomic64_fetch_add +#elif defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 -arch_atomic64_add_return(s64 i, atomic64_t *v) +raw_atomic64_fetch_add(s64 i, atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_add_return_relaxed(i, v); + ret = arch_atomic64_fetch_add_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_add_return arch_atomic64_add_return +#else +#error "Unable to define raw_atomic64_fetch_add" #endif -#endif /* arch_atomic64_add_return_relaxed */ - -#ifndef arch_atomic64_fetch_add_relaxed -#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add -#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add -#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add -#else /* arch_atomic64_fetch_add_relaxed */ - -#ifndef arch_atomic64_fetch_add_acquire +#if defined(arch_atomic64_fetch_add_acquire) +#define raw_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire +#elif defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 -arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_fetch_add_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire +#elif defined(arch_atomic64_fetch_add) +#define raw_atomic64_fetch_add_acquire arch_atomic64_fetch_add +#else +#error "Unable to define raw_atomic64_fetch_add_acquire" #endif -#ifndef arch_atomic64_fetch_add_release +#if defined(arch_atomic64_fetch_add_release) +#define raw_atomic64_fetch_add_release arch_atomic64_fetch_add_release +#elif defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 -arch_atomic64_fetch_add_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_fetch_add_relaxed(i, v); } -#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release +#elif defined(arch_atomic64_fetch_add) +#define raw_atomic64_fetch_add_release arch_atomic64_fetch_add +#else +#error "Unable to define raw_atomic64_fetch_add_release" +#endif + +#if defined(arch_atomic64_fetch_add_relaxed) +#define raw_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#elif defined(arch_atomic64_fetch_add) +#define raw_atomic64_fetch_add_relaxed arch_atomic64_fetch_add +#else +#error "Unable to define raw_atomic64_fetch_add_relaxed" #endif -#ifndef arch_atomic64_fetch_add +#define raw_atomic64_sub arch_atomic64_sub + +#if defined(arch_atomic64_sub_return) +#define raw_atomic64_sub_return arch_atomic64_sub_return +#elif defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 -arch_atomic64_fetch_add(s64 i, atomic64_t *v) +raw_atomic64_sub_return(s64 i, atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_add_relaxed(i, v); + ret = arch_atomic64_sub_return_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#else +#error "Unable to define raw_atomic64_sub_return" #endif -#endif /* arch_atomic64_fetch_add_relaxed */ - -#ifndef arch_atomic64_sub_return_relaxed -#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return -#define arch_atomic64_sub_return_release arch_atomic64_sub_return -#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return -#else /* arch_atomic64_sub_return_relaxed */ - -#ifndef arch_atomic64_sub_return_acquire +#if defined(arch_atomic64_sub_return_acquire) +#define raw_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire +#elif defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 -arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v) +raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_sub_return_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire +#elif defined(arch_atomic64_sub_return) +#define raw_atomic64_sub_return_acquire arch_atomic64_sub_return +#else +#error "Unable to define raw_atomic64_sub_return_acquire" #endif -#ifndef arch_atomic64_sub_return_release +#if defined(arch_atomic64_sub_return_release) +#define raw_atomic64_sub_return_release arch_atomic64_sub_return_release +#elif defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 -arch_atomic64_sub_return_release(s64 i, atomic64_t *v) +raw_atomic64_sub_return_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_sub_return_relaxed(i, v); } -#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release +#elif defined(arch_atomic64_sub_return) +#define raw_atomic64_sub_return_release arch_atomic64_sub_return +#else +#error "Unable to define raw_atomic64_sub_return_release" +#endif + +#if defined(arch_atomic64_sub_return_relaxed) +#define raw_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#elif defined(arch_atomic64_sub_return) +#define raw_atomic64_sub_return_relaxed arch_atomic64_sub_return +#else +#error "Unable to define raw_atomic64_sub_return_relaxed" #endif -#ifndef arch_atomic64_sub_return +#if defined(arch_atomic64_fetch_sub) +#define raw_atomic64_fetch_sub arch_atomic64_fetch_sub +#elif defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 -arch_atomic64_sub_return(s64 i, atomic64_t *v) +raw_atomic64_fetch_sub(s64 i, atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_sub_return_relaxed(i, v); + ret = arch_atomic64_fetch_sub_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_sub_return arch_atomic64_sub_return +#else +#error "Unable to define raw_atomic64_fetch_sub" #endif -#endif /* arch_atomic64_sub_return_relaxed */ - -#ifndef arch_atomic64_fetch_sub_relaxed -#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub -#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub -#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub -#else /* arch_atomic64_fetch_sub_relaxed */ - -#ifndef arch_atomic64_fetch_sub_acquire +#if defined(arch_atomic64_fetch_sub_acquire) +#define raw_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire +#elif defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 -arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire +#elif defined(arch_atomic64_fetch_sub) +#define raw_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub +#else +#error "Unable to define raw_atomic64_fetch_sub_acquire" #endif -#ifndef arch_atomic64_fetch_sub_release +#if defined(arch_atomic64_fetch_sub_release) +#define raw_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release +#elif defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 -arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_fetch_sub_relaxed(i, v); } -#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release +#elif defined(arch_atomic64_fetch_sub) +#define raw_atomic64_fetch_sub_release arch_atomic64_fetch_sub +#else +#error "Unable to define raw_atomic64_fetch_sub_release" #endif -#ifndef arch_atomic64_fetch_sub -static __always_inline s64 -arch_atomic64_fetch_sub(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_sub_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub +#if defined(arch_atomic64_fetch_sub_relaxed) +#define raw_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed +#elif defined(arch_atomic64_fetch_sub) +#define raw_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub +#else +#error "Unable to define raw_atomic64_fetch_sub_relaxed" #endif -#endif /* arch_atomic64_fetch_sub_relaxed */ - -#ifndef arch_atomic64_inc +#if defined(arch_atomic64_inc) +#define raw_atomic64_inc arch_atomic64_inc +#else static __always_inline void -arch_atomic64_inc(atomic64_t *v) +raw_atomic64_inc(atomic64_t *v) { - arch_atomic64_add(1, v); + raw_atomic64_add(1, v); } -#define arch_atomic64_inc arch_atomic64_inc #endif -#ifndef arch_atomic64_inc_return_relaxed -#ifdef arch_atomic64_inc_return -#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return -#define arch_atomic64_inc_return_release arch_atomic64_inc_return -#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return -#endif /* arch_atomic64_inc_return */ - -#ifndef arch_atomic64_inc_return +#if defined(arch_atomic64_inc_return) +#define raw_atomic64_inc_return arch_atomic64_inc_return +#elif defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 -arch_atomic64_inc_return(atomic64_t *v) +raw_atomic64_inc_return(atomic64_t *v) { - return arch_atomic64_add_return(1, v); + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_inc_return_relaxed(v); + __atomic_post_full_fence(); + return ret; } -#define arch_atomic64_inc_return arch_atomic64_inc_return -#endif - -#ifndef arch_atomic64_inc_return_acquire +#else static __always_inline s64 -arch_atomic64_inc_return_acquire(atomic64_t *v) +raw_atomic64_inc_return(atomic64_t *v) { - return arch_atomic64_add_return_acquire(1, v); + return raw_atomic64_add_return(1, v); } -#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire #endif -#ifndef arch_atomic64_inc_return_release +#if defined(arch_atomic64_inc_return_acquire) +#define raw_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire +#elif defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 -arch_atomic64_inc_return_release(atomic64_t *v) +raw_atomic64_inc_return_acquire(atomic64_t *v) { - return arch_atomic64_add_return_release(1, v); + s64 ret = arch_atomic64_inc_return_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release -#endif - -#ifndef arch_atomic64_inc_return_relaxed +#elif defined(arch_atomic64_inc_return) +#define raw_atomic64_inc_return_acquire arch_atomic64_inc_return +#else static __always_inline s64 -arch_atomic64_inc_return_relaxed(atomic64_t *v) +raw_atomic64_inc_return_acquire(atomic64_t *v) { - return arch_atomic64_add_return_relaxed(1, v); + return raw_atomic64_add_return_acquire(1, v); } -#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed #endif -#else /* arch_atomic64_inc_return_relaxed */ - -#ifndef arch_atomic64_inc_return_acquire +#if defined(arch_atomic64_inc_return_release) +#define raw_atomic64_inc_return_release arch_atomic64_inc_return_release +#elif defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 -arch_atomic64_inc_return_acquire(atomic64_t *v) +raw_atomic64_inc_return_release(atomic64_t *v) { - s64 ret = arch_atomic64_inc_return_relaxed(v); - __atomic_acquire_fence(); - return ret; + __atomic_release_fence(); + return arch_atomic64_inc_return_relaxed(v); +} +#elif defined(arch_atomic64_inc_return) +#define raw_atomic64_inc_return_release arch_atomic64_inc_return +#else +static __always_inline s64 +raw_atomic64_inc_return_release(atomic64_t *v) +{ + return raw_atomic64_add_return_release(1, v); } -#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire #endif -#ifndef arch_atomic64_inc_return_release +#if defined(arch_atomic64_inc_return_relaxed) +#define raw_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed +#elif defined(arch_atomic64_inc_return) +#define raw_atomic64_inc_return_relaxed arch_atomic64_inc_return +#else static __always_inline s64 -arch_atomic64_inc_return_release(atomic64_t *v) +raw_atomic64_inc_return_relaxed(atomic64_t *v) { - __atomic_release_fence(); - return arch_atomic64_inc_return_relaxed(v); + return raw_atomic64_add_return_relaxed(1, v); } -#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release #endif -#ifndef arch_atomic64_inc_return +#if defined(arch_atomic64_fetch_inc) +#define raw_atomic64_fetch_inc arch_atomic64_fetch_inc +#elif defined(arch_atomic64_fetch_inc_relaxed) static __always_inline s64 -arch_atomic64_inc_return(atomic64_t *v) +raw_atomic64_fetch_inc(atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_inc_return_relaxed(v); + ret = arch_atomic64_fetch_inc_relaxed(v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_inc_return arch_atomic64_inc_return -#endif - -#endif /* arch_atomic64_inc_return_relaxed */ - -#ifndef arch_atomic64_fetch_inc_relaxed -#ifdef arch_atomic64_fetch_inc -#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc -#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc -#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc -#endif /* arch_atomic64_fetch_inc */ - -#ifndef arch_atomic64_fetch_inc +#else static __always_inline s64 -arch_atomic64_fetch_inc(atomic64_t *v) +raw_atomic64_fetch_inc(atomic64_t *v) { - return arch_atomic64_fetch_add(1, v); + return raw_atomic64_fetch_add(1, v); } -#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc #endif -#ifndef arch_atomic64_fetch_inc_acquire +#if defined(arch_atomic64_fetch_inc_acquire) +#define raw_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire +#elif defined(arch_atomic64_fetch_inc_relaxed) +static __always_inline s64 +raw_atomic64_fetch_inc_acquire(atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_inc_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#elif defined(arch_atomic64_fetch_inc) +#define raw_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc +#else static __always_inline s64 -arch_atomic64_fetch_inc_acquire(atomic64_t *v) +raw_atomic64_fetch_inc_acquire(atomic64_t *v) { - return arch_atomic64_fetch_add_acquire(1, v); + return raw_atomic64_fetch_add_acquire(1, v); } -#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire #endif -#ifndef arch_atomic64_fetch_inc_release +#if defined(arch_atomic64_fetch_inc_release) +#define raw_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release +#elif defined(arch_atomic64_fetch_inc_relaxed) static __always_inline s64 -arch_atomic64_fetch_inc_release(atomic64_t *v) +raw_atomic64_fetch_inc_release(atomic64_t *v) { - return arch_atomic64_fetch_add_release(1, v); + __atomic_release_fence(); + return arch_atomic64_fetch_inc_relaxed(v); } -#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release -#endif - -#ifndef arch_atomic64_fetch_inc_relaxed +#elif defined(arch_atomic64_fetch_inc) +#define raw_atomic64_fetch_inc_release arch_atomic64_fetch_inc +#else static __always_inline s64 -arch_atomic64_fetch_inc_relaxed(atomic64_t *v) +raw_atomic64_fetch_inc_release(atomic64_t *v) { - return arch_atomic64_fetch_add_relaxed(1, v); + return raw_atomic64_fetch_add_release(1, v); } -#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed #endif -#else /* arch_atomic64_fetch_inc_relaxed */ - -#ifndef arch_atomic64_fetch_inc_acquire +#if defined(arch_atomic64_fetch_inc_relaxed) +#define raw_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed +#elif defined(arch_atomic64_fetch_inc) +#define raw_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc +#else static __always_inline s64 -arch_atomic64_fetch_inc_acquire(atomic64_t *v) +raw_atomic64_fetch_inc_relaxed(atomic64_t *v) { - s64 ret = arch_atomic64_fetch_inc_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic64_fetch_add_relaxed(1, v); } -#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire #endif -#ifndef arch_atomic64_fetch_inc_release -static __always_inline s64 -arch_atomic64_fetch_inc_release(atomic64_t *v) +#if defined(arch_atomic64_dec) +#define raw_atomic64_dec arch_atomic64_dec +#else +static __always_inline void +raw_atomic64_dec(atomic64_t *v) { - __atomic_release_fence(); - return arch_atomic64_fetch_inc_relaxed(v); + raw_atomic64_sub(1, v); } -#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release #endif -#ifndef arch_atomic64_fetch_inc +#if defined(arch_atomic64_dec_return) +#define raw_atomic64_dec_return arch_atomic64_dec_return +#elif defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 -arch_atomic64_fetch_inc(atomic64_t *v) +raw_atomic64_dec_return(atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_inc_relaxed(v); + ret = arch_atomic64_dec_return_relaxed(v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc -#endif - -#endif /* arch_atomic64_fetch_inc_relaxed */ - -#ifndef arch_atomic64_dec -static __always_inline void -arch_atomic64_dec(atomic64_t *v) -{ - arch_atomic64_sub(1, v); -} -#define arch_atomic64_dec arch_atomic64_dec -#endif - -#ifndef arch_atomic64_dec_return_relaxed -#ifdef arch_atomic64_dec_return -#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return -#define arch_atomic64_dec_return_release arch_atomic64_dec_return -#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return -#endif /* arch_atomic64_dec_return */ - -#ifndef arch_atomic64_dec_return +#else static __always_inline s64 -arch_atomic64_dec_return(atomic64_t *v) +raw_atomic64_dec_return(atomic64_t *v) { - return arch_atomic64_sub_return(1, v); + return raw_atomic64_sub_return(1, v); } -#define arch_atomic64_dec_return arch_atomic64_dec_return #endif -#ifndef arch_atomic64_dec_return_acquire +#if defined(arch_atomic64_dec_return_acquire) +#define raw_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire +#elif defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 -arch_atomic64_dec_return_acquire(atomic64_t *v) +raw_atomic64_dec_return_acquire(atomic64_t *v) { - return arch_atomic64_sub_return_acquire(1, v); + s64 ret = arch_atomic64_dec_return_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire -#endif - -#ifndef arch_atomic64_dec_return_release +#elif defined(arch_atomic64_dec_return) +#define raw_atomic64_dec_return_acquire arch_atomic64_dec_return +#else static __always_inline s64 -arch_atomic64_dec_return_release(atomic64_t *v) +raw_atomic64_dec_return_acquire(atomic64_t *v) { - return arch_atomic64_sub_return_release(1, v); + return raw_atomic64_sub_return_acquire(1, v); } -#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release #endif -#ifndef arch_atomic64_dec_return_relaxed +#if defined(arch_atomic64_dec_return_release) +#define raw_atomic64_dec_return_release arch_atomic64_dec_return_release +#elif defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 -arch_atomic64_dec_return_relaxed(atomic64_t *v) +raw_atomic64_dec_return_release(atomic64_t *v) { - return arch_atomic64_sub_return_relaxed(1, v); + __atomic_release_fence(); + return arch_atomic64_dec_return_relaxed(v); } -#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed -#endif - -#else /* arch_atomic64_dec_return_relaxed */ - -#ifndef arch_atomic64_dec_return_acquire +#elif defined(arch_atomic64_dec_return) +#define raw_atomic64_dec_return_release arch_atomic64_dec_return +#else static __always_inline s64 -arch_atomic64_dec_return_acquire(atomic64_t *v) +raw_atomic64_dec_return_release(atomic64_t *v) { - s64 ret = arch_atomic64_dec_return_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic64_sub_return_release(1, v); } -#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire #endif -#ifndef arch_atomic64_dec_return_release +#if defined(arch_atomic64_dec_return_relaxed) +#define raw_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed +#elif defined(arch_atomic64_dec_return) +#define raw_atomic64_dec_return_relaxed arch_atomic64_dec_return +#else static __always_inline s64 -arch_atomic64_dec_return_release(atomic64_t *v) +raw_atomic64_dec_return_relaxed(atomic64_t *v) { - __atomic_release_fence(); - return arch_atomic64_dec_return_relaxed(v); + return raw_atomic64_sub_return_relaxed(1, v); } -#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release #endif -#ifndef arch_atomic64_dec_return +#if defined(arch_atomic64_fetch_dec) +#define raw_atomic64_fetch_dec arch_atomic64_fetch_dec +#elif defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 -arch_atomic64_dec_return(atomic64_t *v) +raw_atomic64_fetch_dec(atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_dec_return_relaxed(v); + ret = arch_atomic64_fetch_dec_relaxed(v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_dec_return arch_atomic64_dec_return -#endif - -#endif /* arch_atomic64_dec_return_relaxed */ - -#ifndef arch_atomic64_fetch_dec_relaxed -#ifdef arch_atomic64_fetch_dec -#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec -#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec -#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec -#endif /* arch_atomic64_fetch_dec */ - -#ifndef arch_atomic64_fetch_dec +#else static __always_inline s64 -arch_atomic64_fetch_dec(atomic64_t *v) +raw_atomic64_fetch_dec(atomic64_t *v) { - return arch_atomic64_fetch_sub(1, v); + return raw_atomic64_fetch_sub(1, v); } -#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec #endif -#ifndef arch_atomic64_fetch_dec_acquire +#if defined(arch_atomic64_fetch_dec_acquire) +#define raw_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire +#elif defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 -arch_atomic64_fetch_dec_acquire(atomic64_t *v) +raw_atomic64_fetch_dec_acquire(atomic64_t *v) { - return arch_atomic64_fetch_sub_acquire(1, v); + s64 ret = arch_atomic64_fetch_dec_relaxed(v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire -#endif - -#ifndef arch_atomic64_fetch_dec_release +#elif defined(arch_atomic64_fetch_dec) +#define raw_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec +#else static __always_inline s64 -arch_atomic64_fetch_dec_release(atomic64_t *v) +raw_atomic64_fetch_dec_acquire(atomic64_t *v) { - return arch_atomic64_fetch_sub_release(1, v); + return raw_atomic64_fetch_sub_acquire(1, v); } -#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release #endif -#ifndef arch_atomic64_fetch_dec_relaxed +#if defined(arch_atomic64_fetch_dec_release) +#define raw_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release +#elif defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 -arch_atomic64_fetch_dec_relaxed(atomic64_t *v) +raw_atomic64_fetch_dec_release(atomic64_t *v) { - return arch_atomic64_fetch_sub_relaxed(1, v); + __atomic_release_fence(); + return arch_atomic64_fetch_dec_relaxed(v); } -#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed -#endif - -#else /* arch_atomic64_fetch_dec_relaxed */ - -#ifndef arch_atomic64_fetch_dec_acquire +#elif defined(arch_atomic64_fetch_dec) +#define raw_atomic64_fetch_dec_release arch_atomic64_fetch_dec +#else static __always_inline s64 -arch_atomic64_fetch_dec_acquire(atomic64_t *v) +raw_atomic64_fetch_dec_release(atomic64_t *v) { - s64 ret = arch_atomic64_fetch_dec_relaxed(v); - __atomic_acquire_fence(); - return ret; + return raw_atomic64_fetch_sub_release(1, v); } -#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire #endif -#ifndef arch_atomic64_fetch_dec_release +#if defined(arch_atomic64_fetch_dec_relaxed) +#define raw_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed +#elif defined(arch_atomic64_fetch_dec) +#define raw_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec +#else static __always_inline s64 -arch_atomic64_fetch_dec_release(atomic64_t *v) +raw_atomic64_fetch_dec_relaxed(atomic64_t *v) { - __atomic_release_fence(); - return arch_atomic64_fetch_dec_relaxed(v); + return raw_atomic64_fetch_sub_relaxed(1, v); } -#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release #endif -#ifndef arch_atomic64_fetch_dec +#define raw_atomic64_and arch_atomic64_and + +#if defined(arch_atomic64_fetch_and) +#define raw_atomic64_fetch_and arch_atomic64_fetch_and +#elif defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 -arch_atomic64_fetch_dec(atomic64_t *v) +raw_atomic64_fetch_and(s64 i, atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_dec_relaxed(v); + ret = arch_atomic64_fetch_and_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec +#else +#error "Unable to define raw_atomic64_fetch_and" #endif -#endif /* arch_atomic64_fetch_dec_relaxed */ - -#ifndef arch_atomic64_fetch_and_relaxed -#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and -#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and -#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and -#else /* arch_atomic64_fetch_and_relaxed */ - -#ifndef arch_atomic64_fetch_and_acquire +#if defined(arch_atomic64_fetch_and_acquire) +#define raw_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire +#elif defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 -arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_fetch_and_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire +#elif defined(arch_atomic64_fetch_and) +#define raw_atomic64_fetch_and_acquire arch_atomic64_fetch_and +#else +#error "Unable to define raw_atomic64_fetch_and_acquire" #endif -#ifndef arch_atomic64_fetch_and_release +#if defined(arch_atomic64_fetch_and_release) +#define raw_atomic64_fetch_and_release arch_atomic64_fetch_and_release +#elif defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 -arch_atomic64_fetch_and_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_fetch_and_relaxed(i, v); } -#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release +#elif defined(arch_atomic64_fetch_and) +#define raw_atomic64_fetch_and_release arch_atomic64_fetch_and +#else +#error "Unable to define raw_atomic64_fetch_and_release" #endif -#ifndef arch_atomic64_fetch_and -static __always_inline s64 -arch_atomic64_fetch_and(s64 i, atomic64_t *v) -{ - s64 ret; - __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_and_relaxed(i, v); - __atomic_post_full_fence(); - return ret; -} -#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#if defined(arch_atomic64_fetch_and_relaxed) +#define raw_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#elif defined(arch_atomic64_fetch_and) +#define raw_atomic64_fetch_and_relaxed arch_atomic64_fetch_and +#else +#error "Unable to define raw_atomic64_fetch_and_relaxed" #endif -#endif /* arch_atomic64_fetch_and_relaxed */ - -#ifndef arch_atomic64_andnot +#if defined(arch_atomic64_andnot) +#define raw_atomic64_andnot arch_atomic64_andnot +#else static __always_inline void -arch_atomic64_andnot(s64 i, atomic64_t *v) +raw_atomic64_andnot(s64 i, atomic64_t *v) { - arch_atomic64_and(~i, v); + raw_atomic64_and(~i, v); } -#define arch_atomic64_andnot arch_atomic64_andnot #endif -#ifndef arch_atomic64_fetch_andnot_relaxed -#ifdef arch_atomic64_fetch_andnot -#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot -#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot -#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot -#endif /* arch_atomic64_fetch_andnot */ - -#ifndef arch_atomic64_fetch_andnot +#if defined(arch_atomic64_fetch_andnot) +#define raw_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#elif defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 -arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) +raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) { - return arch_atomic64_fetch_and(~i, v); + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_andnot_relaxed(i, v); + __atomic_post_full_fence(); + return ret; } -#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot -#endif - -#ifndef arch_atomic64_fetch_andnot_acquire +#else static __always_inline s64 -arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) { - return arch_atomic64_fetch_and_acquire(~i, v); + return raw_atomic64_fetch_and(~i, v); } -#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire #endif -#ifndef arch_atomic64_fetch_andnot_release +#if defined(arch_atomic64_fetch_andnot_acquire) +#define raw_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire +#elif defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 -arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { - return arch_atomic64_fetch_and_release(~i, v); + s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release -#endif - -#ifndef arch_atomic64_fetch_andnot_relaxed +#elif defined(arch_atomic64_fetch_andnot) +#define raw_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot +#else static __always_inline s64 -arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) +raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { - return arch_atomic64_fetch_and_relaxed(~i, v); + return raw_atomic64_fetch_and_acquire(~i, v); } -#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed #endif -#else /* arch_atomic64_fetch_andnot_relaxed */ - -#ifndef arch_atomic64_fetch_andnot_acquire +#if defined(arch_atomic64_fetch_andnot_release) +#define raw_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release +#elif defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 -arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { - s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); - __atomic_acquire_fence(); - return ret; + __atomic_release_fence(); + return arch_atomic64_fetch_andnot_relaxed(i, v); +} +#elif defined(arch_atomic64_fetch_andnot) +#define raw_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot +#else +static __always_inline s64 +raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + return raw_atomic64_fetch_and_release(~i, v); } -#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire #endif -#ifndef arch_atomic64_fetch_andnot_release +#if defined(arch_atomic64_fetch_andnot_relaxed) +#define raw_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#elif defined(arch_atomic64_fetch_andnot) +#define raw_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot +#else static __always_inline s64 -arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { - __atomic_release_fence(); - return arch_atomic64_fetch_andnot_relaxed(i, v); + return raw_atomic64_fetch_and_relaxed(~i, v); } -#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release #endif -#ifndef arch_atomic64_fetch_andnot +#define raw_atomic64_or arch_atomic64_or + +#if defined(arch_atomic64_fetch_or) +#define raw_atomic64_fetch_or arch_atomic64_fetch_or +#elif defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 -arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) +raw_atomic64_fetch_or(s64 i, atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_andnot_relaxed(i, v); + ret = arch_atomic64_fetch_or_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#else +#error "Unable to define raw_atomic64_fetch_or" #endif -#endif /* arch_atomic64_fetch_andnot_relaxed */ - -#ifndef arch_atomic64_fetch_or_relaxed -#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or -#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or -#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or -#else /* arch_atomic64_fetch_or_relaxed */ - -#ifndef arch_atomic64_fetch_or_acquire +#if defined(arch_atomic64_fetch_or_acquire) +#define raw_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire +#elif defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 -arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_fetch_or_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire +#elif defined(arch_atomic64_fetch_or) +#define raw_atomic64_fetch_or_acquire arch_atomic64_fetch_or +#else +#error "Unable to define raw_atomic64_fetch_or_acquire" #endif -#ifndef arch_atomic64_fetch_or_release +#if defined(arch_atomic64_fetch_or_release) +#define raw_atomic64_fetch_or_release arch_atomic64_fetch_or_release +#elif defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 -arch_atomic64_fetch_or_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_fetch_or_relaxed(i, v); } -#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release +#elif defined(arch_atomic64_fetch_or) +#define raw_atomic64_fetch_or_release arch_atomic64_fetch_or +#else +#error "Unable to define raw_atomic64_fetch_or_release" +#endif + +#if defined(arch_atomic64_fetch_or_relaxed) +#define raw_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#elif defined(arch_atomic64_fetch_or) +#define raw_atomic64_fetch_or_relaxed arch_atomic64_fetch_or +#else +#error "Unable to define raw_atomic64_fetch_or_relaxed" #endif -#ifndef arch_atomic64_fetch_or +#define raw_atomic64_xor arch_atomic64_xor + +#if defined(arch_atomic64_fetch_xor) +#define raw_atomic64_fetch_xor arch_atomic64_fetch_xor +#elif defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 -arch_atomic64_fetch_or(s64 i, atomic64_t *v) +raw_atomic64_fetch_xor(s64 i, atomic64_t *v) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_or_relaxed(i, v); + ret = arch_atomic64_fetch_xor_relaxed(i, v); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#else +#error "Unable to define raw_atomic64_fetch_xor" #endif -#endif /* arch_atomic64_fetch_or_relaxed */ - -#ifndef arch_atomic64_fetch_xor_relaxed -#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor -#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor -#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor -#else /* arch_atomic64_fetch_xor_relaxed */ - -#ifndef arch_atomic64_fetch_xor_acquire +#if defined(arch_atomic64_fetch_xor_acquire) +#define raw_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire +#elif defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 -arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) +raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); __atomic_acquire_fence(); return ret; } -#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire +#elif defined(arch_atomic64_fetch_xor) +#define raw_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor +#else +#error "Unable to define raw_atomic64_fetch_xor_acquire" #endif -#ifndef arch_atomic64_fetch_xor_release +#if defined(arch_atomic64_fetch_xor_release) +#define raw_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release +#elif defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 -arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v) +raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) { __atomic_release_fence(); return arch_atomic64_fetch_xor_relaxed(i, v); } -#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release +#elif defined(arch_atomic64_fetch_xor) +#define raw_atomic64_fetch_xor_release arch_atomic64_fetch_xor +#else +#error "Unable to define raw_atomic64_fetch_xor_release" +#endif + +#if defined(arch_atomic64_fetch_xor_relaxed) +#define raw_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed +#elif defined(arch_atomic64_fetch_xor) +#define raw_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor +#else +#error "Unable to define raw_atomic64_fetch_xor_relaxed" #endif -#ifndef arch_atomic64_fetch_xor +#if defined(arch_atomic64_xchg) +#define raw_atomic64_xchg arch_atomic64_xchg +#elif defined(arch_atomic64_xchg_relaxed) static __always_inline s64 -arch_atomic64_fetch_xor(s64 i, atomic64_t *v) +raw_atomic64_xchg(atomic64_t *v, s64 i) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_fetch_xor_relaxed(i, v); + ret = arch_atomic64_xchg_relaxed(v, i); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor -#endif - -#endif /* arch_atomic64_fetch_xor_relaxed */ - -#ifndef arch_atomic64_xchg_relaxed -#ifdef arch_atomic64_xchg -#define arch_atomic64_xchg_acquire arch_atomic64_xchg -#define arch_atomic64_xchg_release arch_atomic64_xchg -#define arch_atomic64_xchg_relaxed arch_atomic64_xchg -#endif /* arch_atomic64_xchg */ - -#ifndef arch_atomic64_xchg +#else static __always_inline s64 -arch_atomic64_xchg(atomic64_t *v, s64 new) +raw_atomic64_xchg(atomic64_t *v, s64 new) { - return arch_xchg(&v->counter, new); + return raw_xchg(&v->counter, new); } -#define arch_atomic64_xchg arch_atomic64_xchg #endif -#ifndef arch_atomic64_xchg_acquire +#if defined(arch_atomic64_xchg_acquire) +#define raw_atomic64_xchg_acquire arch_atomic64_xchg_acquire +#elif defined(arch_atomic64_xchg_relaxed) static __always_inline s64 -arch_atomic64_xchg_acquire(atomic64_t *v, s64 new) +raw_atomic64_xchg_acquire(atomic64_t *v, s64 i) { - return arch_xchg_acquire(&v->counter, new); + s64 ret = arch_atomic64_xchg_relaxed(v, i); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire -#endif - -#ifndef arch_atomic64_xchg_release +#elif defined(arch_atomic64_xchg) +#define raw_atomic64_xchg_acquire arch_atomic64_xchg +#else static __always_inline s64 -arch_atomic64_xchg_release(atomic64_t *v, s64 new) +raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) { - return arch_xchg_release(&v->counter, new); + return raw_xchg_acquire(&v->counter, new); } -#define arch_atomic64_xchg_release arch_atomic64_xchg_release #endif -#ifndef arch_atomic64_xchg_relaxed +#if defined(arch_atomic64_xchg_release) +#define raw_atomic64_xchg_release arch_atomic64_xchg_release +#elif defined(arch_atomic64_xchg_relaxed) static __always_inline s64 -arch_atomic64_xchg_relaxed(atomic64_t *v, s64 new) +raw_atomic64_xchg_release(atomic64_t *v, s64 i) { - return arch_xchg_relaxed(&v->counter, new); + __atomic_release_fence(); + return arch_atomic64_xchg_relaxed(v, i); } -#define arch_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed -#endif - -#else /* arch_atomic64_xchg_relaxed */ - -#ifndef arch_atomic64_xchg_acquire +#elif defined(arch_atomic64_xchg) +#define raw_atomic64_xchg_release arch_atomic64_xchg +#else static __always_inline s64 -arch_atomic64_xchg_acquire(atomic64_t *v, s64 i) +raw_atomic64_xchg_release(atomic64_t *v, s64 new) { - s64 ret = arch_atomic64_xchg_relaxed(v, i); - __atomic_acquire_fence(); - return ret; + return raw_xchg_release(&v->counter, new); } -#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire #endif -#ifndef arch_atomic64_xchg_release +#if defined(arch_atomic64_xchg_relaxed) +#define raw_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed +#elif defined(arch_atomic64_xchg) +#define raw_atomic64_xchg_relaxed arch_atomic64_xchg +#else static __always_inline s64 -arch_atomic64_xchg_release(atomic64_t *v, s64 i) +raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) { - __atomic_release_fence(); - return arch_atomic64_xchg_relaxed(v, i); + return raw_xchg_relaxed(&v->counter, new); } -#define arch_atomic64_xchg_release arch_atomic64_xchg_release #endif -#ifndef arch_atomic64_xchg +#if defined(arch_atomic64_cmpxchg) +#define raw_atomic64_cmpxchg arch_atomic64_cmpxchg +#elif defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 -arch_atomic64_xchg(atomic64_t *v, s64 i) +raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_xchg_relaxed(v, i); + ret = arch_atomic64_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_xchg arch_atomic64_xchg -#endif - -#endif /* arch_atomic64_xchg_relaxed */ - -#ifndef arch_atomic64_cmpxchg_relaxed -#ifdef arch_atomic64_cmpxchg -#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg -#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg -#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg -#endif /* arch_atomic64_cmpxchg */ - -#ifndef arch_atomic64_cmpxchg +#else static __always_inline s64 -arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { - return arch_cmpxchg(&v->counter, old, new); + return raw_cmpxchg(&v->counter, old, new); } -#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg #endif -#ifndef arch_atomic64_cmpxchg_acquire +#if defined(arch_atomic64_cmpxchg_acquire) +#define raw_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire +#elif defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 -arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { - return arch_cmpxchg_acquire(&v->counter, old, new); + s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire -#endif - -#ifndef arch_atomic64_cmpxchg_release +#elif defined(arch_atomic64_cmpxchg) +#define raw_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg +#else static __always_inline s64 -arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { - return arch_cmpxchg_release(&v->counter, old, new); + return raw_cmpxchg_acquire(&v->counter, old, new); } -#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release #endif -#ifndef arch_atomic64_cmpxchg_relaxed +#if defined(arch_atomic64_cmpxchg_release) +#define raw_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release +#elif defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 -arch_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { - return arch_cmpxchg_relaxed(&v->counter, old, new); + __atomic_release_fence(); + return arch_atomic64_cmpxchg_relaxed(v, old, new); } -#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed -#endif - -#else /* arch_atomic64_cmpxchg_relaxed */ - -#ifndef arch_atomic64_cmpxchg_acquire +#elif defined(arch_atomic64_cmpxchg) +#define raw_atomic64_cmpxchg_release arch_atomic64_cmpxchg +#else static __always_inline s64 -arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { - s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; + return raw_cmpxchg_release(&v->counter, old, new); } -#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire #endif -#ifndef arch_atomic64_cmpxchg_release +#if defined(arch_atomic64_cmpxchg_relaxed) +#define raw_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed +#elif defined(arch_atomic64_cmpxchg) +#define raw_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg +#else static __always_inline s64 -arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { - __atomic_release_fence(); - return arch_atomic64_cmpxchg_relaxed(v, old, new); + return raw_cmpxchg_relaxed(&v->counter, old, new); } -#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release #endif -#ifndef arch_atomic64_cmpxchg -static __always_inline s64 -arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +#if defined(arch_atomic64_try_cmpxchg) +#define raw_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg +#elif defined(arch_atomic64_try_cmpxchg_relaxed) +static __always_inline bool +raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { - s64 ret; + bool ret; __atomic_pre_full_fence(); - ret = arch_atomic64_cmpxchg_relaxed(v, old, new); + ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; } -#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg -#endif - -#endif /* arch_atomic64_cmpxchg_relaxed */ - -#ifndef arch_atomic64_try_cmpxchg_relaxed -#ifdef arch_atomic64_try_cmpxchg -#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg -#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg -#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg -#endif /* arch_atomic64_try_cmpxchg */ - -#ifndef arch_atomic64_try_cmpxchg +#else static __always_inline bool -arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; - r = arch_atomic64_cmpxchg(v, o, new); + r = raw_atomic64_cmpxchg(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg #endif -#ifndef arch_atomic64_try_cmpxchg_acquire +#if defined(arch_atomic64_try_cmpxchg_acquire) +#define raw_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire +#elif defined(arch_atomic64_try_cmpxchg_relaxed) +static __always_inline bool +raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; +} +#elif defined(arch_atomic64_try_cmpxchg) +#define raw_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg +#else static __always_inline bool -arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; - r = arch_atomic64_cmpxchg_acquire(v, o, new); + r = raw_atomic64_cmpxchg_acquire(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire #endif -#ifndef arch_atomic64_try_cmpxchg_release +#if defined(arch_atomic64_try_cmpxchg_release) +#define raw_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release +#elif defined(arch_atomic64_try_cmpxchg_relaxed) +static __always_inline bool +raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + __atomic_release_fence(); + return arch_atomic64_try_cmpxchg_relaxed(v, old, new); +} +#elif defined(arch_atomic64_try_cmpxchg) +#define raw_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg +#else static __always_inline bool -arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; - r = arch_atomic64_cmpxchg_release(v, o, new); + r = raw_atomic64_cmpxchg_release(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release #endif -#ifndef arch_atomic64_try_cmpxchg_relaxed +#if defined(arch_atomic64_try_cmpxchg_relaxed) +#define raw_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed +#elif defined(arch_atomic64_try_cmpxchg) +#define raw_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg +#else static __always_inline bool -arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) +raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; - r = arch_atomic64_cmpxchg_relaxed(v, o, new); + r = raw_atomic64_cmpxchg_relaxed(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); } -#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed #endif -#else /* arch_atomic64_try_cmpxchg_relaxed */ - -#ifndef arch_atomic64_try_cmpxchg_acquire -static __always_inline bool -arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) -{ - bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); - __atomic_acquire_fence(); - return ret; -} -#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire -#endif - -#ifndef arch_atomic64_try_cmpxchg_release -static __always_inline bool -arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) -{ - __atomic_release_fence(); - return arch_atomic64_try_cmpxchg_relaxed(v, old, new); -} -#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release -#endif - -#ifndef arch_atomic64_try_cmpxchg -static __always_inline bool -arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) -{ - bool ret; - __atomic_pre_full_fence(); - ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); - __atomic_post_full_fence(); - return ret; -} -#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg -#endif - -#endif /* arch_atomic64_try_cmpxchg_relaxed */ - -#ifndef arch_atomic64_sub_and_test +#if defined(arch_atomic64_sub_and_test) +#define raw_atomic64_sub_and_test arch_atomic64_sub_and_test +#else static __always_inline bool -arch_atomic64_sub_and_test(s64 i, atomic64_t *v) +raw_atomic64_sub_and_test(s64 i, atomic64_t *v) { - return arch_atomic64_sub_return(i, v) == 0; + return raw_atomic64_sub_return(i, v) == 0; } -#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test #endif -#ifndef arch_atomic64_dec_and_test +#if defined(arch_atomic64_dec_and_test) +#define raw_atomic64_dec_and_test arch_atomic64_dec_and_test +#else static __always_inline bool -arch_atomic64_dec_and_test(atomic64_t *v) +raw_atomic64_dec_and_test(atomic64_t *v) { - return arch_atomic64_dec_return(v) == 0; + return raw_atomic64_dec_return(v) == 0; } -#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test #endif -#ifndef arch_atomic64_inc_and_test +#if defined(arch_atomic64_inc_and_test) +#define raw_atomic64_inc_and_test arch_atomic64_inc_and_test +#else static __always_inline bool -arch_atomic64_inc_and_test(atomic64_t *v) +raw_atomic64_inc_and_test(atomic64_t *v) { - return arch_atomic64_inc_return(v) == 0; + return raw_atomic64_inc_return(v) == 0; } -#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test #endif -#ifndef arch_atomic64_add_negative_relaxed -#ifdef arch_atomic64_add_negative -#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative -#define arch_atomic64_add_negative_release arch_atomic64_add_negative -#define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative -#endif /* arch_atomic64_add_negative */ - -#ifndef arch_atomic64_add_negative +#if defined(arch_atomic64_add_negative) +#define raw_atomic64_add_negative arch_atomic64_add_negative +#elif defined(arch_atomic64_add_negative_relaxed) static __always_inline bool -arch_atomic64_add_negative(s64 i, atomic64_t *v) +raw_atomic64_add_negative(s64 i, atomic64_t *v) { - return arch_atomic64_add_return(i, v) < 0; + bool ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_add_negative_relaxed(i, v); + __atomic_post_full_fence(); + return ret; } -#define arch_atomic64_add_negative arch_atomic64_add_negative -#endif - -#ifndef arch_atomic64_add_negative_acquire +#else static __always_inline bool -arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) +raw_atomic64_add_negative(s64 i, atomic64_t *v) { - return arch_atomic64_add_return_acquire(i, v) < 0; + return raw_atomic64_add_return(i, v) < 0; } -#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire #endif -#ifndef arch_atomic64_add_negative_release +#if defined(arch_atomic64_add_negative_acquire) +#define raw_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire +#elif defined(arch_atomic64_add_negative_relaxed) static __always_inline bool -arch_atomic64_add_negative_release(s64 i, atomic64_t *v) +raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) { - return arch_atomic64_add_return_release(i, v) < 0; + bool ret = arch_atomic64_add_negative_relaxed(i, v); + __atomic_acquire_fence(); + return ret; } -#define arch_atomic64_add_negative_release arch_atomic64_add_negative_release -#endif - -#ifndef arch_atomic64_add_negative_relaxed +#elif defined(arch_atomic64_add_negative) +#define raw_atomic64_add_negative_acquire arch_atomic64_add_negative +#else static __always_inline bool -arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) +raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) { - return arch_atomic64_add_return_relaxed(i, v) < 0; + return raw_atomic64_add_return_acquire(i, v) < 0; } -#define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed #endif -#else /* arch_atomic64_add_negative_relaxed */ - -#ifndef arch_atomic64_add_negative_acquire +#if defined(arch_atomic64_add_negative_release) +#define raw_atomic64_add_negative_release arch_atomic64_add_negative_release +#elif defined(arch_atomic64_add_negative_relaxed) static __always_inline bool -arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) +raw_atomic64_add_negative_release(s64 i, atomic64_t *v) { - bool ret = arch_atomic64_add_negative_relaxed(i, v); - __atomic_acquire_fence(); - return ret; + __atomic_release_fence(); + return arch_atomic64_add_negative_relaxed(i, v); } -#define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire -#endif - -#ifndef arch_atomic64_add_negative_release +#elif defined(arch_atomic64_add_negative) +#define raw_atomic64_add_negative_release arch_atomic64_add_negative +#else static __always_inline bool -arch_atomic64_add_negative_release(s64 i, atomic64_t *v) +raw_atomic64_add_negative_release(s64 i, atomic64_t *v) { - __atomic_release_fence(); - return arch_atomic64_add_negative_relaxed(i, v); + return raw_atomic64_add_return_release(i, v) < 0; } -#define arch_atomic64_add_negative_release arch_atomic64_add_negative_release #endif -#ifndef arch_atomic64_add_negative +#if defined(arch_atomic64_add_negative_relaxed) +#define raw_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed +#elif defined(arch_atomic64_add_negative) +#define raw_atomic64_add_negative_relaxed arch_atomic64_add_negative +#else static __always_inline bool -arch_atomic64_add_negative(s64 i, atomic64_t *v) +raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) { - bool ret; - __atomic_pre_full_fence(); - ret = arch_atomic64_add_negative_relaxed(i, v); - __atomic_post_full_fence(); - return ret; + return raw_atomic64_add_return_relaxed(i, v) < 0; } -#define arch_atomic64_add_negative arch_atomic64_add_negative #endif -#endif /* arch_atomic64_add_negative_relaxed */ - -#ifndef arch_atomic64_fetch_add_unless +#if defined(arch_atomic64_fetch_add_unless) +#define raw_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless +#else static __always_inline s64 -arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { - s64 c = arch_atomic64_read(v); + s64 c = raw_atomic64_read(v); do { if (unlikely(c == u)) break; - } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); + } while (!raw_atomic64_try_cmpxchg(v, &c, c + a)); return c; } -#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless #endif -#ifndef arch_atomic64_add_unless +#if defined(arch_atomic64_add_unless) +#define raw_atomic64_add_unless arch_atomic64_add_unless +#else static __always_inline bool -arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) +raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { - return arch_atomic64_fetch_add_unless(v, a, u) != u; + return raw_atomic64_fetch_add_unless(v, a, u) != u; } -#define arch_atomic64_add_unless arch_atomic64_add_unless #endif -#ifndef arch_atomic64_inc_not_zero +#if defined(arch_atomic64_inc_not_zero) +#define raw_atomic64_inc_not_zero arch_atomic64_inc_not_zero +#else static __always_inline bool -arch_atomic64_inc_not_zero(atomic64_t *v) +raw_atomic64_inc_not_zero(atomic64_t *v) { - return arch_atomic64_add_unless(v, 1, 0); + return raw_atomic64_add_unless(v, 1, 0); } -#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero #endif -#ifndef arch_atomic64_inc_unless_negative +#if defined(arch_atomic64_inc_unless_negative) +#define raw_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative +#else static __always_inline bool -arch_atomic64_inc_unless_negative(atomic64_t *v) +raw_atomic64_inc_unless_negative(atomic64_t *v) { - s64 c = arch_atomic64_read(v); + s64 c = raw_atomic64_read(v); do { if (unlikely(c < 0)) return false; - } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1)); + } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1)); return true; } -#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative #endif -#ifndef arch_atomic64_dec_unless_positive +#if defined(arch_atomic64_dec_unless_positive) +#define raw_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive +#else static __always_inline bool -arch_atomic64_dec_unless_positive(atomic64_t *v) +raw_atomic64_dec_unless_positive(atomic64_t *v) { - s64 c = arch_atomic64_read(v); + s64 c = raw_atomic64_read(v); do { if (unlikely(c > 0)) return false; - } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1)); + } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1)); return true; } -#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive #endif -#ifndef arch_atomic64_dec_if_positive +#if defined(arch_atomic64_dec_if_positive) +#define raw_atomic64_dec_if_positive arch_atomic64_dec_if_positive +#else static __always_inline s64 -arch_atomic64_dec_if_positive(atomic64_t *v) +raw_atomic64_dec_if_positive(atomic64_t *v) { - s64 dec, c = arch_atomic64_read(v); + s64 dec, c = raw_atomic64_read(v); do { dec = c - 1; if (unlikely(dec < 0)) break; - } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); + } while (!raw_atomic64_try_cmpxchg(v, &c, dec)); return dec; } -#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive #endif #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// e1cee558cc61cae887890db30fcdf93baca9f498 +// c2048fccede6fac923252290e2b303949d5dec83 diff --git a/include/linux/atomic/atomic-raw.h b/include/linux/atomic/atomic-raw.h deleted file mode 100644 index 8b2fc04cf8c5..000000000000 --- a/include/linux/atomic/atomic-raw.h +++ /dev/null @@ -1,1135 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// Generated by scripts/atomic/gen-atomic-raw.sh -// DO NOT MODIFY THIS FILE DIRECTLY - -#ifndef _LINUX_ATOMIC_RAW_H -#define _LINUX_ATOMIC_RAW_H - -static __always_inline int -raw_atomic_read(const atomic_t *v) -{ - return arch_atomic_read(v); -} - -static __always_inline int -raw_atomic_read_acquire(const atomic_t *v) -{ - return arch_atomic_read_acquire(v); -} - -static __always_inline void -raw_atomic_set(atomic_t *v, int i) -{ - arch_atomic_set(v, i); -} - -static __always_inline void -raw_atomic_set_release(atomic_t *v, int i) -{ - arch_atomic_set_release(v, i); -} - -static __always_inline void -raw_atomic_add(int i, atomic_t *v) -{ - arch_atomic_add(i, v); -} - -static __always_inline int -raw_atomic_add_return(int i, atomic_t *v) -{ - return arch_atomic_add_return(i, v); -} - -static __always_inline int -raw_atomic_add_return_acquire(int i, atomic_t *v) -{ - return arch_atomic_add_return_acquire(i, v); -} - -static __always_inline int -raw_atomic_add_return_release(int i, atomic_t *v) -{ - return arch_atomic_add_return_release(i, v); -} - -static __always_inline int -raw_atomic_add_return_relaxed(int i, atomic_t *v) -{ - return arch_atomic_add_return_relaxed(i, v); -} - -static __always_inline int -raw_atomic_fetch_add(int i, atomic_t *v) -{ - return arch_atomic_fetch_add(i, v); -} - -static __always_inline int -raw_atomic_fetch_add_acquire(int i, atomic_t *v) -{ - return arch_atomic_fetch_add_acquire(i, v); -} - -static __always_inline int -raw_atomic_fetch_add_release(int i, atomic_t *v) -{ - return arch_atomic_fetch_add_release(i, v); -} - -static __always_inline int -raw_atomic_fetch_add_relaxed(int i, atomic_t *v) -{ - return arch_atomic_fetch_add_relaxed(i, v); -} - -static __always_inline void -raw_atomic_sub(int i, atomic_t *v) -{ - arch_atomic_sub(i, v); -} - -static __always_inline int -raw_atomic_sub_return(int i, atomic_t *v) -{ - return arch_atomic_sub_return(i, v); -} - -static __always_inline int -raw_atomic_sub_return_acquire(int i, atomic_t *v) -{ - return arch_atomic_sub_return_acquire(i, v); -} - -static __always_inline int -raw_atomic_sub_return_release(int i, atomic_t *v) -{ - return arch_atomic_sub_return_release(i, v); -} - -static __always_inline int -raw_atomic_sub_return_relaxed(int i, atomic_t *v) -{ - return arch_atomic_sub_return_relaxed(i, v); -} - -static __always_inline int -raw_atomic_fetch_sub(int i, atomic_t *v) -{ - return arch_atomic_fetch_sub(i, v); -} - -static __always_inline int -raw_atomic_fetch_sub_acquire(int i, atomic_t *v) -{ - return arch_atomic_fetch_sub_acquire(i, v); -} - -static __always_inline int -raw_atomic_fetch_sub_release(int i, atomic_t *v) -{ - return arch_atomic_fetch_sub_release(i, v); -} - -static __always_inline int -raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) -{ - return arch_atomic_fetch_sub_relaxed(i, v); -} - -static __always_inline void -raw_atomic_inc(atomic_t *v) -{ - arch_atomic_inc(v); -} - -static __always_inline int -raw_atomic_inc_return(atomic_t *v) -{ - return arch_atomic_inc_return(v); -} - -static __always_inline int -raw_atomic_inc_return_acquire(atomic_t *v) -{ - return arch_atomic_inc_return_acquire(v); -} - -static __always_inline int -raw_atomic_inc_return_release(atomic_t *v) -{ - return arch_atomic_inc_return_release(v); -} - -static __always_inline int -raw_atomic_inc_return_relaxed(atomic_t *v) -{ - return arch_atomic_inc_return_relaxed(v); -} - -static __always_inline int -raw_atomic_fetch_inc(atomic_t *v) -{ - return arch_atomic_fetch_inc(v); -} - -static __always_inline int -raw_atomic_fetch_inc_acquire(atomic_t *v) -{ - return arch_atomic_fetch_inc_acquire(v); -} - -static __always_inline int -raw_atomic_fetch_inc_release(atomic_t *v) -{ - return arch_atomic_fetch_inc_release(v); -} - -static __always_inline int -raw_atomic_fetch_inc_relaxed(atomic_t *v) -{ - return arch_atomic_fetch_inc_relaxed(v); -} - -static __always_inline void -raw_atomic_dec(atomic_t *v) -{ - arch_atomic_dec(v); -} - -static __always_inline int -raw_atomic_dec_return(atomic_t *v) -{ - return arch_atomic_dec_return(v); -} - -static __always_inline int -raw_atomic_dec_return_acquire(atomic_t *v) -{ - return arch_atomic_dec_return_acquire(v); -} - -static __always_inline int -raw_atomic_dec_return_release(atomic_t *v) -{ - return arch_atomic_dec_return_release(v); -} - -static __always_inline int -raw_atomic_dec_return_relaxed(atomic_t *v) -{ - return arch_atomic_dec_return_relaxed(v); -} - -static __always_inline int -raw_atomic_fetch_dec(atomic_t *v) -{ - return arch_atomic_fetch_dec(v); -} - -static __always_inline int -raw_atomic_fetch_dec_acquire(atomic_t *v) -{ - return arch_atomic_fetch_dec_acquire(v); -} - -static __always_inline int -raw_atomic_fetch_dec_release(atomic_t *v) -{ - return arch_atomic_fetch_dec_release(v); -} - -static __always_inline int -raw_atomic_fetch_dec_relaxed(atomic_t *v) -{ - return arch_atomic_fetch_dec_relaxed(v); -} - -static __always_inline void -raw_atomic_and(int i, atomic_t *v) -{ - arch_atomic_and(i, v); -} - -static __always_inline int -raw_atomic_fetch_and(int i, atomic_t *v) -{ - return arch_atomic_fetch_and(i, v); -} - -static __always_inline int -raw_atomic_fetch_and_acquire(int i, atomic_t *v) -{ - return arch_atomic_fetch_and_acquire(i, v); -} - -static __always_inline int -raw_atomic_fetch_and_release(int i, atomic_t *v) -{ - return arch_atomic_fetch_and_release(i, v); -} - -static __always_inline int -raw_atomic_fetch_and_relaxed(int i, atomic_t *v) -{ - return arch_atomic_fetch_and_relaxed(i, v); -} - -static __always_inline void -raw_atomic_andnot(int i, atomic_t *v) -{ - arch_atomic_andnot(i, v); -} - -static __always_inline int -raw_atomic_fetch_andnot(int i, atomic_t *v) -{ - return arch_atomic_fetch_andnot(i, v); -} - -static __always_inline int -raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) -{ - return arch_atomic_fetch_andnot_acquire(i, v); -} - -static __always_inline int -raw_atomic_fetch_andnot_release(int i, atomic_t *v) -{ - return arch_atomic_fetch_andnot_release(i, v); -} - -static __always_inline int -raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) -{ - return arch_atomic_fetch_andnot_relaxed(i, v); -} - -static __always_inline void -raw_atomic_or(int i, atomic_t *v) -{ - arch_atomic_or(i, v); -} - -static __always_inline int -raw_atomic_fetch_or(int i, atomic_t *v) -{ - return arch_atomic_fetch_or(i, v); -} - -static __always_inline int -raw_atomic_fetch_or_acquire(int i, atomic_t *v) -{ - return arch_atomic_fetch_or_acquire(i, v); -} - -static __always_inline int -raw_atomic_fetch_or_release(int i, atomic_t *v) -{ - return arch_atomic_fetch_or_release(i, v); -} - -static __always_inline int -raw_atomic_fetch_or_relaxed(int i, atomic_t *v) -{ - return arch_atomic_fetch_or_relaxed(i, v); -} - -static __always_inline void -raw_atomic_xor(int i, atomic_t *v) -{ - arch_atomic_xor(i, v); -} - -static __always_inline int -raw_atomic_fetch_xor(int i, atomic_t *v) -{ - return arch_atomic_fetch_xor(i, v); -} - -static __always_inline int -raw_atomic_fetch_xor_acquire(int i, atomic_t *v) -{ - return arch_atomic_fetch_xor_acquire(i, v); -} - -static __always_inline int -raw_atomic_fetch_xor_release(int i, atomic_t *v) -{ - return arch_atomic_fetch_xor_release(i, v); -} - -static __always_inline int -raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) -{ - return arch_atomic_fetch_xor_relaxed(i, v); -} - -static __always_inline int -raw_atomic_xchg(atomic_t *v, int i) -{ - return arch_atomic_xchg(v, i); -} - -static __always_inline int -raw_atomic_xchg_acquire(atomic_t *v, int i) -{ - return arch_atomic_xchg_acquire(v, i); -} - -static __always_inline int -raw_atomic_xchg_release(atomic_t *v, int i) -{ - return arch_atomic_xchg_release(v, i); -} - -static __always_inline int -raw_atomic_xchg_relaxed(atomic_t *v, int i) -{ - return arch_atomic_xchg_relaxed(v, i); -} - -static __always_inline int -raw_atomic_cmpxchg(atomic_t *v, int old, int new) -{ - return arch_atomic_cmpxchg(v, old, new); -} - -static __always_inline int -raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) -{ - return arch_atomic_cmpxchg_acquire(v, old, new); -} - -static __always_inline int -raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) -{ - return arch_atomic_cmpxchg_release(v, old, new); -} - -static __always_inline int -raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) -{ - return arch_atomic_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) -{ - return arch_atomic_try_cmpxchg(v, old, new); -} - -static __always_inline bool -raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) -{ - return arch_atomic_try_cmpxchg_acquire(v, old, new); -} - -static __always_inline bool -raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) -{ - return arch_atomic_try_cmpxchg_release(v, old, new); -} - -static __always_inline bool -raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) -{ - return arch_atomic_try_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic_sub_and_test(int i, atomic_t *v) -{ - return arch_atomic_sub_and_test(i, v); -} - -static __always_inline bool -raw_atomic_dec_and_test(atomic_t *v) -{ - return arch_atomic_dec_and_test(v); -} - -static __always_inline bool -raw_atomic_inc_and_test(atomic_t *v) -{ - return arch_atomic_inc_and_test(v); -} - -static __always_inline bool -raw_atomic_add_negative(int i, atomic_t *v) -{ - return arch_atomic_add_negative(i, v); -} - -static __always_inline bool -raw_atomic_add_negative_acquire(int i, atomic_t *v) -{ - return arch_atomic_add_negative_acquire(i, v); -} - -static __always_inline bool -raw_atomic_add_negative_release(int i, atomic_t *v) -{ - return arch_atomic_add_negative_release(i, v); -} - -static __always_inline bool -raw_atomic_add_negative_relaxed(int i, atomic_t *v) -{ - return arch_atomic_add_negative_relaxed(i, v); -} - -static __always_inline int -raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) -{ - return arch_atomic_fetch_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic_add_unless(atomic_t *v, int a, int u) -{ - return arch_atomic_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic_inc_not_zero(atomic_t *v) -{ - return arch_atomic_inc_not_zero(v); -} - -static __always_inline bool -raw_atomic_inc_unless_negative(atomic_t *v) -{ - return arch_atomic_inc_unless_negative(v); -} - -static __always_inline bool -raw_atomic_dec_unless_positive(atomic_t *v) -{ - return arch_atomic_dec_unless_positive(v); -} - -static __always_inline int -raw_atomic_dec_if_positive(atomic_t *v) -{ - return arch_atomic_dec_if_positive(v); -} - -static __always_inline s64 -raw_atomic64_read(const atomic64_t *v) -{ - return arch_atomic64_read(v); -} - -static __always_inline s64 -raw_atomic64_read_acquire(const atomic64_t *v) -{ - return arch_atomic64_read_acquire(v); -} - -static __always_inline void -raw_atomic64_set(atomic64_t *v, s64 i) -{ - arch_atomic64_set(v, i); -} - -static __always_inline void -raw_atomic64_set_release(atomic64_t *v, s64 i) -{ - arch_atomic64_set_release(v, i); -} - -static __always_inline void -raw_atomic64_add(s64 i, atomic64_t *v) -{ - arch_atomic64_add(i, v); -} - -static __always_inline s64 -raw_atomic64_add_return(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_return(i, v); -} - -static __always_inline s64 -raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_return_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_add_return_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_return_release(i, v); -} - -static __always_inline s64 -raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_return_relaxed(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_add(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_add(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_add_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_add_release(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_add_relaxed(i, v); -} - -static __always_inline void -raw_atomic64_sub(s64 i, atomic64_t *v) -{ - arch_atomic64_sub(i, v); -} - -static __always_inline s64 -raw_atomic64_sub_return(s64 i, atomic64_t *v) -{ - return arch_atomic64_sub_return(i, v); -} - -static __always_inline s64 -raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_sub_return_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_sub_return_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_sub_return_release(i, v); -} - -static __always_inline s64 -raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_sub_return_relaxed(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_sub(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_sub(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_sub_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_sub_release(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_sub_relaxed(i, v); -} - -static __always_inline void -raw_atomic64_inc(atomic64_t *v) -{ - arch_atomic64_inc(v); -} - -static __always_inline s64 -raw_atomic64_inc_return(atomic64_t *v) -{ - return arch_atomic64_inc_return(v); -} - -static __always_inline s64 -raw_atomic64_inc_return_acquire(atomic64_t *v) -{ - return arch_atomic64_inc_return_acquire(v); -} - -static __always_inline s64 -raw_atomic64_inc_return_release(atomic64_t *v) -{ - return arch_atomic64_inc_return_release(v); -} - -static __always_inline s64 -raw_atomic64_inc_return_relaxed(atomic64_t *v) -{ - return arch_atomic64_inc_return_relaxed(v); -} - -static __always_inline s64 -raw_atomic64_fetch_inc(atomic64_t *v) -{ - return arch_atomic64_fetch_inc(v); -} - -static __always_inline s64 -raw_atomic64_fetch_inc_acquire(atomic64_t *v) -{ - return arch_atomic64_fetch_inc_acquire(v); -} - -static __always_inline s64 -raw_atomic64_fetch_inc_release(atomic64_t *v) -{ - return arch_atomic64_fetch_inc_release(v); -} - -static __always_inline s64 -raw_atomic64_fetch_inc_relaxed(atomic64_t *v) -{ - return arch_atomic64_fetch_inc_relaxed(v); -} - -static __always_inline void -raw_atomic64_dec(atomic64_t *v) -{ - arch_atomic64_dec(v); -} - -static __always_inline s64 -raw_atomic64_dec_return(atomic64_t *v) -{ - return arch_atomic64_dec_return(v); -} - -static __always_inline s64 -raw_atomic64_dec_return_acquire(atomic64_t *v) -{ - return arch_atomic64_dec_return_acquire(v); -} - -static __always_inline s64 -raw_atomic64_dec_return_release(atomic64_t *v) -{ - return arch_atomic64_dec_return_release(v); -} - -static __always_inline s64 -raw_atomic64_dec_return_relaxed(atomic64_t *v) -{ - return arch_atomic64_dec_return_relaxed(v); -} - -static __always_inline s64 -raw_atomic64_fetch_dec(atomic64_t *v) -{ - return arch_atomic64_fetch_dec(v); -} - -static __always_inline s64 -raw_atomic64_fetch_dec_acquire(atomic64_t *v) -{ - return arch_atomic64_fetch_dec_acquire(v); -} - -static __always_inline s64 -raw_atomic64_fetch_dec_release(atomic64_t *v) -{ - return arch_atomic64_fetch_dec_release(v); -} - -static __always_inline s64 -raw_atomic64_fetch_dec_relaxed(atomic64_t *v) -{ - return arch_atomic64_fetch_dec_relaxed(v); -} - -static __always_inline void -raw_atomic64_and(s64 i, atomic64_t *v) -{ - arch_atomic64_and(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_and(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_and(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_and_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_and_release(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_and_relaxed(i, v); -} - -static __always_inline void -raw_atomic64_andnot(s64 i, atomic64_t *v) -{ - arch_atomic64_andnot(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_andnot(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_andnot_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_andnot_release(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_andnot_relaxed(i, v); -} - -static __always_inline void -raw_atomic64_or(s64 i, atomic64_t *v) -{ - arch_atomic64_or(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_or(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_or(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_or_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_or_release(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_or_relaxed(i, v); -} - -static __always_inline void -raw_atomic64_xor(s64 i, atomic64_t *v) -{ - arch_atomic64_xor(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_xor(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_xor(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_xor_acquire(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_xor_release(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_fetch_xor_relaxed(i, v); -} - -static __always_inline s64 -raw_atomic64_xchg(atomic64_t *v, s64 i) -{ - return arch_atomic64_xchg(v, i); -} - -static __always_inline s64 -raw_atomic64_xchg_acquire(atomic64_t *v, s64 i) -{ - return arch_atomic64_xchg_acquire(v, i); -} - -static __always_inline s64 -raw_atomic64_xchg_release(atomic64_t *v, s64 i) -{ - return arch_atomic64_xchg_release(v, i); -} - -static __always_inline s64 -raw_atomic64_xchg_relaxed(atomic64_t *v, s64 i) -{ - return arch_atomic64_xchg_relaxed(v, i); -} - -static __always_inline s64 -raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) -{ - return arch_atomic64_cmpxchg(v, old, new); -} - -static __always_inline s64 -raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) -{ - return arch_atomic64_cmpxchg_acquire(v, old, new); -} - -static __always_inline s64 -raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) -{ - return arch_atomic64_cmpxchg_release(v, old, new); -} - -static __always_inline s64 -raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) -{ - return arch_atomic64_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) -{ - return arch_atomic64_try_cmpxchg(v, old, new); -} - -static __always_inline bool -raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) -{ - return arch_atomic64_try_cmpxchg_acquire(v, old, new); -} - -static __always_inline bool -raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) -{ - return arch_atomic64_try_cmpxchg_release(v, old, new); -} - -static __always_inline bool -raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) -{ - return arch_atomic64_try_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic64_sub_and_test(s64 i, atomic64_t *v) -{ - return arch_atomic64_sub_and_test(i, v); -} - -static __always_inline bool -raw_atomic64_dec_and_test(atomic64_t *v) -{ - return arch_atomic64_dec_and_test(v); -} - -static __always_inline bool -raw_atomic64_inc_and_test(atomic64_t *v) -{ - return arch_atomic64_inc_and_test(v); -} - -static __always_inline bool -raw_atomic64_add_negative(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_negative(i, v); -} - -static __always_inline bool -raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_negative_acquire(i, v); -} - -static __always_inline bool -raw_atomic64_add_negative_release(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_negative_release(i, v); -} - -static __always_inline bool -raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) -{ - return arch_atomic64_add_negative_relaxed(i, v); -} - -static __always_inline s64 -raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) -{ - return arch_atomic64_fetch_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) -{ - return arch_atomic64_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic64_inc_not_zero(atomic64_t *v) -{ - return arch_atomic64_inc_not_zero(v); -} - -static __always_inline bool -raw_atomic64_inc_unless_negative(atomic64_t *v) -{ - return arch_atomic64_inc_unless_negative(v); -} - -static __always_inline bool -raw_atomic64_dec_unless_positive(atomic64_t *v) -{ - return arch_atomic64_dec_unless_positive(v); -} - -static __always_inline s64 -raw_atomic64_dec_if_positive(atomic64_t *v) -{ - return arch_atomic64_dec_if_positive(v); -} - -#define raw_xchg(...) \ - arch_xchg(__VA_ARGS__) - -#define raw_xchg_acquire(...) \ - arch_xchg_acquire(__VA_ARGS__) - -#define raw_xchg_release(...) \ - arch_xchg_release(__VA_ARGS__) - -#define raw_xchg_relaxed(...) \ - arch_xchg_relaxed(__VA_ARGS__) - -#define raw_cmpxchg(...) \ - arch_cmpxchg(__VA_ARGS__) - -#define raw_cmpxchg_acquire(...) \ - arch_cmpxchg_acquire(__VA_ARGS__) - -#define raw_cmpxchg_release(...) \ - arch_cmpxchg_release(__VA_ARGS__) - -#define raw_cmpxchg_relaxed(...) \ - arch_cmpxchg_relaxed(__VA_ARGS__) - -#define raw_cmpxchg64(...) \ - arch_cmpxchg64(__VA_ARGS__) - -#define raw_cmpxchg64_acquire(...) \ - arch_cmpxchg64_acquire(__VA_ARGS__) - -#define raw_cmpxchg64_release(...) \ - arch_cmpxchg64_release(__VA_ARGS__) - -#define raw_cmpxchg64_relaxed(...) \ - arch_cmpxchg64_relaxed(__VA_ARGS__) - -#define raw_cmpxchg128(...) \ - arch_cmpxchg128(__VA_ARGS__) - -#define raw_cmpxchg128_acquire(...) \ - arch_cmpxchg128_acquire(__VA_ARGS__) - -#define raw_cmpxchg128_release(...) \ - arch_cmpxchg128_release(__VA_ARGS__) - -#define raw_cmpxchg128_relaxed(...) \ - arch_cmpxchg128_relaxed(__VA_ARGS__) - -#define raw_try_cmpxchg(...) \ - arch_try_cmpxchg(__VA_ARGS__) - -#define raw_try_cmpxchg_acquire(...) \ - arch_try_cmpxchg_acquire(__VA_ARGS__) - -#define raw_try_cmpxchg_release(...) \ - arch_try_cmpxchg_release(__VA_ARGS__) - -#define raw_try_cmpxchg_relaxed(...) \ - arch_try_cmpxchg_relaxed(__VA_ARGS__) - -#define raw_try_cmpxchg64(...) \ - arch_try_cmpxchg64(__VA_ARGS__) - -#define raw_try_cmpxchg64_acquire(...) \ - arch_try_cmpxchg64_acquire(__VA_ARGS__) - -#define raw_try_cmpxchg64_release(...) \ - arch_try_cmpxchg64_release(__VA_ARGS__) - -#define raw_try_cmpxchg64_relaxed(...) \ - arch_try_cmpxchg64_relaxed(__VA_ARGS__) - -#define raw_try_cmpxchg128(...) \ - arch_try_cmpxchg128(__VA_ARGS__) - -#define raw_try_cmpxchg128_acquire(...) \ - arch_try_cmpxchg128_acquire(__VA_ARGS__) - -#define raw_try_cmpxchg128_release(...) \ - arch_try_cmpxchg128_release(__VA_ARGS__) - -#define raw_try_cmpxchg128_relaxed(...) \ - arch_try_cmpxchg128_relaxed(__VA_ARGS__) - -#define raw_cmpxchg_local(...) \ - arch_cmpxchg_local(__VA_ARGS__) - -#define raw_cmpxchg64_local(...) \ - arch_cmpxchg64_local(__VA_ARGS__) - -#define raw_cmpxchg128_local(...) \ - arch_cmpxchg128_local(__VA_ARGS__) - -#define raw_sync_cmpxchg(...) \ - arch_sync_cmpxchg(__VA_ARGS__) - -#define raw_try_cmpxchg_local(...) \ - arch_try_cmpxchg_local(__VA_ARGS__) - -#define raw_try_cmpxchg64_local(...) \ - arch_try_cmpxchg64_local(__VA_ARGS__) - -#define raw_try_cmpxchg128_local(...) \ - arch_try_cmpxchg128_local(__VA_ARGS__) - -#endif /* _LINUX_ATOMIC_RAW_H */ -// b23ed4424e85200e200ded094522e1d743b3a5b1 -- cgit v1.2.3 From 630399469ffcb937936644fbaa5daf61e700a329 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:19 +0100 Subject: locking/atomic: scripts: simplify raw_atomic_long*() definitions Currently, atomic-long is split into two sections, one defining the raw_atomic_long_*() ops for CONFIG_64BIT, and one defining the raw atomic_long_*() ops for !CONFIG_64BIT. With many lines elided, this looks like: | #ifdef CONFIG_64BIT | ... | static __always_inline bool | raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) | { | return raw_atomic64_try_cmpxchg(v, (s64 *)old, new); | } | ... | #else /* CONFIG_64BIT */ | ... | static __always_inline bool | raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) | { | return raw_atomic_try_cmpxchg(v, (int *)old, new); | } | ... | #endif The two definitions are spread far apart in the file, and duplicate the prototype, making it hard to have a legible set of kerneldoc comments. Make this simpler by defining the C prototype once, and writing the two definitions inline. For example, the above becomes: | static __always_inline bool | raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) | { | #ifdef CONFIG_64BIT | return raw_atomic64_try_cmpxchg(v, (s64 *)old, new); | #else | return raw_atomic_try_cmpxchg(v, (int *)old, new); | #endif | } As we now always have a single copy of the C prototype wrapping all the potential definitions, we now have an obvious single location for kerneldoc comments. As a bonus, both the script and the generated file are somewhat shorter. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-23-mark.rutland@arm.com --- include/linux/atomic/atomic-long.h | 857 +++++++++++++++---------------------- 1 file changed, 341 insertions(+), 516 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h index 92dc82ce1ce6..63e0b4078ebd 100644 --- a/include/linux/atomic/atomic-long.h +++ b/include/linux/atomic/atomic-long.h @@ -21,1030 +21,855 @@ typedef atomic_t atomic_long_t; #define atomic_long_cond_read_relaxed atomic_cond_read_relaxed #endif -#ifdef CONFIG_64BIT - -static __always_inline long -raw_atomic_long_read(const atomic_long_t *v) -{ - return raw_atomic64_read(v); -} - -static __always_inline long -raw_atomic_long_read_acquire(const atomic_long_t *v) -{ - return raw_atomic64_read_acquire(v); -} - -static __always_inline void -raw_atomic_long_set(atomic_long_t *v, long i) -{ - raw_atomic64_set(v, i); -} - -static __always_inline void -raw_atomic_long_set_release(atomic_long_t *v, long i) -{ - raw_atomic64_set_release(v, i); -} - -static __always_inline void -raw_atomic_long_add(long i, atomic_long_t *v) -{ - raw_atomic64_add(i, v); -} - -static __always_inline long -raw_atomic_long_add_return(long i, atomic_long_t *v) -{ - return raw_atomic64_add_return(i, v); -} - -static __always_inline long -raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_add_return_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_add_return_release(long i, atomic_long_t *v) -{ - return raw_atomic64_add_return_release(i, v); -} - -static __always_inline long -raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_add_return_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_add(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_add_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_add_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_add_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_sub(long i, atomic_long_t *v) -{ - raw_atomic64_sub(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return(long i, atomic_long_t *v) -{ - return raw_atomic64_sub_return(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_sub_return_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return_release(long i, atomic_long_t *v) -{ - return raw_atomic64_sub_return_release(i, v); -} - -static __always_inline long -raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_sub_return_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_sub(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_sub_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_sub_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_sub_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_inc(atomic_long_t *v) -{ - raw_atomic64_inc(v); -} - -static __always_inline long -raw_atomic_long_inc_return(atomic_long_t *v) -{ - return raw_atomic64_inc_return(v); -} - -static __always_inline long -raw_atomic_long_inc_return_acquire(atomic_long_t *v) -{ - return raw_atomic64_inc_return_acquire(v); -} - -static __always_inline long -raw_atomic_long_inc_return_release(atomic_long_t *v) -{ - return raw_atomic64_inc_return_release(v); -} - -static __always_inline long -raw_atomic_long_inc_return_relaxed(atomic_long_t *v) -{ - return raw_atomic64_inc_return_relaxed(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc(atomic_long_t *v) -{ - return raw_atomic64_fetch_inc(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) -{ - return raw_atomic64_fetch_inc_acquire(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc_release(atomic_long_t *v) -{ - return raw_atomic64_fetch_inc_release(v); -} - -static __always_inline long -raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) -{ - return raw_atomic64_fetch_inc_relaxed(v); -} - -static __always_inline void -raw_atomic_long_dec(atomic_long_t *v) -{ - raw_atomic64_dec(v); -} - -static __always_inline long -raw_atomic_long_dec_return(atomic_long_t *v) -{ - return raw_atomic64_dec_return(v); -} - -static __always_inline long -raw_atomic_long_dec_return_acquire(atomic_long_t *v) -{ - return raw_atomic64_dec_return_acquire(v); -} - -static __always_inline long -raw_atomic_long_dec_return_release(atomic_long_t *v) -{ - return raw_atomic64_dec_return_release(v); -} - -static __always_inline long -raw_atomic_long_dec_return_relaxed(atomic_long_t *v) -{ - return raw_atomic64_dec_return_relaxed(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec(atomic_long_t *v) -{ - return raw_atomic64_fetch_dec(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) -{ - return raw_atomic64_fetch_dec_acquire(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec_release(atomic_long_t *v) -{ - return raw_atomic64_fetch_dec_release(v); -} - -static __always_inline long -raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) -{ - return raw_atomic64_fetch_dec_relaxed(v); -} - -static __always_inline void -raw_atomic_long_and(long i, atomic_long_t *v) -{ - raw_atomic64_and(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_and(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_and_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_and_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_and_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_andnot(long i, atomic_long_t *v) -{ - raw_atomic64_andnot(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_andnot(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_andnot_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_andnot_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_andnot_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_or(long i, atomic_long_t *v) -{ - raw_atomic64_or(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_or(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_or_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_or_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_or_relaxed(i, v); -} - -static __always_inline void -raw_atomic_long_xor(long i, atomic_long_t *v) -{ - raw_atomic64_xor(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_xor(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_xor_acquire(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_xor_release(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_fetch_xor_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_xchg(atomic_long_t *v, long i) -{ - return raw_atomic64_xchg(v, i); -} - -static __always_inline long -raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) -{ - return raw_atomic64_xchg_acquire(v, i); -} - -static __always_inline long -raw_atomic_long_xchg_release(atomic_long_t *v, long i) -{ - return raw_atomic64_xchg_release(v, i); -} - -static __always_inline long -raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) -{ - return raw_atomic64_xchg_relaxed(v, i); -} - -static __always_inline long -raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) -{ - return raw_atomic64_cmpxchg(v, old, new); -} - -static __always_inline long -raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) -{ - return raw_atomic64_cmpxchg_acquire(v, old, new); -} - -static __always_inline long -raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) -{ - return raw_atomic64_cmpxchg_release(v, old, new); -} - -static __always_inline long -raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) -{ - return raw_atomic64_cmpxchg_relaxed(v, old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) -{ - return raw_atomic64_try_cmpxchg(v, (s64 *)old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) -{ - return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) -{ - return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new); -} - -static __always_inline bool -raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) -{ - return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new); -} - -static __always_inline bool -raw_atomic_long_sub_and_test(long i, atomic_long_t *v) -{ - return raw_atomic64_sub_and_test(i, v); -} - -static __always_inline bool -raw_atomic_long_dec_and_test(atomic_long_t *v) -{ - return raw_atomic64_dec_and_test(v); -} - -static __always_inline bool -raw_atomic_long_inc_and_test(atomic_long_t *v) -{ - return raw_atomic64_inc_and_test(v); -} - -static __always_inline bool -raw_atomic_long_add_negative(long i, atomic_long_t *v) -{ - return raw_atomic64_add_negative(i, v); -} - -static __always_inline bool -raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) -{ - return raw_atomic64_add_negative_acquire(i, v); -} - -static __always_inline bool -raw_atomic_long_add_negative_release(long i, atomic_long_t *v) -{ - return raw_atomic64_add_negative_release(i, v); -} - -static __always_inline bool -raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) -{ - return raw_atomic64_add_negative_relaxed(i, v); -} - -static __always_inline long -raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) -{ - return raw_atomic64_fetch_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) -{ - return raw_atomic64_add_unless(v, a, u); -} - -static __always_inline bool -raw_atomic_long_inc_not_zero(atomic_long_t *v) -{ - return raw_atomic64_inc_not_zero(v); -} - -static __always_inline bool -raw_atomic_long_inc_unless_negative(atomic_long_t *v) -{ - return raw_atomic64_inc_unless_negative(v); -} - -static __always_inline bool -raw_atomic_long_dec_unless_positive(atomic_long_t *v) -{ - return raw_atomic64_dec_unless_positive(v); -} - -static __always_inline long -raw_atomic_long_dec_if_positive(atomic_long_t *v) -{ - return raw_atomic64_dec_if_positive(v); -} - -#else /* CONFIG_64BIT */ - static __always_inline long raw_atomic_long_read(const atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_read(v); +#else return raw_atomic_read(v); +#endif } static __always_inline long raw_atomic_long_read_acquire(const atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_read_acquire(v); +#else return raw_atomic_read_acquire(v); +#endif } static __always_inline void raw_atomic_long_set(atomic_long_t *v, long i) { +#ifdef CONFIG_64BIT + raw_atomic64_set(v, i); +#else raw_atomic_set(v, i); +#endif } static __always_inline void raw_atomic_long_set_release(atomic_long_t *v, long i) { +#ifdef CONFIG_64BIT + raw_atomic64_set_release(v, i); +#else raw_atomic_set_release(v, i); +#endif } static __always_inline void raw_atomic_long_add(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_add(i, v); +#else raw_atomic_add(i, v); +#endif } static __always_inline long raw_atomic_long_add_return(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_return(i, v); +#else return raw_atomic_add_return(i, v); +#endif } static __always_inline long raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_return_acquire(i, v); +#else return raw_atomic_add_return_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_add_return_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_return_release(i, v); +#else return raw_atomic_add_return_release(i, v); +#endif } static __always_inline long raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_return_relaxed(i, v); +#else return raw_atomic_add_return_relaxed(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_add(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_add(i, v); +#else return raw_atomic_fetch_add(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_add_acquire(i, v); +#else return raw_atomic_fetch_add_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_add_release(i, v); +#else return raw_atomic_fetch_add_release(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_add_relaxed(i, v); +#else return raw_atomic_fetch_add_relaxed(i, v); +#endif } static __always_inline void raw_atomic_long_sub(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_sub(i, v); +#else raw_atomic_sub(i, v); +#endif } static __always_inline long raw_atomic_long_sub_return(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_sub_return(i, v); +#else return raw_atomic_sub_return(i, v); +#endif } static __always_inline long raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_sub_return_acquire(i, v); +#else return raw_atomic_sub_return_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_sub_return_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_sub_return_release(i, v); +#else return raw_atomic_sub_return_release(i, v); +#endif } static __always_inline long raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_sub_return_relaxed(i, v); +#else return raw_atomic_sub_return_relaxed(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_sub(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_sub(i, v); +#else return raw_atomic_fetch_sub(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_sub_acquire(i, v); +#else return raw_atomic_fetch_sub_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_sub_release(i, v); +#else return raw_atomic_fetch_sub_release(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_sub_relaxed(i, v); +#else return raw_atomic_fetch_sub_relaxed(i, v); +#endif } static __always_inline void raw_atomic_long_inc(atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_inc(v); +#else raw_atomic_inc(v); +#endif } static __always_inline long raw_atomic_long_inc_return(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_return(v); +#else return raw_atomic_inc_return(v); +#endif } static __always_inline long raw_atomic_long_inc_return_acquire(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_return_acquire(v); +#else return raw_atomic_inc_return_acquire(v); +#endif } static __always_inline long raw_atomic_long_inc_return_release(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_return_release(v); +#else return raw_atomic_inc_return_release(v); +#endif } static __always_inline long raw_atomic_long_inc_return_relaxed(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_return_relaxed(v); +#else return raw_atomic_inc_return_relaxed(v); +#endif } static __always_inline long raw_atomic_long_fetch_inc(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_inc(v); +#else return raw_atomic_fetch_inc(v); +#endif } static __always_inline long raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_inc_acquire(v); +#else return raw_atomic_fetch_inc_acquire(v); +#endif } static __always_inline long raw_atomic_long_fetch_inc_release(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_inc_release(v); +#else return raw_atomic_fetch_inc_release(v); +#endif } static __always_inline long raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_inc_relaxed(v); +#else return raw_atomic_fetch_inc_relaxed(v); +#endif } static __always_inline void raw_atomic_long_dec(atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_dec(v); +#else raw_atomic_dec(v); +#endif } static __always_inline long raw_atomic_long_dec_return(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_return(v); +#else return raw_atomic_dec_return(v); +#endif } static __always_inline long raw_atomic_long_dec_return_acquire(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_return_acquire(v); +#else return raw_atomic_dec_return_acquire(v); +#endif } static __always_inline long raw_atomic_long_dec_return_release(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_return_release(v); +#else return raw_atomic_dec_return_release(v); +#endif } static __always_inline long raw_atomic_long_dec_return_relaxed(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_return_relaxed(v); +#else return raw_atomic_dec_return_relaxed(v); +#endif } static __always_inline long raw_atomic_long_fetch_dec(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_dec(v); +#else return raw_atomic_fetch_dec(v); +#endif } static __always_inline long raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_dec_acquire(v); +#else return raw_atomic_fetch_dec_acquire(v); +#endif } static __always_inline long raw_atomic_long_fetch_dec_release(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_dec_release(v); +#else return raw_atomic_fetch_dec_release(v); +#endif } static __always_inline long raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_dec_relaxed(v); +#else return raw_atomic_fetch_dec_relaxed(v); +#endif } static __always_inline void raw_atomic_long_and(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_and(i, v); +#else raw_atomic_and(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_and(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_and(i, v); +#else return raw_atomic_fetch_and(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_and_acquire(i, v); +#else return raw_atomic_fetch_and_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_and_release(i, v); +#else return raw_atomic_fetch_and_release(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_and_relaxed(i, v); +#else return raw_atomic_fetch_and_relaxed(i, v); +#endif } static __always_inline void raw_atomic_long_andnot(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_andnot(i, v); +#else raw_atomic_andnot(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_andnot(i, v); +#else return raw_atomic_fetch_andnot(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_andnot_acquire(i, v); +#else return raw_atomic_fetch_andnot_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_andnot_release(i, v); +#else return raw_atomic_fetch_andnot_release(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_andnot_relaxed(i, v); +#else return raw_atomic_fetch_andnot_relaxed(i, v); +#endif } static __always_inline void raw_atomic_long_or(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_or(i, v); +#else raw_atomic_or(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_or(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_or(i, v); +#else return raw_atomic_fetch_or(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_or_acquire(i, v); +#else return raw_atomic_fetch_or_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_or_release(i, v); +#else return raw_atomic_fetch_or_release(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_or_relaxed(i, v); +#else return raw_atomic_fetch_or_relaxed(i, v); +#endif } static __always_inline void raw_atomic_long_xor(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + raw_atomic64_xor(i, v); +#else raw_atomic_xor(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_xor(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_xor(i, v); +#else return raw_atomic_fetch_xor(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_xor_acquire(i, v); +#else return raw_atomic_fetch_xor_acquire(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_xor_release(i, v); +#else return raw_atomic_fetch_xor_release(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_xor_relaxed(i, v); +#else return raw_atomic_fetch_xor_relaxed(i, v); +#endif } static __always_inline long raw_atomic_long_xchg(atomic_long_t *v, long i) { +#ifdef CONFIG_64BIT + return raw_atomic64_xchg(v, i); +#else return raw_atomic_xchg(v, i); +#endif } static __always_inline long raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) { +#ifdef CONFIG_64BIT + return raw_atomic64_xchg_acquire(v, i); +#else return raw_atomic_xchg_acquire(v, i); +#endif } static __always_inline long raw_atomic_long_xchg_release(atomic_long_t *v, long i) { +#ifdef CONFIG_64BIT + return raw_atomic64_xchg_release(v, i); +#else return raw_atomic_xchg_release(v, i); +#endif } static __always_inline long raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) { +#ifdef CONFIG_64BIT + return raw_atomic64_xchg_relaxed(v, i); +#else return raw_atomic_xchg_relaxed(v, i); +#endif } static __always_inline long raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_cmpxchg(v, old, new); +#else return raw_atomic_cmpxchg(v, old, new); +#endif } static __always_inline long raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_cmpxchg_acquire(v, old, new); +#else return raw_atomic_cmpxchg_acquire(v, old, new); +#endif } static __always_inline long raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_cmpxchg_release(v, old, new); +#else return raw_atomic_cmpxchg_release(v, old, new); +#endif } static __always_inline long raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_cmpxchg_relaxed(v, old, new); +#else return raw_atomic_cmpxchg_relaxed(v, old, new); +#endif } static __always_inline bool raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_try_cmpxchg(v, (s64 *)old, new); +#else return raw_atomic_try_cmpxchg(v, (int *)old, new); +#endif } static __always_inline bool raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_try_cmpxchg_acquire(v, (s64 *)old, new); +#else return raw_atomic_try_cmpxchg_acquire(v, (int *)old, new); +#endif } static __always_inline bool raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_try_cmpxchg_release(v, (s64 *)old, new); +#else return raw_atomic_try_cmpxchg_release(v, (int *)old, new); +#endif } static __always_inline bool raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { +#ifdef CONFIG_64BIT + return raw_atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new); +#else return raw_atomic_try_cmpxchg_relaxed(v, (int *)old, new); +#endif } static __always_inline bool raw_atomic_long_sub_and_test(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_sub_and_test(i, v); +#else return raw_atomic_sub_and_test(i, v); +#endif } static __always_inline bool raw_atomic_long_dec_and_test(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_and_test(v); +#else return raw_atomic_dec_and_test(v); +#endif } static __always_inline bool raw_atomic_long_inc_and_test(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_and_test(v); +#else return raw_atomic_inc_and_test(v); +#endif } static __always_inline bool raw_atomic_long_add_negative(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_negative(i, v); +#else return raw_atomic_add_negative(i, v); +#endif } static __always_inline bool raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_negative_acquire(i, v); +#else return raw_atomic_add_negative_acquire(i, v); +#endif } static __always_inline bool raw_atomic_long_add_negative_release(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_negative_release(i, v); +#else return raw_atomic_add_negative_release(i, v); +#endif } static __always_inline bool raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_negative_relaxed(i, v); +#else return raw_atomic_add_negative_relaxed(i, v); +#endif } static __always_inline long raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { +#ifdef CONFIG_64BIT + return raw_atomic64_fetch_add_unless(v, a, u); +#else return raw_atomic_fetch_add_unless(v, a, u); +#endif } static __always_inline bool raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) { +#ifdef CONFIG_64BIT + return raw_atomic64_add_unless(v, a, u); +#else return raw_atomic_add_unless(v, a, u); +#endif } static __always_inline bool raw_atomic_long_inc_not_zero(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_not_zero(v); +#else return raw_atomic_inc_not_zero(v); +#endif } static __always_inline bool raw_atomic_long_inc_unless_negative(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_inc_unless_negative(v); +#else return raw_atomic_inc_unless_negative(v); +#endif } static __always_inline bool raw_atomic_long_dec_unless_positive(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_unless_positive(v); +#else return raw_atomic_dec_unless_positive(v); +#endif } static __always_inline long raw_atomic_long_dec_if_positive(atomic_long_t *v) { +#ifdef CONFIG_64BIT + return raw_atomic64_dec_if_positive(v); +#else return raw_atomic_dec_if_positive(v); +#endif } -#endif /* CONFIG_64BIT */ #endif /* _LINUX_ATOMIC_LONG_H */ -// 108784846d3bbbb201b8dabe621c5dc30b216206 +// ad09f849db0db5b30c82e497eeb9056a394c5f22 -- cgit v1.2.3 From 1d78814d41701c216e28fcf2656526146dec4a1a Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:20 +0100 Subject: locking/atomic: scripts: simplify raw_atomic*() definitions Currently each ordering variant has several potential definitions, with a mixture of preprocessor and C definitions, including several copies of its C prototype, e.g. | #if defined(arch_atomic_fetch_andnot_acquire) | #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire | #elif defined(arch_atomic_fetch_andnot_relaxed) | static __always_inline int | raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) | { | int ret = arch_atomic_fetch_andnot_relaxed(i, v); | __atomic_acquire_fence(); | return ret; | } | #elif defined(arch_atomic_fetch_andnot) | #define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot | #else | static __always_inline int | raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) | { | return raw_atomic_fetch_and_acquire(~i, v); | } | #endif Make this a bit simpler by defining the C prototype once, and writing the various potential definitions as plain C code guarded by ifdeffery. For example, the above becomes: | static __always_inline int | raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) | { | #if defined(arch_atomic_fetch_andnot_acquire) | return arch_atomic_fetch_andnot_acquire(i, v); | #elif defined(arch_atomic_fetch_andnot_relaxed) | int ret = arch_atomic_fetch_andnot_relaxed(i, v); | __atomic_acquire_fence(); | return ret; | #elif defined(arch_atomic_fetch_andnot) | return arch_atomic_fetch_andnot(i, v); | #else | return raw_atomic_fetch_and_acquire(~i, v); | #endif | } Which is far easier to read. As we now always have a single copy of the C prototype wrapping all the potential definitions, we now have an obvious single location for kerneldoc comments. At the same time, the fallbacks for raw_atomic*_xhcg() are made to use 'new' rather than 'i' as the name of the new value. This is what the existing fallback template used, and is more consistent with the raw_atomic{_try,}cmpxchg() fallbacks. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-24-mark.rutland@arm.com --- include/linux/atomic/atomic-arch-fallback.h | 1790 +++++++++++++-------------- include/linux/atomic/atomic-instrumented.h | 50 +- include/linux/atomic/atomic-long.h | 26 +- 3 files changed, 881 insertions(+), 985 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 99bc1a871dc1..470c2890ab8d 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -428,16 +428,20 @@ extern void raw_cmpxchg128_relaxed_not_implemented(void); #define raw_sync_cmpxchg arch_sync_cmpxchg -#define raw_atomic_read arch_atomic_read +static __always_inline int +raw_atomic_read(const atomic_t *v) +{ + return arch_atomic_read(v); +} -#if defined(arch_atomic_read_acquire) -#define raw_atomic_read_acquire arch_atomic_read_acquire -#elif defined(arch_atomic_read) -#define raw_atomic_read_acquire arch_atomic_read -#else static __always_inline int raw_atomic_read_acquire(const atomic_t *v) { +#if defined(arch_atomic_read_acquire) + return arch_atomic_read_acquire(v); +#elif defined(arch_atomic_read) + return arch_atomic_read(v); +#else int ret; if (__native_word(atomic_t)) { @@ -448,1144 +452,1088 @@ raw_atomic_read_acquire(const atomic_t *v) } return ret; -} #endif +} -#define raw_atomic_set arch_atomic_set +static __always_inline void +raw_atomic_set(atomic_t *v, int i) +{ + arch_atomic_set(v, i); +} -#if defined(arch_atomic_set_release) -#define raw_atomic_set_release arch_atomic_set_release -#elif defined(arch_atomic_set) -#define raw_atomic_set_release arch_atomic_set -#else static __always_inline void raw_atomic_set_release(atomic_t *v, int i) { +#if defined(arch_atomic_set_release) + arch_atomic_set_release(v, i); +#elif defined(arch_atomic_set) + arch_atomic_set(v, i); +#else if (__native_word(atomic_t)) { smp_store_release(&(v)->counter, i); } else { __atomic_release_fence(); raw_atomic_set(v, i); } -} #endif +} -#define raw_atomic_add arch_atomic_add +static __always_inline void +raw_atomic_add(int i, atomic_t *v) +{ + arch_atomic_add(i, v); +} -#if defined(arch_atomic_add_return) -#define raw_atomic_add_return arch_atomic_add_return -#elif defined(arch_atomic_add_return_relaxed) static __always_inline int raw_atomic_add_return(int i, atomic_t *v) { +#if defined(arch_atomic_add_return) + return arch_atomic_add_return(i, v); +#elif defined(arch_atomic_add_return_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_add_return_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_add_return" #endif +} -#if defined(arch_atomic_add_return_acquire) -#define raw_atomic_add_return_acquire arch_atomic_add_return_acquire -#elif defined(arch_atomic_add_return_relaxed) static __always_inline int raw_atomic_add_return_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_add_return_acquire) + return arch_atomic_add_return_acquire(i, v); +#elif defined(arch_atomic_add_return_relaxed) int ret = arch_atomic_add_return_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_add_return) -#define raw_atomic_add_return_acquire arch_atomic_add_return + return arch_atomic_add_return(i, v); #else #error "Unable to define raw_atomic_add_return_acquire" #endif +} -#if defined(arch_atomic_add_return_release) -#define raw_atomic_add_return_release arch_atomic_add_return_release -#elif defined(arch_atomic_add_return_relaxed) static __always_inline int raw_atomic_add_return_release(int i, atomic_t *v) { +#if defined(arch_atomic_add_return_release) + return arch_atomic_add_return_release(i, v); +#elif defined(arch_atomic_add_return_relaxed) __atomic_release_fence(); return arch_atomic_add_return_relaxed(i, v); -} #elif defined(arch_atomic_add_return) -#define raw_atomic_add_return_release arch_atomic_add_return + return arch_atomic_add_return(i, v); #else #error "Unable to define raw_atomic_add_return_release" #endif +} +static __always_inline int +raw_atomic_add_return_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_add_return_relaxed) -#define raw_atomic_add_return_relaxed arch_atomic_add_return_relaxed + return arch_atomic_add_return_relaxed(i, v); #elif defined(arch_atomic_add_return) -#define raw_atomic_add_return_relaxed arch_atomic_add_return + return arch_atomic_add_return(i, v); #else #error "Unable to define raw_atomic_add_return_relaxed" #endif +} -#if defined(arch_atomic_fetch_add) -#define raw_atomic_fetch_add arch_atomic_fetch_add -#elif defined(arch_atomic_fetch_add_relaxed) static __always_inline int raw_atomic_fetch_add(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_add) + return arch_atomic_fetch_add(i, v); +#elif defined(arch_atomic_fetch_add_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_add_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_fetch_add" #endif +} -#if defined(arch_atomic_fetch_add_acquire) -#define raw_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire -#elif defined(arch_atomic_fetch_add_relaxed) static __always_inline int raw_atomic_fetch_add_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_add_acquire) + return arch_atomic_fetch_add_acquire(i, v); +#elif defined(arch_atomic_fetch_add_relaxed) int ret = arch_atomic_fetch_add_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_add) -#define raw_atomic_fetch_add_acquire arch_atomic_fetch_add + return arch_atomic_fetch_add(i, v); #else #error "Unable to define raw_atomic_fetch_add_acquire" #endif +} -#if defined(arch_atomic_fetch_add_release) -#define raw_atomic_fetch_add_release arch_atomic_fetch_add_release -#elif defined(arch_atomic_fetch_add_relaxed) static __always_inline int raw_atomic_fetch_add_release(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_add_release) + return arch_atomic_fetch_add_release(i, v); +#elif defined(arch_atomic_fetch_add_relaxed) __atomic_release_fence(); return arch_atomic_fetch_add_relaxed(i, v); -} #elif defined(arch_atomic_fetch_add) -#define raw_atomic_fetch_add_release arch_atomic_fetch_add + return arch_atomic_fetch_add(i, v); #else #error "Unable to define raw_atomic_fetch_add_release" #endif +} +static __always_inline int +raw_atomic_fetch_add_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_fetch_add_relaxed) -#define raw_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed + return arch_atomic_fetch_add_relaxed(i, v); #elif defined(arch_atomic_fetch_add) -#define raw_atomic_fetch_add_relaxed arch_atomic_fetch_add + return arch_atomic_fetch_add(i, v); #else #error "Unable to define raw_atomic_fetch_add_relaxed" #endif +} -#define raw_atomic_sub arch_atomic_sub +static __always_inline void +raw_atomic_sub(int i, atomic_t *v) +{ + arch_atomic_sub(i, v); +} -#if defined(arch_atomic_sub_return) -#define raw_atomic_sub_return arch_atomic_sub_return -#elif defined(arch_atomic_sub_return_relaxed) static __always_inline int raw_atomic_sub_return(int i, atomic_t *v) { +#if defined(arch_atomic_sub_return) + return arch_atomic_sub_return(i, v); +#elif defined(arch_atomic_sub_return_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_sub_return_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_sub_return" #endif +} -#if defined(arch_atomic_sub_return_acquire) -#define raw_atomic_sub_return_acquire arch_atomic_sub_return_acquire -#elif defined(arch_atomic_sub_return_relaxed) static __always_inline int raw_atomic_sub_return_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_sub_return_acquire) + return arch_atomic_sub_return_acquire(i, v); +#elif defined(arch_atomic_sub_return_relaxed) int ret = arch_atomic_sub_return_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_sub_return) -#define raw_atomic_sub_return_acquire arch_atomic_sub_return + return arch_atomic_sub_return(i, v); #else #error "Unable to define raw_atomic_sub_return_acquire" #endif +} -#if defined(arch_atomic_sub_return_release) -#define raw_atomic_sub_return_release arch_atomic_sub_return_release -#elif defined(arch_atomic_sub_return_relaxed) static __always_inline int raw_atomic_sub_return_release(int i, atomic_t *v) { +#if defined(arch_atomic_sub_return_release) + return arch_atomic_sub_return_release(i, v); +#elif defined(arch_atomic_sub_return_relaxed) __atomic_release_fence(); return arch_atomic_sub_return_relaxed(i, v); -} #elif defined(arch_atomic_sub_return) -#define raw_atomic_sub_return_release arch_atomic_sub_return + return arch_atomic_sub_return(i, v); #else #error "Unable to define raw_atomic_sub_return_release" #endif +} +static __always_inline int +raw_atomic_sub_return_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_sub_return_relaxed) -#define raw_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed + return arch_atomic_sub_return_relaxed(i, v); #elif defined(arch_atomic_sub_return) -#define raw_atomic_sub_return_relaxed arch_atomic_sub_return + return arch_atomic_sub_return(i, v); #else #error "Unable to define raw_atomic_sub_return_relaxed" #endif +} -#if defined(arch_atomic_fetch_sub) -#define raw_atomic_fetch_sub arch_atomic_fetch_sub -#elif defined(arch_atomic_fetch_sub_relaxed) static __always_inline int raw_atomic_fetch_sub(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_sub) + return arch_atomic_fetch_sub(i, v); +#elif defined(arch_atomic_fetch_sub_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_sub_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_fetch_sub" #endif +} -#if defined(arch_atomic_fetch_sub_acquire) -#define raw_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire -#elif defined(arch_atomic_fetch_sub_relaxed) static __always_inline int raw_atomic_fetch_sub_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_sub_acquire) + return arch_atomic_fetch_sub_acquire(i, v); +#elif defined(arch_atomic_fetch_sub_relaxed) int ret = arch_atomic_fetch_sub_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_sub) -#define raw_atomic_fetch_sub_acquire arch_atomic_fetch_sub + return arch_atomic_fetch_sub(i, v); #else #error "Unable to define raw_atomic_fetch_sub_acquire" #endif +} -#if defined(arch_atomic_fetch_sub_release) -#define raw_atomic_fetch_sub_release arch_atomic_fetch_sub_release -#elif defined(arch_atomic_fetch_sub_relaxed) static __always_inline int raw_atomic_fetch_sub_release(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_sub_release) + return arch_atomic_fetch_sub_release(i, v); +#elif defined(arch_atomic_fetch_sub_relaxed) __atomic_release_fence(); return arch_atomic_fetch_sub_relaxed(i, v); -} #elif defined(arch_atomic_fetch_sub) -#define raw_atomic_fetch_sub_release arch_atomic_fetch_sub + return arch_atomic_fetch_sub(i, v); #else #error "Unable to define raw_atomic_fetch_sub_release" #endif +} +static __always_inline int +raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_fetch_sub_relaxed) -#define raw_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed + return arch_atomic_fetch_sub_relaxed(i, v); #elif defined(arch_atomic_fetch_sub) -#define raw_atomic_fetch_sub_relaxed arch_atomic_fetch_sub + return arch_atomic_fetch_sub(i, v); #else #error "Unable to define raw_atomic_fetch_sub_relaxed" #endif +} -#if defined(arch_atomic_inc) -#define raw_atomic_inc arch_atomic_inc -#else static __always_inline void raw_atomic_inc(atomic_t *v) { +#if defined(arch_atomic_inc) + arch_atomic_inc(v); +#else raw_atomic_add(1, v); -} #endif +} -#if defined(arch_atomic_inc_return) -#define raw_atomic_inc_return arch_atomic_inc_return -#elif defined(arch_atomic_inc_return_relaxed) static __always_inline int raw_atomic_inc_return(atomic_t *v) { +#if defined(arch_atomic_inc_return) + return arch_atomic_inc_return(v); +#elif defined(arch_atomic_inc_return_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_inc_return_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_inc_return(atomic_t *v) -{ return raw_atomic_add_return(1, v); -} #endif +} -#if defined(arch_atomic_inc_return_acquire) -#define raw_atomic_inc_return_acquire arch_atomic_inc_return_acquire -#elif defined(arch_atomic_inc_return_relaxed) static __always_inline int raw_atomic_inc_return_acquire(atomic_t *v) { +#if defined(arch_atomic_inc_return_acquire) + return arch_atomic_inc_return_acquire(v); +#elif defined(arch_atomic_inc_return_relaxed) int ret = arch_atomic_inc_return_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_inc_return) -#define raw_atomic_inc_return_acquire arch_atomic_inc_return + return arch_atomic_inc_return(v); #else -static __always_inline int -raw_atomic_inc_return_acquire(atomic_t *v) -{ return raw_atomic_add_return_acquire(1, v); -} #endif +} -#if defined(arch_atomic_inc_return_release) -#define raw_atomic_inc_return_release arch_atomic_inc_return_release -#elif defined(arch_atomic_inc_return_relaxed) static __always_inline int raw_atomic_inc_return_release(atomic_t *v) { +#if defined(arch_atomic_inc_return_release) + return arch_atomic_inc_return_release(v); +#elif defined(arch_atomic_inc_return_relaxed) __atomic_release_fence(); return arch_atomic_inc_return_relaxed(v); -} #elif defined(arch_atomic_inc_return) -#define raw_atomic_inc_return_release arch_atomic_inc_return + return arch_atomic_inc_return(v); #else -static __always_inline int -raw_atomic_inc_return_release(atomic_t *v) -{ return raw_atomic_add_return_release(1, v); -} #endif +} -#if defined(arch_atomic_inc_return_relaxed) -#define raw_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed -#elif defined(arch_atomic_inc_return) -#define raw_atomic_inc_return_relaxed arch_atomic_inc_return -#else static __always_inline int raw_atomic_inc_return_relaxed(atomic_t *v) { +#if defined(arch_atomic_inc_return_relaxed) + return arch_atomic_inc_return_relaxed(v); +#elif defined(arch_atomic_inc_return) + return arch_atomic_inc_return(v); +#else return raw_atomic_add_return_relaxed(1, v); -} #endif +} -#if defined(arch_atomic_fetch_inc) -#define raw_atomic_fetch_inc arch_atomic_fetch_inc -#elif defined(arch_atomic_fetch_inc_relaxed) static __always_inline int raw_atomic_fetch_inc(atomic_t *v) { +#if defined(arch_atomic_fetch_inc) + return arch_atomic_fetch_inc(v); +#elif defined(arch_atomic_fetch_inc_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_inc_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_fetch_inc(atomic_t *v) -{ return raw_atomic_fetch_add(1, v); -} #endif +} -#if defined(arch_atomic_fetch_inc_acquire) -#define raw_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire -#elif defined(arch_atomic_fetch_inc_relaxed) static __always_inline int raw_atomic_fetch_inc_acquire(atomic_t *v) { +#if defined(arch_atomic_fetch_inc_acquire) + return arch_atomic_fetch_inc_acquire(v); +#elif defined(arch_atomic_fetch_inc_relaxed) int ret = arch_atomic_fetch_inc_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_inc) -#define raw_atomic_fetch_inc_acquire arch_atomic_fetch_inc + return arch_atomic_fetch_inc(v); #else -static __always_inline int -raw_atomic_fetch_inc_acquire(atomic_t *v) -{ return raw_atomic_fetch_add_acquire(1, v); -} #endif +} -#if defined(arch_atomic_fetch_inc_release) -#define raw_atomic_fetch_inc_release arch_atomic_fetch_inc_release -#elif defined(arch_atomic_fetch_inc_relaxed) static __always_inline int raw_atomic_fetch_inc_release(atomic_t *v) { +#if defined(arch_atomic_fetch_inc_release) + return arch_atomic_fetch_inc_release(v); +#elif defined(arch_atomic_fetch_inc_relaxed) __atomic_release_fence(); return arch_atomic_fetch_inc_relaxed(v); -} #elif defined(arch_atomic_fetch_inc) -#define raw_atomic_fetch_inc_release arch_atomic_fetch_inc + return arch_atomic_fetch_inc(v); #else -static __always_inline int -raw_atomic_fetch_inc_release(atomic_t *v) -{ return raw_atomic_fetch_add_release(1, v); -} #endif +} -#if defined(arch_atomic_fetch_inc_relaxed) -#define raw_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed -#elif defined(arch_atomic_fetch_inc) -#define raw_atomic_fetch_inc_relaxed arch_atomic_fetch_inc -#else static __always_inline int raw_atomic_fetch_inc_relaxed(atomic_t *v) { +#if defined(arch_atomic_fetch_inc_relaxed) + return arch_atomic_fetch_inc_relaxed(v); +#elif defined(arch_atomic_fetch_inc) + return arch_atomic_fetch_inc(v); +#else return raw_atomic_fetch_add_relaxed(1, v); -} #endif +} -#if defined(arch_atomic_dec) -#define raw_atomic_dec arch_atomic_dec -#else static __always_inline void raw_atomic_dec(atomic_t *v) { +#if defined(arch_atomic_dec) + arch_atomic_dec(v); +#else raw_atomic_sub(1, v); -} #endif +} -#if defined(arch_atomic_dec_return) -#define raw_atomic_dec_return arch_atomic_dec_return -#elif defined(arch_atomic_dec_return_relaxed) static __always_inline int raw_atomic_dec_return(atomic_t *v) { +#if defined(arch_atomic_dec_return) + return arch_atomic_dec_return(v); +#elif defined(arch_atomic_dec_return_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_dec_return_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_dec_return(atomic_t *v) -{ return raw_atomic_sub_return(1, v); -} #endif +} -#if defined(arch_atomic_dec_return_acquire) -#define raw_atomic_dec_return_acquire arch_atomic_dec_return_acquire -#elif defined(arch_atomic_dec_return_relaxed) static __always_inline int raw_atomic_dec_return_acquire(atomic_t *v) { +#if defined(arch_atomic_dec_return_acquire) + return arch_atomic_dec_return_acquire(v); +#elif defined(arch_atomic_dec_return_relaxed) int ret = arch_atomic_dec_return_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_dec_return) -#define raw_atomic_dec_return_acquire arch_atomic_dec_return + return arch_atomic_dec_return(v); #else -static __always_inline int -raw_atomic_dec_return_acquire(atomic_t *v) -{ return raw_atomic_sub_return_acquire(1, v); -} #endif +} -#if defined(arch_atomic_dec_return_release) -#define raw_atomic_dec_return_release arch_atomic_dec_return_release -#elif defined(arch_atomic_dec_return_relaxed) static __always_inline int raw_atomic_dec_return_release(atomic_t *v) { +#if defined(arch_atomic_dec_return_release) + return arch_atomic_dec_return_release(v); +#elif defined(arch_atomic_dec_return_relaxed) __atomic_release_fence(); return arch_atomic_dec_return_relaxed(v); -} #elif defined(arch_atomic_dec_return) -#define raw_atomic_dec_return_release arch_atomic_dec_return + return arch_atomic_dec_return(v); #else -static __always_inline int -raw_atomic_dec_return_release(atomic_t *v) -{ return raw_atomic_sub_return_release(1, v); -} #endif +} -#if defined(arch_atomic_dec_return_relaxed) -#define raw_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed -#elif defined(arch_atomic_dec_return) -#define raw_atomic_dec_return_relaxed arch_atomic_dec_return -#else static __always_inline int raw_atomic_dec_return_relaxed(atomic_t *v) { +#if defined(arch_atomic_dec_return_relaxed) + return arch_atomic_dec_return_relaxed(v); +#elif defined(arch_atomic_dec_return) + return arch_atomic_dec_return(v); +#else return raw_atomic_sub_return_relaxed(1, v); -} #endif +} -#if defined(arch_atomic_fetch_dec) -#define raw_atomic_fetch_dec arch_atomic_fetch_dec -#elif defined(arch_atomic_fetch_dec_relaxed) static __always_inline int raw_atomic_fetch_dec(atomic_t *v) { +#if defined(arch_atomic_fetch_dec) + return arch_atomic_fetch_dec(v); +#elif defined(arch_atomic_fetch_dec_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_dec_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_fetch_dec(atomic_t *v) -{ return raw_atomic_fetch_sub(1, v); -} #endif +} -#if defined(arch_atomic_fetch_dec_acquire) -#define raw_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire -#elif defined(arch_atomic_fetch_dec_relaxed) static __always_inline int raw_atomic_fetch_dec_acquire(atomic_t *v) { +#if defined(arch_atomic_fetch_dec_acquire) + return arch_atomic_fetch_dec_acquire(v); +#elif defined(arch_atomic_fetch_dec_relaxed) int ret = arch_atomic_fetch_dec_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_dec) -#define raw_atomic_fetch_dec_acquire arch_atomic_fetch_dec + return arch_atomic_fetch_dec(v); #else -static __always_inline int -raw_atomic_fetch_dec_acquire(atomic_t *v) -{ return raw_atomic_fetch_sub_acquire(1, v); -} #endif +} -#if defined(arch_atomic_fetch_dec_release) -#define raw_atomic_fetch_dec_release arch_atomic_fetch_dec_release -#elif defined(arch_atomic_fetch_dec_relaxed) static __always_inline int raw_atomic_fetch_dec_release(atomic_t *v) { +#if defined(arch_atomic_fetch_dec_release) + return arch_atomic_fetch_dec_release(v); +#elif defined(arch_atomic_fetch_dec_relaxed) __atomic_release_fence(); return arch_atomic_fetch_dec_relaxed(v); -} #elif defined(arch_atomic_fetch_dec) -#define raw_atomic_fetch_dec_release arch_atomic_fetch_dec + return arch_atomic_fetch_dec(v); #else -static __always_inline int -raw_atomic_fetch_dec_release(atomic_t *v) -{ return raw_atomic_fetch_sub_release(1, v); -} #endif +} -#if defined(arch_atomic_fetch_dec_relaxed) -#define raw_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed -#elif defined(arch_atomic_fetch_dec) -#define raw_atomic_fetch_dec_relaxed arch_atomic_fetch_dec -#else static __always_inline int raw_atomic_fetch_dec_relaxed(atomic_t *v) { +#if defined(arch_atomic_fetch_dec_relaxed) + return arch_atomic_fetch_dec_relaxed(v); +#elif defined(arch_atomic_fetch_dec) + return arch_atomic_fetch_dec(v); +#else return raw_atomic_fetch_sub_relaxed(1, v); -} #endif +} -#define raw_atomic_and arch_atomic_and +static __always_inline void +raw_atomic_and(int i, atomic_t *v) +{ + arch_atomic_and(i, v); +} -#if defined(arch_atomic_fetch_and) -#define raw_atomic_fetch_and arch_atomic_fetch_and -#elif defined(arch_atomic_fetch_and_relaxed) static __always_inline int raw_atomic_fetch_and(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_and) + return arch_atomic_fetch_and(i, v); +#elif defined(arch_atomic_fetch_and_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_and_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_fetch_and" #endif +} -#if defined(arch_atomic_fetch_and_acquire) -#define raw_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire -#elif defined(arch_atomic_fetch_and_relaxed) static __always_inline int raw_atomic_fetch_and_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_and_acquire) + return arch_atomic_fetch_and_acquire(i, v); +#elif defined(arch_atomic_fetch_and_relaxed) int ret = arch_atomic_fetch_and_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_and) -#define raw_atomic_fetch_and_acquire arch_atomic_fetch_and + return arch_atomic_fetch_and(i, v); #else #error "Unable to define raw_atomic_fetch_and_acquire" #endif +} -#if defined(arch_atomic_fetch_and_release) -#define raw_atomic_fetch_and_release arch_atomic_fetch_and_release -#elif defined(arch_atomic_fetch_and_relaxed) static __always_inline int raw_atomic_fetch_and_release(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_and_release) + return arch_atomic_fetch_and_release(i, v); +#elif defined(arch_atomic_fetch_and_relaxed) __atomic_release_fence(); return arch_atomic_fetch_and_relaxed(i, v); -} #elif defined(arch_atomic_fetch_and) -#define raw_atomic_fetch_and_release arch_atomic_fetch_and + return arch_atomic_fetch_and(i, v); #else #error "Unable to define raw_atomic_fetch_and_release" #endif +} +static __always_inline int +raw_atomic_fetch_and_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_fetch_and_relaxed) -#define raw_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed + return arch_atomic_fetch_and_relaxed(i, v); #elif defined(arch_atomic_fetch_and) -#define raw_atomic_fetch_and_relaxed arch_atomic_fetch_and + return arch_atomic_fetch_and(i, v); #else #error "Unable to define raw_atomic_fetch_and_relaxed" #endif +} -#if defined(arch_atomic_andnot) -#define raw_atomic_andnot arch_atomic_andnot -#else static __always_inline void raw_atomic_andnot(int i, atomic_t *v) { +#if defined(arch_atomic_andnot) + arch_atomic_andnot(i, v); +#else raw_atomic_and(~i, v); -} #endif +} -#if defined(arch_atomic_fetch_andnot) -#define raw_atomic_fetch_andnot arch_atomic_fetch_andnot -#elif defined(arch_atomic_fetch_andnot_relaxed) static __always_inline int raw_atomic_fetch_andnot(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_andnot) + return arch_atomic_fetch_andnot(i, v); +#elif defined(arch_atomic_fetch_andnot_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_andnot_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_fetch_andnot(int i, atomic_t *v) -{ return raw_atomic_fetch_and(~i, v); -} #endif +} -#if defined(arch_atomic_fetch_andnot_acquire) -#define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire -#elif defined(arch_atomic_fetch_andnot_relaxed) static __always_inline int raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_andnot_acquire) + return arch_atomic_fetch_andnot_acquire(i, v); +#elif defined(arch_atomic_fetch_andnot_relaxed) int ret = arch_atomic_fetch_andnot_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_andnot) -#define raw_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot + return arch_atomic_fetch_andnot(i, v); #else -static __always_inline int -raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) -{ return raw_atomic_fetch_and_acquire(~i, v); -} #endif +} -#if defined(arch_atomic_fetch_andnot_release) -#define raw_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release -#elif defined(arch_atomic_fetch_andnot_relaxed) static __always_inline int raw_atomic_fetch_andnot_release(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_andnot_release) + return arch_atomic_fetch_andnot_release(i, v); +#elif defined(arch_atomic_fetch_andnot_relaxed) __atomic_release_fence(); return arch_atomic_fetch_andnot_relaxed(i, v); -} #elif defined(arch_atomic_fetch_andnot) -#define raw_atomic_fetch_andnot_release arch_atomic_fetch_andnot + return arch_atomic_fetch_andnot(i, v); #else -static __always_inline int -raw_atomic_fetch_andnot_release(int i, atomic_t *v) -{ return raw_atomic_fetch_and_release(~i, v); -} #endif +} -#if defined(arch_atomic_fetch_andnot_relaxed) -#define raw_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed -#elif defined(arch_atomic_fetch_andnot) -#define raw_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot -#else static __always_inline int raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_andnot_relaxed) + return arch_atomic_fetch_andnot_relaxed(i, v); +#elif defined(arch_atomic_fetch_andnot) + return arch_atomic_fetch_andnot(i, v); +#else return raw_atomic_fetch_and_relaxed(~i, v); -} #endif +} -#define raw_atomic_or arch_atomic_or +static __always_inline void +raw_atomic_or(int i, atomic_t *v) +{ + arch_atomic_or(i, v); +} -#if defined(arch_atomic_fetch_or) -#define raw_atomic_fetch_or arch_atomic_fetch_or -#elif defined(arch_atomic_fetch_or_relaxed) static __always_inline int raw_atomic_fetch_or(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_or) + return arch_atomic_fetch_or(i, v); +#elif defined(arch_atomic_fetch_or_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_or_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_fetch_or" #endif +} -#if defined(arch_atomic_fetch_or_acquire) -#define raw_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire -#elif defined(arch_atomic_fetch_or_relaxed) static __always_inline int raw_atomic_fetch_or_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_or_acquire) + return arch_atomic_fetch_or_acquire(i, v); +#elif defined(arch_atomic_fetch_or_relaxed) int ret = arch_atomic_fetch_or_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_or) -#define raw_atomic_fetch_or_acquire arch_atomic_fetch_or + return arch_atomic_fetch_or(i, v); #else #error "Unable to define raw_atomic_fetch_or_acquire" #endif +} -#if defined(arch_atomic_fetch_or_release) -#define raw_atomic_fetch_or_release arch_atomic_fetch_or_release -#elif defined(arch_atomic_fetch_or_relaxed) static __always_inline int raw_atomic_fetch_or_release(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_or_release) + return arch_atomic_fetch_or_release(i, v); +#elif defined(arch_atomic_fetch_or_relaxed) __atomic_release_fence(); return arch_atomic_fetch_or_relaxed(i, v); -} #elif defined(arch_atomic_fetch_or) -#define raw_atomic_fetch_or_release arch_atomic_fetch_or + return arch_atomic_fetch_or(i, v); #else #error "Unable to define raw_atomic_fetch_or_release" #endif +} +static __always_inline int +raw_atomic_fetch_or_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_fetch_or_relaxed) -#define raw_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed + return arch_atomic_fetch_or_relaxed(i, v); #elif defined(arch_atomic_fetch_or) -#define raw_atomic_fetch_or_relaxed arch_atomic_fetch_or + return arch_atomic_fetch_or(i, v); #else #error "Unable to define raw_atomic_fetch_or_relaxed" #endif +} -#define raw_atomic_xor arch_atomic_xor +static __always_inline void +raw_atomic_xor(int i, atomic_t *v) +{ + arch_atomic_xor(i, v); +} -#if defined(arch_atomic_fetch_xor) -#define raw_atomic_fetch_xor arch_atomic_fetch_xor -#elif defined(arch_atomic_fetch_xor_relaxed) static __always_inline int raw_atomic_fetch_xor(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_xor) + return arch_atomic_fetch_xor(i, v); +#elif defined(arch_atomic_fetch_xor_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_fetch_xor_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic_fetch_xor" #endif +} -#if defined(arch_atomic_fetch_xor_acquire) -#define raw_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire -#elif defined(arch_atomic_fetch_xor_relaxed) static __always_inline int raw_atomic_fetch_xor_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_xor_acquire) + return arch_atomic_fetch_xor_acquire(i, v); +#elif defined(arch_atomic_fetch_xor_relaxed) int ret = arch_atomic_fetch_xor_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_fetch_xor) -#define raw_atomic_fetch_xor_acquire arch_atomic_fetch_xor + return arch_atomic_fetch_xor(i, v); #else #error "Unable to define raw_atomic_fetch_xor_acquire" #endif +} -#if defined(arch_atomic_fetch_xor_release) -#define raw_atomic_fetch_xor_release arch_atomic_fetch_xor_release -#elif defined(arch_atomic_fetch_xor_relaxed) static __always_inline int raw_atomic_fetch_xor_release(int i, atomic_t *v) { +#if defined(arch_atomic_fetch_xor_release) + return arch_atomic_fetch_xor_release(i, v); +#elif defined(arch_atomic_fetch_xor_relaxed) __atomic_release_fence(); return arch_atomic_fetch_xor_relaxed(i, v); -} #elif defined(arch_atomic_fetch_xor) -#define raw_atomic_fetch_xor_release arch_atomic_fetch_xor + return arch_atomic_fetch_xor(i, v); #else #error "Unable to define raw_atomic_fetch_xor_release" #endif +} +static __always_inline int +raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) +{ #if defined(arch_atomic_fetch_xor_relaxed) -#define raw_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed + return arch_atomic_fetch_xor_relaxed(i, v); #elif defined(arch_atomic_fetch_xor) -#define raw_atomic_fetch_xor_relaxed arch_atomic_fetch_xor + return arch_atomic_fetch_xor(i, v); #else #error "Unable to define raw_atomic_fetch_xor_relaxed" #endif +} -#if defined(arch_atomic_xchg) -#define raw_atomic_xchg arch_atomic_xchg -#elif defined(arch_atomic_xchg_relaxed) static __always_inline int -raw_atomic_xchg(atomic_t *v, int i) +raw_atomic_xchg(atomic_t *v, int new) { +#if defined(arch_atomic_xchg) + return arch_atomic_xchg(v, new); +#elif defined(arch_atomic_xchg_relaxed) int ret; __atomic_pre_full_fence(); - ret = arch_atomic_xchg_relaxed(v, i); + ret = arch_atomic_xchg_relaxed(v, new); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_xchg(atomic_t *v, int new) -{ return raw_xchg(&v->counter, new); -} #endif +} -#if defined(arch_atomic_xchg_acquire) -#define raw_atomic_xchg_acquire arch_atomic_xchg_acquire -#elif defined(arch_atomic_xchg_relaxed) static __always_inline int -raw_atomic_xchg_acquire(atomic_t *v, int i) +raw_atomic_xchg_acquire(atomic_t *v, int new) { - int ret = arch_atomic_xchg_relaxed(v, i); +#if defined(arch_atomic_xchg_acquire) + return arch_atomic_xchg_acquire(v, new); +#elif defined(arch_atomic_xchg_relaxed) + int ret = arch_atomic_xchg_relaxed(v, new); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_xchg) -#define raw_atomic_xchg_acquire arch_atomic_xchg + return arch_atomic_xchg(v, new); #else -static __always_inline int -raw_atomic_xchg_acquire(atomic_t *v, int new) -{ return raw_xchg_acquire(&v->counter, new); -} #endif +} -#if defined(arch_atomic_xchg_release) -#define raw_atomic_xchg_release arch_atomic_xchg_release -#elif defined(arch_atomic_xchg_relaxed) static __always_inline int -raw_atomic_xchg_release(atomic_t *v, int i) +raw_atomic_xchg_release(atomic_t *v, int new) { +#if defined(arch_atomic_xchg_release) + return arch_atomic_xchg_release(v, new); +#elif defined(arch_atomic_xchg_relaxed) __atomic_release_fence(); - return arch_atomic_xchg_relaxed(v, i); -} + return arch_atomic_xchg_relaxed(v, new); #elif defined(arch_atomic_xchg) -#define raw_atomic_xchg_release arch_atomic_xchg + return arch_atomic_xchg(v, new); #else -static __always_inline int -raw_atomic_xchg_release(atomic_t *v, int new) -{ return raw_xchg_release(&v->counter, new); -} #endif +} -#if defined(arch_atomic_xchg_relaxed) -#define raw_atomic_xchg_relaxed arch_atomic_xchg_relaxed -#elif defined(arch_atomic_xchg) -#define raw_atomic_xchg_relaxed arch_atomic_xchg -#else static __always_inline int raw_atomic_xchg_relaxed(atomic_t *v, int new) { +#if defined(arch_atomic_xchg_relaxed) + return arch_atomic_xchg_relaxed(v, new); +#elif defined(arch_atomic_xchg) + return arch_atomic_xchg(v, new); +#else return raw_xchg_relaxed(&v->counter, new); -} #endif +} -#if defined(arch_atomic_cmpxchg) -#define raw_atomic_cmpxchg arch_atomic_cmpxchg -#elif defined(arch_atomic_cmpxchg_relaxed) static __always_inline int raw_atomic_cmpxchg(atomic_t *v, int old, int new) { +#if defined(arch_atomic_cmpxchg) + return arch_atomic_cmpxchg(v, old, new); +#elif defined(arch_atomic_cmpxchg_relaxed) int ret; __atomic_pre_full_fence(); ret = arch_atomic_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; -} #else -static __always_inline int -raw_atomic_cmpxchg(atomic_t *v, int old, int new) -{ return raw_cmpxchg(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic_cmpxchg_acquire) -#define raw_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire -#elif defined(arch_atomic_cmpxchg_relaxed) static __always_inline int raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { +#if defined(arch_atomic_cmpxchg_acquire) + return arch_atomic_cmpxchg_acquire(v, old, new); +#elif defined(arch_atomic_cmpxchg_relaxed) int ret = arch_atomic_cmpxchg_relaxed(v, old, new); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_cmpxchg) -#define raw_atomic_cmpxchg_acquire arch_atomic_cmpxchg + return arch_atomic_cmpxchg(v, old, new); #else -static __always_inline int -raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) -{ return raw_cmpxchg_acquire(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic_cmpxchg_release) -#define raw_atomic_cmpxchg_release arch_atomic_cmpxchg_release -#elif defined(arch_atomic_cmpxchg_relaxed) static __always_inline int raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) { +#if defined(arch_atomic_cmpxchg_release) + return arch_atomic_cmpxchg_release(v, old, new); +#elif defined(arch_atomic_cmpxchg_relaxed) __atomic_release_fence(); return arch_atomic_cmpxchg_relaxed(v, old, new); -} #elif defined(arch_atomic_cmpxchg) -#define raw_atomic_cmpxchg_release arch_atomic_cmpxchg + return arch_atomic_cmpxchg(v, old, new); #else -static __always_inline int -raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) -{ return raw_cmpxchg_release(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic_cmpxchg_relaxed) -#define raw_atomic_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed -#elif defined(arch_atomic_cmpxchg) -#define raw_atomic_cmpxchg_relaxed arch_atomic_cmpxchg -#else static __always_inline int raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { +#if defined(arch_atomic_cmpxchg_relaxed) + return arch_atomic_cmpxchg_relaxed(v, old, new); +#elif defined(arch_atomic_cmpxchg) + return arch_atomic_cmpxchg(v, old, new); +#else return raw_cmpxchg_relaxed(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic_try_cmpxchg) -#define raw_atomic_try_cmpxchg arch_atomic_try_cmpxchg -#elif defined(arch_atomic_try_cmpxchg_relaxed) static __always_inline bool raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) { +#if defined(arch_atomic_try_cmpxchg) + return arch_atomic_try_cmpxchg(v, old, new); +#elif defined(arch_atomic_try_cmpxchg_relaxed) bool ret; __atomic_pre_full_fence(); ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; -} #else -static __always_inline bool -raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) -{ int r, o = *old; r = raw_atomic_cmpxchg(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic_try_cmpxchg_acquire) -#define raw_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire -#elif defined(arch_atomic_try_cmpxchg_relaxed) static __always_inline bool raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { +#if defined(arch_atomic_try_cmpxchg_acquire) + return arch_atomic_try_cmpxchg_acquire(v, old, new); +#elif defined(arch_atomic_try_cmpxchg_relaxed) bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_try_cmpxchg) -#define raw_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg + return arch_atomic_try_cmpxchg(v, old, new); #else -static __always_inline bool -raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) -{ int r, o = *old; r = raw_atomic_cmpxchg_acquire(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic_try_cmpxchg_release) -#define raw_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release -#elif defined(arch_atomic_try_cmpxchg_relaxed) static __always_inline bool raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { +#if defined(arch_atomic_try_cmpxchg_release) + return arch_atomic_try_cmpxchg_release(v, old, new); +#elif defined(arch_atomic_try_cmpxchg_relaxed) __atomic_release_fence(); return arch_atomic_try_cmpxchg_relaxed(v, old, new); -} #elif defined(arch_atomic_try_cmpxchg) -#define raw_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg + return arch_atomic_try_cmpxchg(v, old, new); #else -static __always_inline bool -raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) -{ int r, o = *old; r = raw_atomic_cmpxchg_release(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic_try_cmpxchg_relaxed) -#define raw_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed -#elif defined(arch_atomic_try_cmpxchg) -#define raw_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg -#else static __always_inline bool raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { +#if defined(arch_atomic_try_cmpxchg_relaxed) + return arch_atomic_try_cmpxchg_relaxed(v, old, new); +#elif defined(arch_atomic_try_cmpxchg) + return arch_atomic_try_cmpxchg(v, old, new); +#else int r, o = *old; r = raw_atomic_cmpxchg_relaxed(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic_sub_and_test) -#define raw_atomic_sub_and_test arch_atomic_sub_and_test -#else static __always_inline bool raw_atomic_sub_and_test(int i, atomic_t *v) { +#if defined(arch_atomic_sub_and_test) + return arch_atomic_sub_and_test(i, v); +#else return raw_atomic_sub_return(i, v) == 0; -} #endif +} -#if defined(arch_atomic_dec_and_test) -#define raw_atomic_dec_and_test arch_atomic_dec_and_test -#else static __always_inline bool raw_atomic_dec_and_test(atomic_t *v) { +#if defined(arch_atomic_dec_and_test) + return arch_atomic_dec_and_test(v); +#else return raw_atomic_dec_return(v) == 0; -} #endif +} -#if defined(arch_atomic_inc_and_test) -#define raw_atomic_inc_and_test arch_atomic_inc_and_test -#else static __always_inline bool raw_atomic_inc_and_test(atomic_t *v) { +#if defined(arch_atomic_inc_and_test) + return arch_atomic_inc_and_test(v); +#else return raw_atomic_inc_return(v) == 0; -} #endif +} -#if defined(arch_atomic_add_negative) -#define raw_atomic_add_negative arch_atomic_add_negative -#elif defined(arch_atomic_add_negative_relaxed) static __always_inline bool raw_atomic_add_negative(int i, atomic_t *v) { +#if defined(arch_atomic_add_negative) + return arch_atomic_add_negative(i, v); +#elif defined(arch_atomic_add_negative_relaxed) bool ret; __atomic_pre_full_fence(); ret = arch_atomic_add_negative_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline bool -raw_atomic_add_negative(int i, atomic_t *v) -{ return raw_atomic_add_return(i, v) < 0; -} #endif +} -#if defined(arch_atomic_add_negative_acquire) -#define raw_atomic_add_negative_acquire arch_atomic_add_negative_acquire -#elif defined(arch_atomic_add_negative_relaxed) static __always_inline bool raw_atomic_add_negative_acquire(int i, atomic_t *v) { +#if defined(arch_atomic_add_negative_acquire) + return arch_atomic_add_negative_acquire(i, v); +#elif defined(arch_atomic_add_negative_relaxed) bool ret = arch_atomic_add_negative_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic_add_negative) -#define raw_atomic_add_negative_acquire arch_atomic_add_negative + return arch_atomic_add_negative(i, v); #else -static __always_inline bool -raw_atomic_add_negative_acquire(int i, atomic_t *v) -{ return raw_atomic_add_return_acquire(i, v) < 0; -} #endif +} -#if defined(arch_atomic_add_negative_release) -#define raw_atomic_add_negative_release arch_atomic_add_negative_release -#elif defined(arch_atomic_add_negative_relaxed) static __always_inline bool raw_atomic_add_negative_release(int i, atomic_t *v) { +#if defined(arch_atomic_add_negative_release) + return arch_atomic_add_negative_release(i, v); +#elif defined(arch_atomic_add_negative_relaxed) __atomic_release_fence(); return arch_atomic_add_negative_relaxed(i, v); -} #elif defined(arch_atomic_add_negative) -#define raw_atomic_add_negative_release arch_atomic_add_negative + return arch_atomic_add_negative(i, v); #else -static __always_inline bool -raw_atomic_add_negative_release(int i, atomic_t *v) -{ return raw_atomic_add_return_release(i, v) < 0; -} #endif +} -#if defined(arch_atomic_add_negative_relaxed) -#define raw_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed -#elif defined(arch_atomic_add_negative) -#define raw_atomic_add_negative_relaxed arch_atomic_add_negative -#else static __always_inline bool raw_atomic_add_negative_relaxed(int i, atomic_t *v) { +#if defined(arch_atomic_add_negative_relaxed) + return arch_atomic_add_negative_relaxed(i, v); +#elif defined(arch_atomic_add_negative) + return arch_atomic_add_negative(i, v); +#else return raw_atomic_add_return_relaxed(i, v) < 0; -} #endif +} -#if defined(arch_atomic_fetch_add_unless) -#define raw_atomic_fetch_add_unless arch_atomic_fetch_add_unless -#else static __always_inline int raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) { +#if defined(arch_atomic_fetch_add_unless) + return arch_atomic_fetch_add_unless(v, a, u); +#else int c = raw_atomic_read(v); do { @@ -1594,35 +1542,35 @@ raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) } while (!raw_atomic_try_cmpxchg(v, &c, c + a)); return c; -} #endif +} -#if defined(arch_atomic_add_unless) -#define raw_atomic_add_unless arch_atomic_add_unless -#else static __always_inline bool raw_atomic_add_unless(atomic_t *v, int a, int u) { +#if defined(arch_atomic_add_unless) + return arch_atomic_add_unless(v, a, u); +#else return raw_atomic_fetch_add_unless(v, a, u) != u; -} #endif +} -#if defined(arch_atomic_inc_not_zero) -#define raw_atomic_inc_not_zero arch_atomic_inc_not_zero -#else static __always_inline bool raw_atomic_inc_not_zero(atomic_t *v) { +#if defined(arch_atomic_inc_not_zero) + return arch_atomic_inc_not_zero(v); +#else return raw_atomic_add_unless(v, 1, 0); -} #endif +} -#if defined(arch_atomic_inc_unless_negative) -#define raw_atomic_inc_unless_negative arch_atomic_inc_unless_negative -#else static __always_inline bool raw_atomic_inc_unless_negative(atomic_t *v) { +#if defined(arch_atomic_inc_unless_negative) + return arch_atomic_inc_unless_negative(v); +#else int c = raw_atomic_read(v); do { @@ -1631,15 +1579,15 @@ raw_atomic_inc_unless_negative(atomic_t *v) } while (!raw_atomic_try_cmpxchg(v, &c, c + 1)); return true; -} #endif +} -#if defined(arch_atomic_dec_unless_positive) -#define raw_atomic_dec_unless_positive arch_atomic_dec_unless_positive -#else static __always_inline bool raw_atomic_dec_unless_positive(atomic_t *v) { +#if defined(arch_atomic_dec_unless_positive) + return arch_atomic_dec_unless_positive(v); +#else int c = raw_atomic_read(v); do { @@ -1648,15 +1596,15 @@ raw_atomic_dec_unless_positive(atomic_t *v) } while (!raw_atomic_try_cmpxchg(v, &c, c - 1)); return true; -} #endif +} -#if defined(arch_atomic_dec_if_positive) -#define raw_atomic_dec_if_positive arch_atomic_dec_if_positive -#else static __always_inline int raw_atomic_dec_if_positive(atomic_t *v) { +#if defined(arch_atomic_dec_if_positive) + return arch_atomic_dec_if_positive(v); +#else int dec, c = raw_atomic_read(v); do { @@ -1666,23 +1614,27 @@ raw_atomic_dec_if_positive(atomic_t *v) } while (!raw_atomic_try_cmpxchg(v, &c, dec)); return dec; -} #endif +} #ifdef CONFIG_GENERIC_ATOMIC64 #include #endif -#define raw_atomic64_read arch_atomic64_read +static __always_inline s64 +raw_atomic64_read(const atomic64_t *v) +{ + return arch_atomic64_read(v); +} -#if defined(arch_atomic64_read_acquire) -#define raw_atomic64_read_acquire arch_atomic64_read_acquire -#elif defined(arch_atomic64_read) -#define raw_atomic64_read_acquire arch_atomic64_read -#else static __always_inline s64 raw_atomic64_read_acquire(const atomic64_t *v) { +#if defined(arch_atomic64_read_acquire) + return arch_atomic64_read_acquire(v); +#elif defined(arch_atomic64_read) + return arch_atomic64_read(v); +#else s64 ret; if (__native_word(atomic64_t)) { @@ -1693,1144 +1645,1088 @@ raw_atomic64_read_acquire(const atomic64_t *v) } return ret; -} #endif +} -#define raw_atomic64_set arch_atomic64_set +static __always_inline void +raw_atomic64_set(atomic64_t *v, s64 i) +{ + arch_atomic64_set(v, i); +} -#if defined(arch_atomic64_set_release) -#define raw_atomic64_set_release arch_atomic64_set_release -#elif defined(arch_atomic64_set) -#define raw_atomic64_set_release arch_atomic64_set -#else static __always_inline void raw_atomic64_set_release(atomic64_t *v, s64 i) { +#if defined(arch_atomic64_set_release) + arch_atomic64_set_release(v, i); +#elif defined(arch_atomic64_set) + arch_atomic64_set(v, i); +#else if (__native_word(atomic64_t)) { smp_store_release(&(v)->counter, i); } else { __atomic_release_fence(); raw_atomic64_set(v, i); } -} #endif +} -#define raw_atomic64_add arch_atomic64_add +static __always_inline void +raw_atomic64_add(s64 i, atomic64_t *v) +{ + arch_atomic64_add(i, v); +} -#if defined(arch_atomic64_add_return) -#define raw_atomic64_add_return arch_atomic64_add_return -#elif defined(arch_atomic64_add_return_relaxed) static __always_inline s64 raw_atomic64_add_return(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_return) + return arch_atomic64_add_return(i, v); +#elif defined(arch_atomic64_add_return_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_add_return_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_add_return" #endif +} -#if defined(arch_atomic64_add_return_acquire) -#define raw_atomic64_add_return_acquire arch_atomic64_add_return_acquire -#elif defined(arch_atomic64_add_return_relaxed) static __always_inline s64 raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_return_acquire) + return arch_atomic64_add_return_acquire(i, v); +#elif defined(arch_atomic64_add_return_relaxed) s64 ret = arch_atomic64_add_return_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_add_return) -#define raw_atomic64_add_return_acquire arch_atomic64_add_return + return arch_atomic64_add_return(i, v); #else #error "Unable to define raw_atomic64_add_return_acquire" #endif +} -#if defined(arch_atomic64_add_return_release) -#define raw_atomic64_add_return_release arch_atomic64_add_return_release -#elif defined(arch_atomic64_add_return_relaxed) static __always_inline s64 raw_atomic64_add_return_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_return_release) + return arch_atomic64_add_return_release(i, v); +#elif defined(arch_atomic64_add_return_relaxed) __atomic_release_fence(); return arch_atomic64_add_return_relaxed(i, v); -} #elif defined(arch_atomic64_add_return) -#define raw_atomic64_add_return_release arch_atomic64_add_return + return arch_atomic64_add_return(i, v); #else #error "Unable to define raw_atomic64_add_return_release" #endif +} +static __always_inline s64 +raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_add_return_relaxed) -#define raw_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed + return arch_atomic64_add_return_relaxed(i, v); #elif defined(arch_atomic64_add_return) -#define raw_atomic64_add_return_relaxed arch_atomic64_add_return + return arch_atomic64_add_return(i, v); #else #error "Unable to define raw_atomic64_add_return_relaxed" #endif +} -#if defined(arch_atomic64_fetch_add) -#define raw_atomic64_fetch_add arch_atomic64_fetch_add -#elif defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 raw_atomic64_fetch_add(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_add) + return arch_atomic64_fetch_add(i, v); +#elif defined(arch_atomic64_fetch_add_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_add_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_fetch_add" #endif +} -#if defined(arch_atomic64_fetch_add_acquire) -#define raw_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire -#elif defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_add_acquire) + return arch_atomic64_fetch_add_acquire(i, v); +#elif defined(arch_atomic64_fetch_add_relaxed) s64 ret = arch_atomic64_fetch_add_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_add) -#define raw_atomic64_fetch_add_acquire arch_atomic64_fetch_add + return arch_atomic64_fetch_add(i, v); #else #error "Unable to define raw_atomic64_fetch_add_acquire" #endif +} -#if defined(arch_atomic64_fetch_add_release) -#define raw_atomic64_fetch_add_release arch_atomic64_fetch_add_release -#elif defined(arch_atomic64_fetch_add_relaxed) static __always_inline s64 raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_add_release) + return arch_atomic64_fetch_add_release(i, v); +#elif defined(arch_atomic64_fetch_add_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_add_relaxed(i, v); -} #elif defined(arch_atomic64_fetch_add) -#define raw_atomic64_fetch_add_release arch_atomic64_fetch_add + return arch_atomic64_fetch_add(i, v); #else #error "Unable to define raw_atomic64_fetch_add_release" #endif +} +static __always_inline s64 +raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_fetch_add_relaxed) -#define raw_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed + return arch_atomic64_fetch_add_relaxed(i, v); #elif defined(arch_atomic64_fetch_add) -#define raw_atomic64_fetch_add_relaxed arch_atomic64_fetch_add + return arch_atomic64_fetch_add(i, v); #else #error "Unable to define raw_atomic64_fetch_add_relaxed" #endif +} -#define raw_atomic64_sub arch_atomic64_sub +static __always_inline void +raw_atomic64_sub(s64 i, atomic64_t *v) +{ + arch_atomic64_sub(i, v); +} -#if defined(arch_atomic64_sub_return) -#define raw_atomic64_sub_return arch_atomic64_sub_return -#elif defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 raw_atomic64_sub_return(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_sub_return) + return arch_atomic64_sub_return(i, v); +#elif defined(arch_atomic64_sub_return_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_sub_return_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_sub_return" #endif +} -#if defined(arch_atomic64_sub_return_acquire) -#define raw_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire -#elif defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_sub_return_acquire) + return arch_atomic64_sub_return_acquire(i, v); +#elif defined(arch_atomic64_sub_return_relaxed) s64 ret = arch_atomic64_sub_return_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_sub_return) -#define raw_atomic64_sub_return_acquire arch_atomic64_sub_return + return arch_atomic64_sub_return(i, v); #else #error "Unable to define raw_atomic64_sub_return_acquire" #endif +} -#if defined(arch_atomic64_sub_return_release) -#define raw_atomic64_sub_return_release arch_atomic64_sub_return_release -#elif defined(arch_atomic64_sub_return_relaxed) static __always_inline s64 raw_atomic64_sub_return_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_sub_return_release) + return arch_atomic64_sub_return_release(i, v); +#elif defined(arch_atomic64_sub_return_relaxed) __atomic_release_fence(); return arch_atomic64_sub_return_relaxed(i, v); -} #elif defined(arch_atomic64_sub_return) -#define raw_atomic64_sub_return_release arch_atomic64_sub_return + return arch_atomic64_sub_return(i, v); #else #error "Unable to define raw_atomic64_sub_return_release" #endif +} +static __always_inline s64 +raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_sub_return_relaxed) -#define raw_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed + return arch_atomic64_sub_return_relaxed(i, v); #elif defined(arch_atomic64_sub_return) -#define raw_atomic64_sub_return_relaxed arch_atomic64_sub_return + return arch_atomic64_sub_return(i, v); #else #error "Unable to define raw_atomic64_sub_return_relaxed" #endif +} -#if defined(arch_atomic64_fetch_sub) -#define raw_atomic64_fetch_sub arch_atomic64_fetch_sub -#elif defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 raw_atomic64_fetch_sub(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_sub) + return arch_atomic64_fetch_sub(i, v); +#elif defined(arch_atomic64_fetch_sub_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_sub_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_fetch_sub" #endif +} -#if defined(arch_atomic64_fetch_sub_acquire) -#define raw_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire -#elif defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_sub_acquire) + return arch_atomic64_fetch_sub_acquire(i, v); +#elif defined(arch_atomic64_fetch_sub_relaxed) s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_sub) -#define raw_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub + return arch_atomic64_fetch_sub(i, v); #else #error "Unable to define raw_atomic64_fetch_sub_acquire" #endif +} -#if defined(arch_atomic64_fetch_sub_release) -#define raw_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release -#elif defined(arch_atomic64_fetch_sub_relaxed) static __always_inline s64 raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_sub_release) + return arch_atomic64_fetch_sub_release(i, v); +#elif defined(arch_atomic64_fetch_sub_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_sub_relaxed(i, v); -} #elif defined(arch_atomic64_fetch_sub) -#define raw_atomic64_fetch_sub_release arch_atomic64_fetch_sub + return arch_atomic64_fetch_sub(i, v); #else #error "Unable to define raw_atomic64_fetch_sub_release" #endif +} +static __always_inline s64 +raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_fetch_sub_relaxed) -#define raw_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed + return arch_atomic64_fetch_sub_relaxed(i, v); #elif defined(arch_atomic64_fetch_sub) -#define raw_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub + return arch_atomic64_fetch_sub(i, v); #else #error "Unable to define raw_atomic64_fetch_sub_relaxed" #endif +} -#if defined(arch_atomic64_inc) -#define raw_atomic64_inc arch_atomic64_inc -#else static __always_inline void raw_atomic64_inc(atomic64_t *v) { +#if defined(arch_atomic64_inc) + arch_atomic64_inc(v); +#else raw_atomic64_add(1, v); -} #endif +} -#if defined(arch_atomic64_inc_return) -#define raw_atomic64_inc_return arch_atomic64_inc_return -#elif defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 raw_atomic64_inc_return(atomic64_t *v) { +#if defined(arch_atomic64_inc_return) + return arch_atomic64_inc_return(v); +#elif defined(arch_atomic64_inc_return_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_inc_return_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_inc_return(atomic64_t *v) -{ return raw_atomic64_add_return(1, v); -} #endif +} -#if defined(arch_atomic64_inc_return_acquire) -#define raw_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire -#elif defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 raw_atomic64_inc_return_acquire(atomic64_t *v) { +#if defined(arch_atomic64_inc_return_acquire) + return arch_atomic64_inc_return_acquire(v); +#elif defined(arch_atomic64_inc_return_relaxed) s64 ret = arch_atomic64_inc_return_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_inc_return) -#define raw_atomic64_inc_return_acquire arch_atomic64_inc_return + return arch_atomic64_inc_return(v); #else -static __always_inline s64 -raw_atomic64_inc_return_acquire(atomic64_t *v) -{ return raw_atomic64_add_return_acquire(1, v); -} #endif +} -#if defined(arch_atomic64_inc_return_release) -#define raw_atomic64_inc_return_release arch_atomic64_inc_return_release -#elif defined(arch_atomic64_inc_return_relaxed) static __always_inline s64 raw_atomic64_inc_return_release(atomic64_t *v) { +#if defined(arch_atomic64_inc_return_release) + return arch_atomic64_inc_return_release(v); +#elif defined(arch_atomic64_inc_return_relaxed) __atomic_release_fence(); return arch_atomic64_inc_return_relaxed(v); -} #elif defined(arch_atomic64_inc_return) -#define raw_atomic64_inc_return_release arch_atomic64_inc_return + return arch_atomic64_inc_return(v); #else -static __always_inline s64 -raw_atomic64_inc_return_release(atomic64_t *v) -{ return raw_atomic64_add_return_release(1, v); -} #endif +} -#if defined(arch_atomic64_inc_return_relaxed) -#define raw_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed -#elif defined(arch_atomic64_inc_return) -#define raw_atomic64_inc_return_relaxed arch_atomic64_inc_return -#else static __always_inline s64 raw_atomic64_inc_return_relaxed(atomic64_t *v) { +#if defined(arch_atomic64_inc_return_relaxed) + return arch_atomic64_inc_return_relaxed(v); +#elif defined(arch_atomic64_inc_return) + return arch_atomic64_inc_return(v); +#else return raw_atomic64_add_return_relaxed(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_inc) -#define raw_atomic64_fetch_inc arch_atomic64_fetch_inc -#elif defined(arch_atomic64_fetch_inc_relaxed) static __always_inline s64 raw_atomic64_fetch_inc(atomic64_t *v) { +#if defined(arch_atomic64_fetch_inc) + return arch_atomic64_fetch_inc(v); +#elif defined(arch_atomic64_fetch_inc_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_inc_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_fetch_inc(atomic64_t *v) -{ return raw_atomic64_fetch_add(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_inc_acquire) -#define raw_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire -#elif defined(arch_atomic64_fetch_inc_relaxed) static __always_inline s64 raw_atomic64_fetch_inc_acquire(atomic64_t *v) { +#if defined(arch_atomic64_fetch_inc_acquire) + return arch_atomic64_fetch_inc_acquire(v); +#elif defined(arch_atomic64_fetch_inc_relaxed) s64 ret = arch_atomic64_fetch_inc_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_inc) -#define raw_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc + return arch_atomic64_fetch_inc(v); #else -static __always_inline s64 -raw_atomic64_fetch_inc_acquire(atomic64_t *v) -{ return raw_atomic64_fetch_add_acquire(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_inc_release) -#define raw_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release -#elif defined(arch_atomic64_fetch_inc_relaxed) static __always_inline s64 raw_atomic64_fetch_inc_release(atomic64_t *v) { +#if defined(arch_atomic64_fetch_inc_release) + return arch_atomic64_fetch_inc_release(v); +#elif defined(arch_atomic64_fetch_inc_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_inc_relaxed(v); -} #elif defined(arch_atomic64_fetch_inc) -#define raw_atomic64_fetch_inc_release arch_atomic64_fetch_inc + return arch_atomic64_fetch_inc(v); #else -static __always_inline s64 -raw_atomic64_fetch_inc_release(atomic64_t *v) -{ return raw_atomic64_fetch_add_release(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_inc_relaxed) -#define raw_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed -#elif defined(arch_atomic64_fetch_inc) -#define raw_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc -#else static __always_inline s64 raw_atomic64_fetch_inc_relaxed(atomic64_t *v) { +#if defined(arch_atomic64_fetch_inc_relaxed) + return arch_atomic64_fetch_inc_relaxed(v); +#elif defined(arch_atomic64_fetch_inc) + return arch_atomic64_fetch_inc(v); +#else return raw_atomic64_fetch_add_relaxed(1, v); -} #endif +} -#if defined(arch_atomic64_dec) -#define raw_atomic64_dec arch_atomic64_dec -#else static __always_inline void raw_atomic64_dec(atomic64_t *v) { +#if defined(arch_atomic64_dec) + arch_atomic64_dec(v); +#else raw_atomic64_sub(1, v); -} #endif +} -#if defined(arch_atomic64_dec_return) -#define raw_atomic64_dec_return arch_atomic64_dec_return -#elif defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 raw_atomic64_dec_return(atomic64_t *v) { +#if defined(arch_atomic64_dec_return) + return arch_atomic64_dec_return(v); +#elif defined(arch_atomic64_dec_return_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_dec_return_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_dec_return(atomic64_t *v) -{ return raw_atomic64_sub_return(1, v); -} #endif +} -#if defined(arch_atomic64_dec_return_acquire) -#define raw_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire -#elif defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 raw_atomic64_dec_return_acquire(atomic64_t *v) { +#if defined(arch_atomic64_dec_return_acquire) + return arch_atomic64_dec_return_acquire(v); +#elif defined(arch_atomic64_dec_return_relaxed) s64 ret = arch_atomic64_dec_return_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_dec_return) -#define raw_atomic64_dec_return_acquire arch_atomic64_dec_return + return arch_atomic64_dec_return(v); #else -static __always_inline s64 -raw_atomic64_dec_return_acquire(atomic64_t *v) -{ return raw_atomic64_sub_return_acquire(1, v); -} #endif +} -#if defined(arch_atomic64_dec_return_release) -#define raw_atomic64_dec_return_release arch_atomic64_dec_return_release -#elif defined(arch_atomic64_dec_return_relaxed) static __always_inline s64 raw_atomic64_dec_return_release(atomic64_t *v) { +#if defined(arch_atomic64_dec_return_release) + return arch_atomic64_dec_return_release(v); +#elif defined(arch_atomic64_dec_return_relaxed) __atomic_release_fence(); return arch_atomic64_dec_return_relaxed(v); -} #elif defined(arch_atomic64_dec_return) -#define raw_atomic64_dec_return_release arch_atomic64_dec_return + return arch_atomic64_dec_return(v); #else -static __always_inline s64 -raw_atomic64_dec_return_release(atomic64_t *v) -{ return raw_atomic64_sub_return_release(1, v); -} #endif +} -#if defined(arch_atomic64_dec_return_relaxed) -#define raw_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed -#elif defined(arch_atomic64_dec_return) -#define raw_atomic64_dec_return_relaxed arch_atomic64_dec_return -#else static __always_inline s64 raw_atomic64_dec_return_relaxed(atomic64_t *v) { +#if defined(arch_atomic64_dec_return_relaxed) + return arch_atomic64_dec_return_relaxed(v); +#elif defined(arch_atomic64_dec_return) + return arch_atomic64_dec_return(v); +#else return raw_atomic64_sub_return_relaxed(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_dec) -#define raw_atomic64_fetch_dec arch_atomic64_fetch_dec -#elif defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 raw_atomic64_fetch_dec(atomic64_t *v) { +#if defined(arch_atomic64_fetch_dec) + return arch_atomic64_fetch_dec(v); +#elif defined(arch_atomic64_fetch_dec_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_dec_relaxed(v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_fetch_dec(atomic64_t *v) -{ return raw_atomic64_fetch_sub(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_dec_acquire) -#define raw_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire -#elif defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 raw_atomic64_fetch_dec_acquire(atomic64_t *v) { +#if defined(arch_atomic64_fetch_dec_acquire) + return arch_atomic64_fetch_dec_acquire(v); +#elif defined(arch_atomic64_fetch_dec_relaxed) s64 ret = arch_atomic64_fetch_dec_relaxed(v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_dec) -#define raw_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec + return arch_atomic64_fetch_dec(v); #else -static __always_inline s64 -raw_atomic64_fetch_dec_acquire(atomic64_t *v) -{ return raw_atomic64_fetch_sub_acquire(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_dec_release) -#define raw_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release -#elif defined(arch_atomic64_fetch_dec_relaxed) static __always_inline s64 raw_atomic64_fetch_dec_release(atomic64_t *v) { +#if defined(arch_atomic64_fetch_dec_release) + return arch_atomic64_fetch_dec_release(v); +#elif defined(arch_atomic64_fetch_dec_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_dec_relaxed(v); -} #elif defined(arch_atomic64_fetch_dec) -#define raw_atomic64_fetch_dec_release arch_atomic64_fetch_dec + return arch_atomic64_fetch_dec(v); #else -static __always_inline s64 -raw_atomic64_fetch_dec_release(atomic64_t *v) -{ return raw_atomic64_fetch_sub_release(1, v); -} #endif +} -#if defined(arch_atomic64_fetch_dec_relaxed) -#define raw_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed -#elif defined(arch_atomic64_fetch_dec) -#define raw_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec -#else static __always_inline s64 raw_atomic64_fetch_dec_relaxed(atomic64_t *v) { +#if defined(arch_atomic64_fetch_dec_relaxed) + return arch_atomic64_fetch_dec_relaxed(v); +#elif defined(arch_atomic64_fetch_dec) + return arch_atomic64_fetch_dec(v); +#else return raw_atomic64_fetch_sub_relaxed(1, v); -} #endif +} -#define raw_atomic64_and arch_atomic64_and +static __always_inline void +raw_atomic64_and(s64 i, atomic64_t *v) +{ + arch_atomic64_and(i, v); +} -#if defined(arch_atomic64_fetch_and) -#define raw_atomic64_fetch_and arch_atomic64_fetch_and -#elif defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 raw_atomic64_fetch_and(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_and) + return arch_atomic64_fetch_and(i, v); +#elif defined(arch_atomic64_fetch_and_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_and_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_fetch_and" #endif +} -#if defined(arch_atomic64_fetch_and_acquire) -#define raw_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire -#elif defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_and_acquire) + return arch_atomic64_fetch_and_acquire(i, v); +#elif defined(arch_atomic64_fetch_and_relaxed) s64 ret = arch_atomic64_fetch_and_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_and) -#define raw_atomic64_fetch_and_acquire arch_atomic64_fetch_and + return arch_atomic64_fetch_and(i, v); #else #error "Unable to define raw_atomic64_fetch_and_acquire" #endif +} -#if defined(arch_atomic64_fetch_and_release) -#define raw_atomic64_fetch_and_release arch_atomic64_fetch_and_release -#elif defined(arch_atomic64_fetch_and_relaxed) static __always_inline s64 raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_and_release) + return arch_atomic64_fetch_and_release(i, v); +#elif defined(arch_atomic64_fetch_and_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_and_relaxed(i, v); -} #elif defined(arch_atomic64_fetch_and) -#define raw_atomic64_fetch_and_release arch_atomic64_fetch_and + return arch_atomic64_fetch_and(i, v); #else #error "Unable to define raw_atomic64_fetch_and_release" #endif +} +static __always_inline s64 +raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_fetch_and_relaxed) -#define raw_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed + return arch_atomic64_fetch_and_relaxed(i, v); #elif defined(arch_atomic64_fetch_and) -#define raw_atomic64_fetch_and_relaxed arch_atomic64_fetch_and + return arch_atomic64_fetch_and(i, v); #else #error "Unable to define raw_atomic64_fetch_and_relaxed" #endif +} -#if defined(arch_atomic64_andnot) -#define raw_atomic64_andnot arch_atomic64_andnot -#else static __always_inline void raw_atomic64_andnot(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_andnot) + arch_atomic64_andnot(i, v); +#else raw_atomic64_and(~i, v); -} #endif +} -#if defined(arch_atomic64_fetch_andnot) -#define raw_atomic64_fetch_andnot arch_atomic64_fetch_andnot -#elif defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_andnot) + return arch_atomic64_fetch_andnot(i, v); +#elif defined(arch_atomic64_fetch_andnot_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_andnot_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) -{ return raw_atomic64_fetch_and(~i, v); -} #endif +} -#if defined(arch_atomic64_fetch_andnot_acquire) -#define raw_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire -#elif defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_andnot_acquire) + return arch_atomic64_fetch_andnot_acquire(i, v); +#elif defined(arch_atomic64_fetch_andnot_relaxed) s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_andnot) -#define raw_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot + return arch_atomic64_fetch_andnot(i, v); #else -static __always_inline s64 -raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) -{ return raw_atomic64_fetch_and_acquire(~i, v); -} #endif +} -#if defined(arch_atomic64_fetch_andnot_release) -#define raw_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release -#elif defined(arch_atomic64_fetch_andnot_relaxed) static __always_inline s64 raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_andnot_release) + return arch_atomic64_fetch_andnot_release(i, v); +#elif defined(arch_atomic64_fetch_andnot_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_andnot_relaxed(i, v); -} #elif defined(arch_atomic64_fetch_andnot) -#define raw_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot + return arch_atomic64_fetch_andnot(i, v); #else -static __always_inline s64 -raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) -{ return raw_atomic64_fetch_and_release(~i, v); -} #endif +} -#if defined(arch_atomic64_fetch_andnot_relaxed) -#define raw_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed -#elif defined(arch_atomic64_fetch_andnot) -#define raw_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot -#else static __always_inline s64 raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_andnot_relaxed) + return arch_atomic64_fetch_andnot_relaxed(i, v); +#elif defined(arch_atomic64_fetch_andnot) + return arch_atomic64_fetch_andnot(i, v); +#else return raw_atomic64_fetch_and_relaxed(~i, v); -} #endif +} -#define raw_atomic64_or arch_atomic64_or +static __always_inline void +raw_atomic64_or(s64 i, atomic64_t *v) +{ + arch_atomic64_or(i, v); +} -#if defined(arch_atomic64_fetch_or) -#define raw_atomic64_fetch_or arch_atomic64_fetch_or -#elif defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 raw_atomic64_fetch_or(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_or) + return arch_atomic64_fetch_or(i, v); +#elif defined(arch_atomic64_fetch_or_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_or_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_fetch_or" #endif +} -#if defined(arch_atomic64_fetch_or_acquire) -#define raw_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire -#elif defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_or_acquire) + return arch_atomic64_fetch_or_acquire(i, v); +#elif defined(arch_atomic64_fetch_or_relaxed) s64 ret = arch_atomic64_fetch_or_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_or) -#define raw_atomic64_fetch_or_acquire arch_atomic64_fetch_or + return arch_atomic64_fetch_or(i, v); #else #error "Unable to define raw_atomic64_fetch_or_acquire" #endif +} -#if defined(arch_atomic64_fetch_or_release) -#define raw_atomic64_fetch_or_release arch_atomic64_fetch_or_release -#elif defined(arch_atomic64_fetch_or_relaxed) static __always_inline s64 raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_or_release) + return arch_atomic64_fetch_or_release(i, v); +#elif defined(arch_atomic64_fetch_or_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_or_relaxed(i, v); -} #elif defined(arch_atomic64_fetch_or) -#define raw_atomic64_fetch_or_release arch_atomic64_fetch_or + return arch_atomic64_fetch_or(i, v); #else #error "Unable to define raw_atomic64_fetch_or_release" #endif +} +static __always_inline s64 +raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_fetch_or_relaxed) -#define raw_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed + return arch_atomic64_fetch_or_relaxed(i, v); #elif defined(arch_atomic64_fetch_or) -#define raw_atomic64_fetch_or_relaxed arch_atomic64_fetch_or + return arch_atomic64_fetch_or(i, v); #else #error "Unable to define raw_atomic64_fetch_or_relaxed" #endif +} -#define raw_atomic64_xor arch_atomic64_xor +static __always_inline void +raw_atomic64_xor(s64 i, atomic64_t *v) +{ + arch_atomic64_xor(i, v); +} -#if defined(arch_atomic64_fetch_xor) -#define raw_atomic64_fetch_xor arch_atomic64_fetch_xor -#elif defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 raw_atomic64_fetch_xor(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_xor) + return arch_atomic64_fetch_xor(i, v); +#elif defined(arch_atomic64_fetch_xor_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_fetch_xor_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else #error "Unable to define raw_atomic64_fetch_xor" #endif +} -#if defined(arch_atomic64_fetch_xor_acquire) -#define raw_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire -#elif defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_xor_acquire) + return arch_atomic64_fetch_xor_acquire(i, v); +#elif defined(arch_atomic64_fetch_xor_relaxed) s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_fetch_xor) -#define raw_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor + return arch_atomic64_fetch_xor(i, v); #else #error "Unable to define raw_atomic64_fetch_xor_acquire" #endif +} -#if defined(arch_atomic64_fetch_xor_release) -#define raw_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release -#elif defined(arch_atomic64_fetch_xor_relaxed) static __always_inline s64 raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_fetch_xor_release) + return arch_atomic64_fetch_xor_release(i, v); +#elif defined(arch_atomic64_fetch_xor_relaxed) __atomic_release_fence(); return arch_atomic64_fetch_xor_relaxed(i, v); -} #elif defined(arch_atomic64_fetch_xor) -#define raw_atomic64_fetch_xor_release arch_atomic64_fetch_xor + return arch_atomic64_fetch_xor(i, v); #else #error "Unable to define raw_atomic64_fetch_xor_release" #endif +} +static __always_inline s64 +raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) +{ #if defined(arch_atomic64_fetch_xor_relaxed) -#define raw_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed + return arch_atomic64_fetch_xor_relaxed(i, v); #elif defined(arch_atomic64_fetch_xor) -#define raw_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor + return arch_atomic64_fetch_xor(i, v); #else #error "Unable to define raw_atomic64_fetch_xor_relaxed" #endif +} -#if defined(arch_atomic64_xchg) -#define raw_atomic64_xchg arch_atomic64_xchg -#elif defined(arch_atomic64_xchg_relaxed) static __always_inline s64 -raw_atomic64_xchg(atomic64_t *v, s64 i) +raw_atomic64_xchg(atomic64_t *v, s64 new) { +#if defined(arch_atomic64_xchg) + return arch_atomic64_xchg(v, new); +#elif defined(arch_atomic64_xchg_relaxed) s64 ret; __atomic_pre_full_fence(); - ret = arch_atomic64_xchg_relaxed(v, i); + ret = arch_atomic64_xchg_relaxed(v, new); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_xchg(atomic64_t *v, s64 new) -{ return raw_xchg(&v->counter, new); -} #endif +} -#if defined(arch_atomic64_xchg_acquire) -#define raw_atomic64_xchg_acquire arch_atomic64_xchg_acquire -#elif defined(arch_atomic64_xchg_relaxed) static __always_inline s64 -raw_atomic64_xchg_acquire(atomic64_t *v, s64 i) +raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) { - s64 ret = arch_atomic64_xchg_relaxed(v, i); +#if defined(arch_atomic64_xchg_acquire) + return arch_atomic64_xchg_acquire(v, new); +#elif defined(arch_atomic64_xchg_relaxed) + s64 ret = arch_atomic64_xchg_relaxed(v, new); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_xchg) -#define raw_atomic64_xchg_acquire arch_atomic64_xchg + return arch_atomic64_xchg(v, new); #else -static __always_inline s64 -raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) -{ return raw_xchg_acquire(&v->counter, new); -} #endif +} -#if defined(arch_atomic64_xchg_release) -#define raw_atomic64_xchg_release arch_atomic64_xchg_release -#elif defined(arch_atomic64_xchg_relaxed) static __always_inline s64 -raw_atomic64_xchg_release(atomic64_t *v, s64 i) +raw_atomic64_xchg_release(atomic64_t *v, s64 new) { +#if defined(arch_atomic64_xchg_release) + return arch_atomic64_xchg_release(v, new); +#elif defined(arch_atomic64_xchg_relaxed) __atomic_release_fence(); - return arch_atomic64_xchg_relaxed(v, i); -} + return arch_atomic64_xchg_relaxed(v, new); #elif defined(arch_atomic64_xchg) -#define raw_atomic64_xchg_release arch_atomic64_xchg + return arch_atomic64_xchg(v, new); #else -static __always_inline s64 -raw_atomic64_xchg_release(atomic64_t *v, s64 new) -{ return raw_xchg_release(&v->counter, new); -} #endif +} -#if defined(arch_atomic64_xchg_relaxed) -#define raw_atomic64_xchg_relaxed arch_atomic64_xchg_relaxed -#elif defined(arch_atomic64_xchg) -#define raw_atomic64_xchg_relaxed arch_atomic64_xchg -#else static __always_inline s64 raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) { +#if defined(arch_atomic64_xchg_relaxed) + return arch_atomic64_xchg_relaxed(v, new); +#elif defined(arch_atomic64_xchg) + return arch_atomic64_xchg(v, new); +#else return raw_xchg_relaxed(&v->counter, new); -} #endif +} -#if defined(arch_atomic64_cmpxchg) -#define raw_atomic64_cmpxchg arch_atomic64_cmpxchg -#elif defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { +#if defined(arch_atomic64_cmpxchg) + return arch_atomic64_cmpxchg(v, old, new); +#elif defined(arch_atomic64_cmpxchg_relaxed) s64 ret; __atomic_pre_full_fence(); ret = arch_atomic64_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; -} #else -static __always_inline s64 -raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) -{ return raw_cmpxchg(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic64_cmpxchg_acquire) -#define raw_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire -#elif defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { +#if defined(arch_atomic64_cmpxchg_acquire) + return arch_atomic64_cmpxchg_acquire(v, old, new); +#elif defined(arch_atomic64_cmpxchg_relaxed) s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_cmpxchg) -#define raw_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg + return arch_atomic64_cmpxchg(v, old, new); #else -static __always_inline s64 -raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) -{ return raw_cmpxchg_acquire(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic64_cmpxchg_release) -#define raw_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release -#elif defined(arch_atomic64_cmpxchg_relaxed) static __always_inline s64 raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { +#if defined(arch_atomic64_cmpxchg_release) + return arch_atomic64_cmpxchg_release(v, old, new); +#elif defined(arch_atomic64_cmpxchg_relaxed) __atomic_release_fence(); return arch_atomic64_cmpxchg_relaxed(v, old, new); -} #elif defined(arch_atomic64_cmpxchg) -#define raw_atomic64_cmpxchg_release arch_atomic64_cmpxchg + return arch_atomic64_cmpxchg(v, old, new); #else -static __always_inline s64 -raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) -{ return raw_cmpxchg_release(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic64_cmpxchg_relaxed) -#define raw_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed -#elif defined(arch_atomic64_cmpxchg) -#define raw_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg -#else static __always_inline s64 raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { +#if defined(arch_atomic64_cmpxchg_relaxed) + return arch_atomic64_cmpxchg_relaxed(v, old, new); +#elif defined(arch_atomic64_cmpxchg) + return arch_atomic64_cmpxchg(v, old, new); +#else return raw_cmpxchg_relaxed(&v->counter, old, new); -} #endif +} -#if defined(arch_atomic64_try_cmpxchg) -#define raw_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg -#elif defined(arch_atomic64_try_cmpxchg_relaxed) static __always_inline bool raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { +#if defined(arch_atomic64_try_cmpxchg) + return arch_atomic64_try_cmpxchg(v, old, new); +#elif defined(arch_atomic64_try_cmpxchg_relaxed) bool ret; __atomic_pre_full_fence(); ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); __atomic_post_full_fence(); return ret; -} #else -static __always_inline bool -raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) -{ s64 r, o = *old; r = raw_atomic64_cmpxchg(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic64_try_cmpxchg_acquire) -#define raw_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire -#elif defined(arch_atomic64_try_cmpxchg_relaxed) static __always_inline bool raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { +#if defined(arch_atomic64_try_cmpxchg_acquire) + return arch_atomic64_try_cmpxchg_acquire(v, old, new); +#elif defined(arch_atomic64_try_cmpxchg_relaxed) bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_try_cmpxchg) -#define raw_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg + return arch_atomic64_try_cmpxchg(v, old, new); #else -static __always_inline bool -raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) -{ s64 r, o = *old; r = raw_atomic64_cmpxchg_acquire(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic64_try_cmpxchg_release) -#define raw_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release -#elif defined(arch_atomic64_try_cmpxchg_relaxed) static __always_inline bool raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { +#if defined(arch_atomic64_try_cmpxchg_release) + return arch_atomic64_try_cmpxchg_release(v, old, new); +#elif defined(arch_atomic64_try_cmpxchg_relaxed) __atomic_release_fence(); return arch_atomic64_try_cmpxchg_relaxed(v, old, new); -} #elif defined(arch_atomic64_try_cmpxchg) -#define raw_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg + return arch_atomic64_try_cmpxchg(v, old, new); #else -static __always_inline bool -raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) -{ s64 r, o = *old; r = raw_atomic64_cmpxchg_release(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic64_try_cmpxchg_relaxed) -#define raw_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed -#elif defined(arch_atomic64_try_cmpxchg) -#define raw_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg -#else static __always_inline bool raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { +#if defined(arch_atomic64_try_cmpxchg_relaxed) + return arch_atomic64_try_cmpxchg_relaxed(v, old, new); +#elif defined(arch_atomic64_try_cmpxchg) + return arch_atomic64_try_cmpxchg(v, old, new); +#else s64 r, o = *old; r = raw_atomic64_cmpxchg_relaxed(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); -} #endif +} -#if defined(arch_atomic64_sub_and_test) -#define raw_atomic64_sub_and_test arch_atomic64_sub_and_test -#else static __always_inline bool raw_atomic64_sub_and_test(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_sub_and_test) + return arch_atomic64_sub_and_test(i, v); +#else return raw_atomic64_sub_return(i, v) == 0; -} #endif +} -#if defined(arch_atomic64_dec_and_test) -#define raw_atomic64_dec_and_test arch_atomic64_dec_and_test -#else static __always_inline bool raw_atomic64_dec_and_test(atomic64_t *v) { +#if defined(arch_atomic64_dec_and_test) + return arch_atomic64_dec_and_test(v); +#else return raw_atomic64_dec_return(v) == 0; -} #endif +} -#if defined(arch_atomic64_inc_and_test) -#define raw_atomic64_inc_and_test arch_atomic64_inc_and_test -#else static __always_inline bool raw_atomic64_inc_and_test(atomic64_t *v) { +#if defined(arch_atomic64_inc_and_test) + return arch_atomic64_inc_and_test(v); +#else return raw_atomic64_inc_return(v) == 0; -} #endif +} -#if defined(arch_atomic64_add_negative) -#define raw_atomic64_add_negative arch_atomic64_add_negative -#elif defined(arch_atomic64_add_negative_relaxed) static __always_inline bool raw_atomic64_add_negative(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_negative) + return arch_atomic64_add_negative(i, v); +#elif defined(arch_atomic64_add_negative_relaxed) bool ret; __atomic_pre_full_fence(); ret = arch_atomic64_add_negative_relaxed(i, v); __atomic_post_full_fence(); return ret; -} #else -static __always_inline bool -raw_atomic64_add_negative(s64 i, atomic64_t *v) -{ return raw_atomic64_add_return(i, v) < 0; -} #endif +} -#if defined(arch_atomic64_add_negative_acquire) -#define raw_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire -#elif defined(arch_atomic64_add_negative_relaxed) static __always_inline bool raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_negative_acquire) + return arch_atomic64_add_negative_acquire(i, v); +#elif defined(arch_atomic64_add_negative_relaxed) bool ret = arch_atomic64_add_negative_relaxed(i, v); __atomic_acquire_fence(); return ret; -} #elif defined(arch_atomic64_add_negative) -#define raw_atomic64_add_negative_acquire arch_atomic64_add_negative + return arch_atomic64_add_negative(i, v); #else -static __always_inline bool -raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) -{ return raw_atomic64_add_return_acquire(i, v) < 0; -} #endif +} -#if defined(arch_atomic64_add_negative_release) -#define raw_atomic64_add_negative_release arch_atomic64_add_negative_release -#elif defined(arch_atomic64_add_negative_relaxed) static __always_inline bool raw_atomic64_add_negative_release(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_negative_release) + return arch_atomic64_add_negative_release(i, v); +#elif defined(arch_atomic64_add_negative_relaxed) __atomic_release_fence(); return arch_atomic64_add_negative_relaxed(i, v); -} #elif defined(arch_atomic64_add_negative) -#define raw_atomic64_add_negative_release arch_atomic64_add_negative + return arch_atomic64_add_negative(i, v); #else -static __always_inline bool -raw_atomic64_add_negative_release(s64 i, atomic64_t *v) -{ return raw_atomic64_add_return_release(i, v) < 0; -} #endif +} -#if defined(arch_atomic64_add_negative_relaxed) -#define raw_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed -#elif defined(arch_atomic64_add_negative) -#define raw_atomic64_add_negative_relaxed arch_atomic64_add_negative -#else static __always_inline bool raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) { +#if defined(arch_atomic64_add_negative_relaxed) + return arch_atomic64_add_negative_relaxed(i, v); +#elif defined(arch_atomic64_add_negative) + return arch_atomic64_add_negative(i, v); +#else return raw_atomic64_add_return_relaxed(i, v) < 0; -} #endif +} -#if defined(arch_atomic64_fetch_add_unless) -#define raw_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless -#else static __always_inline s64 raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { +#if defined(arch_atomic64_fetch_add_unless) + return arch_atomic64_fetch_add_unless(v, a, u); +#else s64 c = raw_atomic64_read(v); do { @@ -2839,35 +2735,35 @@ raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) } while (!raw_atomic64_try_cmpxchg(v, &c, c + a)); return c; -} #endif +} -#if defined(arch_atomic64_add_unless) -#define raw_atomic64_add_unless arch_atomic64_add_unless -#else static __always_inline bool raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { +#if defined(arch_atomic64_add_unless) + return arch_atomic64_add_unless(v, a, u); +#else return raw_atomic64_fetch_add_unless(v, a, u) != u; -} #endif +} -#if defined(arch_atomic64_inc_not_zero) -#define raw_atomic64_inc_not_zero arch_atomic64_inc_not_zero -#else static __always_inline bool raw_atomic64_inc_not_zero(atomic64_t *v) { +#if defined(arch_atomic64_inc_not_zero) + return arch_atomic64_inc_not_zero(v); +#else return raw_atomic64_add_unless(v, 1, 0); -} #endif +} -#if defined(arch_atomic64_inc_unless_negative) -#define raw_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative -#else static __always_inline bool raw_atomic64_inc_unless_negative(atomic64_t *v) { +#if defined(arch_atomic64_inc_unless_negative) + return arch_atomic64_inc_unless_negative(v); +#else s64 c = raw_atomic64_read(v); do { @@ -2876,15 +2772,15 @@ raw_atomic64_inc_unless_negative(atomic64_t *v) } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1)); return true; -} #endif +} -#if defined(arch_atomic64_dec_unless_positive) -#define raw_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive -#else static __always_inline bool raw_atomic64_dec_unless_positive(atomic64_t *v) { +#if defined(arch_atomic64_dec_unless_positive) + return arch_atomic64_dec_unless_positive(v); +#else s64 c = raw_atomic64_read(v); do { @@ -2893,15 +2789,15 @@ raw_atomic64_dec_unless_positive(atomic64_t *v) } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1)); return true; -} #endif +} -#if defined(arch_atomic64_dec_if_positive) -#define raw_atomic64_dec_if_positive arch_atomic64_dec_if_positive -#else static __always_inline s64 raw_atomic64_dec_if_positive(atomic64_t *v) { +#if defined(arch_atomic64_dec_if_positive) + return arch_atomic64_dec_if_positive(v); +#else s64 dec, c = raw_atomic64_read(v); do { @@ -2911,8 +2807,8 @@ raw_atomic64_dec_if_positive(atomic64_t *v) } while (!raw_atomic64_try_cmpxchg(v, &c, dec)); return dec; -} #endif +} #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// c2048fccede6fac923252290e2b303949d5dec83 +// 205e090382132f1fc85e48b46e722865f9c81309 diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 90ee2f55af77..5491c89dc03a 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -462,33 +462,33 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v) } static __always_inline int -atomic_xchg(atomic_t *v, int i) +atomic_xchg(atomic_t *v, int new) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_xchg(v, i); + return raw_atomic_xchg(v, new); } static __always_inline int -atomic_xchg_acquire(atomic_t *v, int i) +atomic_xchg_acquire(atomic_t *v, int new) { instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_xchg_acquire(v, i); + return raw_atomic_xchg_acquire(v, new); } static __always_inline int -atomic_xchg_release(atomic_t *v, int i) +atomic_xchg_release(atomic_t *v, int new) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_xchg_release(v, i); + return raw_atomic_xchg_release(v, new); } static __always_inline int -atomic_xchg_relaxed(atomic_t *v, int i) +atomic_xchg_relaxed(atomic_t *v, int new) { instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_xchg_relaxed(v, i); + return raw_atomic_xchg_relaxed(v, new); } static __always_inline int @@ -1103,33 +1103,33 @@ atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) } static __always_inline s64 -atomic64_xchg(atomic64_t *v, s64 i) +atomic64_xchg(atomic64_t *v, s64 new) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic64_xchg(v, i); + return raw_atomic64_xchg(v, new); } static __always_inline s64 -atomic64_xchg_acquire(atomic64_t *v, s64 i) +atomic64_xchg_acquire(atomic64_t *v, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic64_xchg_acquire(v, i); + return raw_atomic64_xchg_acquire(v, new); } static __always_inline s64 -atomic64_xchg_release(atomic64_t *v, s64 i) +atomic64_xchg_release(atomic64_t *v, s64 new) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic64_xchg_release(v, i); + return raw_atomic64_xchg_release(v, new); } static __always_inline s64 -atomic64_xchg_relaxed(atomic64_t *v, s64 i) +atomic64_xchg_relaxed(atomic64_t *v, s64 new) { instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic64_xchg_relaxed(v, i); + return raw_atomic64_xchg_relaxed(v, new); } static __always_inline s64 @@ -1744,33 +1744,33 @@ atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) } static __always_inline long -atomic_long_xchg(atomic_long_t *v, long i) +atomic_long_xchg(atomic_long_t *v, long new) { kcsan_mb(); instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_long_xchg(v, i); + return raw_atomic_long_xchg(v, new); } static __always_inline long -atomic_long_xchg_acquire(atomic_long_t *v, long i) +atomic_long_xchg_acquire(atomic_long_t *v, long new) { instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_long_xchg_acquire(v, i); + return raw_atomic_long_xchg_acquire(v, new); } static __always_inline long -atomic_long_xchg_release(atomic_long_t *v, long i) +atomic_long_xchg_release(atomic_long_t *v, long new) { kcsan_release(); instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_long_xchg_release(v, i); + return raw_atomic_long_xchg_release(v, new); } static __always_inline long -atomic_long_xchg_relaxed(atomic_long_t *v, long i) +atomic_long_xchg_relaxed(atomic_long_t *v, long new) { instrument_atomic_read_write(v, sizeof(*v)); - return raw_atomic_long_xchg_relaxed(v, i); + return raw_atomic_long_xchg_relaxed(v, new); } static __always_inline long @@ -2231,4 +2231,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// f6502977180430e61c1a7c4e5e665f04f501fb8d +// a4c3d2b229f907654cc53cb5d40e80f7fed1ec9c diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h index 63e0b4078ebd..f564f71ff8af 100644 --- a/include/linux/atomic/atomic-long.h +++ b/include/linux/atomic/atomic-long.h @@ -622,42 +622,42 @@ raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) } static __always_inline long -raw_atomic_long_xchg(atomic_long_t *v, long i) +raw_atomic_long_xchg(atomic_long_t *v, long new) { #ifdef CONFIG_64BIT - return raw_atomic64_xchg(v, i); + return raw_atomic64_xchg(v, new); #else - return raw_atomic_xchg(v, i); + return raw_atomic_xchg(v, new); #endif } static __always_inline long -raw_atomic_long_xchg_acquire(atomic_long_t *v, long i) +raw_atomic_long_xchg_acquire(atomic_long_t *v, long new) { #ifdef CONFIG_64BIT - return raw_atomic64_xchg_acquire(v, i); + return raw_atomic64_xchg_acquire(v, new); #else - return raw_atomic_xchg_acquire(v, i); + return raw_atomic_xchg_acquire(v, new); #endif } static __always_inline long -raw_atomic_long_xchg_release(atomic_long_t *v, long i) +raw_atomic_long_xchg_release(atomic_long_t *v, long new) { #ifdef CONFIG_64BIT - return raw_atomic64_xchg_release(v, i); + return raw_atomic64_xchg_release(v, new); #else - return raw_atomic_xchg_release(v, i); + return raw_atomic_xchg_release(v, new); #endif } static __always_inline long -raw_atomic_long_xchg_relaxed(atomic_long_t *v, long i) +raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new) { #ifdef CONFIG_64BIT - return raw_atomic64_xchg_relaxed(v, i); + return raw_atomic64_xchg_relaxed(v, new); #else - return raw_atomic_xchg_relaxed(v, i); + return raw_atomic_xchg_relaxed(v, new); #endif } @@ -872,4 +872,4 @@ raw_atomic_long_dec_if_positive(atomic_long_t *v) } #endif /* _LINUX_ATOMIC_LONG_H */ -// ad09f849db0db5b30c82e497eeb9056a394c5f22 +// e785d25cc3f220b7d473d36aac9da85dd7eb13a8 -- cgit v1.2.3 From ad8110706f381170c9f9975f1cb06010fd3ca381 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 5 Jun 2023 08:01:22 +0100 Subject: locking/atomic: scripts: generate kerneldoc comments Currently the atomics are documented in Documentation/atomic_t.txt, and have no kerneldoc comments. There are a sufficient number of gotchas (e.g. semantics, noinstr-safety) that it would be nice to have comments to call these out, and it would be nice to have kerneldoc comments such that these can be collated. While it's possible to derive the semantics from the code, this can be painful given the amount of indirection we currently have (e.g. fallback paths), and it's easy to be mislead by naming, e.g. * The unconditional void-returning ops *only* have relaxed variants without a _relaxed suffix, and can easily be mistaken for being fully ordered. It would be nice to give these a _relaxed() suffix, but this would result in significant churn throughout the kernel. * Our naming of conditional and unconditional+test ops is rather inconsistent, and it can be difficult to derive the name of an operation, or to identify where an op is conditional or unconditional+test. Some ops are clearly conditional: - dec_if_positive - add_unless - dec_unless_positive - inc_unless_negative Some ops are clearly unconditional+test: - sub_and_test - dec_and_test - inc_and_test However, what exactly those test is not obvious. A _test_zero suffix might be clearer. Others could be read ambiguously: - inc_not_zero // conditional - add_negative // unconditional+test It would probably be worth renaming these, e.g. to inc_unless_zero and add_test_negative. As a step towards making this more consistent and easier to understand, this patch adds kerneldoc comments for all generated *atomic*_*() functions. These are generated from templates, with some common text shared, making it easy to extend these in future if necessary. I've tried to make these as consistent and clear as possible, and I've deliberately ensured: * All ops have their ordering explicitly mentioned in the short and long description. * All test ops have "test" in their short description. * All ops are described as an expression using their usual C operator. For example: andnot: "Atomically updates @v to (@v & ~@i)" inc: "Atomically updates @v to (@v + 1)" Which may be clearer to non-naative English speakers, and allows all the operations to be described in the same style. * All conditional ops have their condition described as an expression using the usual C operators. For example: add_unless: "If (@v != @u), atomically updates @v to (@v + @i)" cmpxchg: "If (@v == @old), atomically updates @v to @new" Which may be clearer to non-naative English speakers, and allows all the operations to be described in the same style. * All bitwise ops (and,andnot,or,xor) explicitly mention that they are bitwise in their short description, so that they are not mistaken for performing their logical equivalents. * The noinstr safety of each op is explicitly described, with a description of whether or not to use the raw_ form of the op. There should be no functional change as a result of this patch. Reported-by: Paul E. McKenney Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230605070124.3741859-26-mark.rutland@arm.com --- include/linux/atomic/atomic-arch-fallback.h | 1848 +++++++++++++++++- include/linux/atomic/atomic-instrumented.h | 2771 ++++++++++++++++++++++++++- include/linux/atomic/atomic-long.h | 925 ++++++++- 3 files changed, 5541 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 470c2890ab8d..8cded57dd7a6 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -428,12 +428,32 @@ extern void raw_cmpxchg128_relaxed_not_implemented(void); #define raw_sync_cmpxchg arch_sync_cmpxchg +/** + * raw_atomic_read() - atomic load with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically loads the value of @v with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_read() elsewhere. + * + * Return: The value loaded from @v. + */ static __always_inline int raw_atomic_read(const atomic_t *v) { return arch_atomic_read(v); } +/** + * raw_atomic_read_acquire() - atomic load with acquire ordering + * @v: pointer to atomic_t + * + * Atomically loads the value of @v with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere. + * + * Return: The value loaded from @v. + */ static __always_inline int raw_atomic_read_acquire(const atomic_t *v) { @@ -455,12 +475,34 @@ raw_atomic_read_acquire(const atomic_t *v) #endif } +/** + * raw_atomic_set() - atomic set with relaxed ordering + * @v: pointer to atomic_t + * @i: int value to assign + * + * Atomically sets @v to @i with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_set() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_set(atomic_t *v, int i) { arch_atomic_set(v, i); } +/** + * raw_atomic_set_release() - atomic set with release ordering + * @v: pointer to atomic_t + * @i: int value to assign + * + * Atomically sets @v to @i with release ordering. + * + * Safe to use in noinstr code; prefer atomic_set_release() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_set_release(atomic_t *v, int i) { @@ -478,12 +520,34 @@ raw_atomic_set_release(atomic_t *v, int i) #endif } +/** + * raw_atomic_add() - atomic add with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_add() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_add(int i, atomic_t *v) { arch_atomic_add(i, v); } +/** + * raw_atomic_add_return() - atomic add with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_add_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_add_return(int i, atomic_t *v) { @@ -500,6 +564,17 @@ raw_atomic_add_return(int i, atomic_t *v) #endif } +/** + * raw_atomic_add_return_acquire() - atomic add with acquire ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_add_return_acquire(int i, atomic_t *v) { @@ -516,6 +591,17 @@ raw_atomic_add_return_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_add_return_release() - atomic add with release ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_add_return_release(int i, atomic_t *v) { @@ -531,6 +617,17 @@ raw_atomic_add_return_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_add_return_relaxed(int i, atomic_t *v) { @@ -543,6 +640,17 @@ raw_atomic_add_return_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_add() - atomic add with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_add(int i, atomic_t *v) { @@ -559,6 +667,17 @@ raw_atomic_fetch_add(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_add_acquire(int i, atomic_t *v) { @@ -575,6 +694,17 @@ raw_atomic_fetch_add_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_add_release() - atomic add with release ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_add_release(int i, atomic_t *v) { @@ -590,6 +720,17 @@ raw_atomic_fetch_add_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_add_relaxed(int i, atomic_t *v) { @@ -602,12 +743,34 @@ raw_atomic_fetch_add_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_sub() - atomic subtract with relaxed ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_sub() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_sub(int i, atomic_t *v) { arch_atomic_sub(i, v); } +/** + * raw_atomic_sub_return() - atomic subtract with full ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_sub_return(int i, atomic_t *v) { @@ -624,6 +787,17 @@ raw_atomic_sub_return(int i, atomic_t *v) #endif } +/** + * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_sub_return_acquire(int i, atomic_t *v) { @@ -640,6 +814,17 @@ raw_atomic_sub_return_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_sub_return_release() - atomic subtract with release ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_sub_return_release(int i, atomic_t *v) { @@ -655,6 +840,17 @@ raw_atomic_sub_return_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_sub_return_relaxed(int i, atomic_t *v) { @@ -667,6 +863,17 @@ raw_atomic_sub_return_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_sub() - atomic subtract with full ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_sub(int i, atomic_t *v) { @@ -683,6 +890,17 @@ raw_atomic_fetch_sub(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_sub_acquire(int i, atomic_t *v) { @@ -699,6 +917,17 @@ raw_atomic_fetch_sub_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_sub_release() - atomic subtract with release ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_sub_release(int i, atomic_t *v) { @@ -714,6 +943,17 @@ raw_atomic_fetch_sub_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) { @@ -726,6 +966,16 @@ raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_inc() - atomic increment with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_inc() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_inc(atomic_t *v) { @@ -736,6 +986,16 @@ raw_atomic_inc(atomic_t *v) #endif } +/** + * raw_atomic_inc_return() - atomic increment with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_inc_return(atomic_t *v) { @@ -752,6 +1012,16 @@ raw_atomic_inc_return(atomic_t *v) #endif } +/** + * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_inc_return_acquire(atomic_t *v) { @@ -768,6 +1038,16 @@ raw_atomic_inc_return_acquire(atomic_t *v) #endif } +/** + * raw_atomic_inc_return_release() - atomic increment with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_inc_return_release(atomic_t *v) { @@ -783,6 +1063,16 @@ raw_atomic_inc_return_release(atomic_t *v) #endif } +/** + * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_inc_return_relaxed(atomic_t *v) { @@ -795,6 +1085,16 @@ raw_atomic_inc_return_relaxed(atomic_t *v) #endif } +/** + * raw_atomic_fetch_inc() - atomic increment with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_inc(atomic_t *v) { @@ -811,6 +1111,16 @@ raw_atomic_fetch_inc(atomic_t *v) #endif } +/** + * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_inc_acquire(atomic_t *v) { @@ -827,6 +1137,16 @@ raw_atomic_fetch_inc_acquire(atomic_t *v) #endif } +/** + * raw_atomic_fetch_inc_release() - atomic increment with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_inc_release(atomic_t *v) { @@ -842,6 +1162,16 @@ raw_atomic_fetch_inc_release(atomic_t *v) #endif } +/** + * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_inc_relaxed(atomic_t *v) { @@ -854,6 +1184,16 @@ raw_atomic_fetch_inc_relaxed(atomic_t *v) #endif } +/** + * raw_atomic_dec() - atomic decrement with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_dec() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_dec(atomic_t *v) { @@ -864,6 +1204,16 @@ raw_atomic_dec(atomic_t *v) #endif } +/** + * raw_atomic_dec_return() - atomic decrement with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_dec_return(atomic_t *v) { @@ -880,6 +1230,16 @@ raw_atomic_dec_return(atomic_t *v) #endif } +/** + * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_dec_return_acquire(atomic_t *v) { @@ -896,6 +1256,16 @@ raw_atomic_dec_return_acquire(atomic_t *v) #endif } +/** + * raw_atomic_dec_return_release() - atomic decrement with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_dec_return_release(atomic_t *v) { @@ -911,6 +1281,16 @@ raw_atomic_dec_return_release(atomic_t *v) #endif } +/** + * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline int raw_atomic_dec_return_relaxed(atomic_t *v) { @@ -923,6 +1303,16 @@ raw_atomic_dec_return_relaxed(atomic_t *v) #endif } +/** + * raw_atomic_fetch_dec() - atomic decrement with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_dec(atomic_t *v) { @@ -939,6 +1329,16 @@ raw_atomic_fetch_dec(atomic_t *v) #endif } +/** + * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_dec_acquire(atomic_t *v) { @@ -955,6 +1355,16 @@ raw_atomic_fetch_dec_acquire(atomic_t *v) #endif } +/** + * raw_atomic_fetch_dec_release() - atomic decrement with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_dec_release(atomic_t *v) { @@ -970,6 +1380,16 @@ raw_atomic_fetch_dec_release(atomic_t *v) #endif } +/** + * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_dec_relaxed(atomic_t *v) { @@ -982,12 +1402,34 @@ raw_atomic_fetch_dec_relaxed(atomic_t *v) #endif } +/** + * raw_atomic_and() - atomic bitwise AND with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_and() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_and(int i, atomic_t *v) { arch_atomic_and(i, v); } +/** + * raw_atomic_fetch_and() - atomic bitwise AND with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_and(int i, atomic_t *v) { @@ -1004,6 +1446,17 @@ raw_atomic_fetch_and(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_and_acquire(int i, atomic_t *v) { @@ -1020,6 +1473,17 @@ raw_atomic_fetch_and_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_and_release(int i, atomic_t *v) { @@ -1035,6 +1499,17 @@ raw_atomic_fetch_and_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_and_relaxed(int i, atomic_t *v) { @@ -1047,6 +1522,17 @@ raw_atomic_fetch_and_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_andnot() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_andnot(int i, atomic_t *v) { @@ -1057,6 +1543,17 @@ raw_atomic_andnot(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_andnot(int i, atomic_t *v) { @@ -1073,6 +1570,17 @@ raw_atomic_fetch_andnot(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) { @@ -1089,6 +1597,17 @@ raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_andnot_release(int i, atomic_t *v) { @@ -1104,6 +1623,17 @@ raw_atomic_fetch_andnot_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) { @@ -1116,12 +1646,34 @@ raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_or() - atomic bitwise OR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_or() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_or(int i, atomic_t *v) { arch_atomic_or(i, v); } +/** + * raw_atomic_fetch_or() - atomic bitwise OR with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_or(int i, atomic_t *v) { @@ -1138,6 +1690,17 @@ raw_atomic_fetch_or(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_or_acquire(int i, atomic_t *v) { @@ -1154,6 +1717,17 @@ raw_atomic_fetch_or_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_or_release(int i, atomic_t *v) { @@ -1169,6 +1743,17 @@ raw_atomic_fetch_or_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_or_relaxed(int i, atomic_t *v) { @@ -1181,12 +1766,34 @@ raw_atomic_fetch_or_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_xor() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_xor(int i, atomic_t *v) { arch_atomic_xor(i, v); } +/** + * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_xor(int i, atomic_t *v) { @@ -1203,6 +1810,17 @@ raw_atomic_fetch_xor(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_xor_acquire(int i, atomic_t *v) { @@ -1219,6 +1837,17 @@ raw_atomic_fetch_xor_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_xor_release(int i, atomic_t *v) { @@ -1234,6 +1863,17 @@ raw_atomic_fetch_xor_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) { @@ -1246,6 +1886,17 @@ raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_xchg() - atomic exchange with full ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with full ordering. + * + * Safe to use in noinstr code; prefer atomic_xchg() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_xchg(atomic_t *v, int new) { @@ -1262,6 +1913,17 @@ raw_atomic_xchg(atomic_t *v, int new) #endif } +/** + * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_xchg_acquire(atomic_t *v, int new) { @@ -1278,6 +1940,17 @@ raw_atomic_xchg_acquire(atomic_t *v, int new) #endif } +/** + * raw_atomic_xchg_release() - atomic exchange with release ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with release ordering. + * + * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_xchg_release(atomic_t *v, int new) { @@ -1293,6 +1966,17 @@ raw_atomic_xchg_release(atomic_t *v, int new) #endif } +/** + * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_xchg_relaxed(atomic_t *v, int new) { @@ -1305,6 +1989,18 @@ raw_atomic_xchg_relaxed(atomic_t *v, int new) #endif } +/** + * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * + * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_cmpxchg(atomic_t *v, int old, int new) { @@ -1321,6 +2017,18 @@ raw_atomic_cmpxchg(atomic_t *v, int old, int new) #endif } +/** + * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { @@ -1337,6 +2045,18 @@ raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) #endif } +/** + * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * + * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) { @@ -1352,6 +2072,18 @@ raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) #endif } +/** + * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { @@ -1364,6 +2096,19 @@ raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) #endif } +/** + * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) { @@ -1384,6 +2129,19 @@ raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) #endif } +/** + * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { @@ -1404,6 +2162,19 @@ raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) #endif } +/** + * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { @@ -1423,6 +2194,19 @@ raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) #endif } +/** + * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { @@ -1439,6 +2223,17 @@ raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) #endif } +/** + * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic_sub_and_test(int i, atomic_t *v) { @@ -1449,6 +2244,16 @@ raw_atomic_sub_and_test(int i, atomic_t *v) #endif } +/** + * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic_dec_and_test(atomic_t *v) { @@ -1459,6 +2264,16 @@ raw_atomic_dec_and_test(atomic_t *v) #endif } +/** + * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic_inc_and_test(atomic_t *v) { @@ -1469,6 +2284,17 @@ raw_atomic_inc_and_test(atomic_t *v) #endif } +/** + * raw_atomic_add_negative() - atomic add and test if negative with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_add_negative(int i, atomic_t *v) { @@ -1485,6 +2311,17 @@ raw_atomic_add_negative(int i, atomic_t *v) #endif } +/** + * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_add_negative_acquire(int i, atomic_t *v) { @@ -1501,6 +2338,17 @@ raw_atomic_add_negative_acquire(int i, atomic_t *v) #endif } +/** + * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_add_negative_release(int i, atomic_t *v) { @@ -1516,6 +2364,17 @@ raw_atomic_add_negative_release(int i, atomic_t *v) #endif } +/** + * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_add_negative_relaxed(int i, atomic_t *v) { @@ -1528,6 +2387,18 @@ raw_atomic_add_negative_relaxed(int i, atomic_t *v) #endif } +/** + * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_t + * @a: int value to add + * @u: int value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline int raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) { @@ -1545,6 +2416,18 @@ raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) #endif } +/** + * raw_atomic_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_t + * @a: int value to add + * @u: int value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_add_unless(atomic_t *v, int a, int u) { @@ -1555,6 +2438,16 @@ raw_atomic_add_unless(atomic_t *v, int a, int u) #endif } +/** + * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering + * @v: pointer to atomic_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_inc_not_zero(atomic_t *v) { @@ -1565,6 +2458,16 @@ raw_atomic_inc_not_zero(atomic_t *v) #endif } +/** + * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering + * @v: pointer to atomic_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_inc_unless_negative(atomic_t *v) { @@ -1582,6 +2485,16 @@ raw_atomic_inc_unless_negative(atomic_t *v) #endif } +/** + * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering + * @v: pointer to atomic_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_dec_unless_positive(atomic_t *v) { @@ -1599,6 +2512,16 @@ raw_atomic_dec_unless_positive(atomic_t *v) #endif } +/** + * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering + * @v: pointer to atomic_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline int raw_atomic_dec_if_positive(atomic_t *v) { @@ -1621,12 +2544,32 @@ raw_atomic_dec_if_positive(atomic_t *v) #include #endif +/** + * raw_atomic64_read() - atomic load with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically loads the value of @v with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_read() elsewhere. + * + * Return: The value loaded from @v. + */ static __always_inline s64 raw_atomic64_read(const atomic64_t *v) { return arch_atomic64_read(v); } +/** + * raw_atomic64_read_acquire() - atomic load with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically loads the value of @v with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere. + * + * Return: The value loaded from @v. + */ static __always_inline s64 raw_atomic64_read_acquire(const atomic64_t *v) { @@ -1648,12 +2591,34 @@ raw_atomic64_read_acquire(const atomic64_t *v) #endif } +/** + * raw_atomic64_set() - atomic set with relaxed ordering + * @v: pointer to atomic64_t + * @i: s64 value to assign + * + * Atomically sets @v to @i with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_set() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_set(atomic64_t *v, s64 i) { arch_atomic64_set(v, i); } +/** + * raw_atomic64_set_release() - atomic set with release ordering + * @v: pointer to atomic64_t + * @i: s64 value to assign + * + * Atomically sets @v to @i with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_set_release(atomic64_t *v, s64 i) { @@ -1671,12 +2636,34 @@ raw_atomic64_set_release(atomic64_t *v, s64 i) #endif } +/** + * raw_atomic64_add() - atomic add with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_add() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_add(s64 i, atomic64_t *v) { arch_atomic64_add(i, v); } +/** + * raw_atomic64_add_return() - atomic add with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_add_return(s64 i, atomic64_t *v) { @@ -1693,6 +2680,17 @@ raw_atomic64_add_return(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_add_return_acquire() - atomic add with acquire ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) { @@ -1709,6 +2707,17 @@ raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_add_return_release() - atomic add with release ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_add_return_release(s64 i, atomic64_t *v) { @@ -1724,6 +2733,17 @@ raw_atomic64_add_return_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) { @@ -1736,6 +2756,17 @@ raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_add() - atomic add with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_add(s64 i, atomic64_t *v) { @@ -1752,6 +2783,17 @@ raw_atomic64_fetch_add(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { @@ -1768,6 +2810,17 @@ raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_add_release() - atomic add with release ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) { @@ -1783,6 +2836,17 @@ raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) { @@ -1795,12 +2859,34 @@ raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_sub() - atomic subtract with relaxed ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_sub() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_sub(s64 i, atomic64_t *v) { arch_atomic64_sub(i, v); } +/** + * raw_atomic64_sub_return() - atomic subtract with full ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_sub_return(s64 i, atomic64_t *v) { @@ -1817,6 +2903,17 @@ raw_atomic64_sub_return(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) { @@ -1833,6 +2930,17 @@ raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_sub_return_release() - atomic subtract with release ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_sub_return_release(s64 i, atomic64_t *v) { @@ -1848,6 +2956,17 @@ raw_atomic64_sub_return_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) { @@ -1860,6 +2979,17 @@ raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_sub() - atomic subtract with full ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_sub(s64 i, atomic64_t *v) { @@ -1876,6 +3006,17 @@ raw_atomic64_fetch_sub(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { @@ -1892,6 +3033,17 @@ raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) { @@ -1907,6 +3059,17 @@ raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) { @@ -1919,6 +3082,16 @@ raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_inc() - atomic increment with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_inc(atomic64_t *v) { @@ -1929,6 +3102,16 @@ raw_atomic64_inc(atomic64_t *v) #endif } +/** + * raw_atomic64_inc_return() - atomic increment with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_inc_return(atomic64_t *v) { @@ -1945,6 +3128,16 @@ raw_atomic64_inc_return(atomic64_t *v) #endif } +/** + * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_inc_return_acquire(atomic64_t *v) { @@ -1961,6 +3154,16 @@ raw_atomic64_inc_return_acquire(atomic64_t *v) #endif } +/** + * raw_atomic64_inc_return_release() - atomic increment with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_inc_return_release(atomic64_t *v) { @@ -1976,6 +3179,16 @@ raw_atomic64_inc_return_release(atomic64_t *v) #endif } +/** + * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_inc_return_relaxed(atomic64_t *v) { @@ -1988,6 +3201,16 @@ raw_atomic64_inc_return_relaxed(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_inc() - atomic increment with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_inc(atomic64_t *v) { @@ -2004,6 +3227,16 @@ raw_atomic64_fetch_inc(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_inc_acquire(atomic64_t *v) { @@ -2020,6 +3253,16 @@ raw_atomic64_fetch_inc_acquire(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_inc_release() - atomic increment with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_inc_release(atomic64_t *v) { @@ -2035,6 +3278,16 @@ raw_atomic64_fetch_inc_release(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_inc_relaxed(atomic64_t *v) { @@ -2047,6 +3300,16 @@ raw_atomic64_fetch_inc_relaxed(atomic64_t *v) #endif } +/** + * raw_atomic64_dec() - atomic decrement with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_dec(atomic64_t *v) { @@ -2057,6 +3320,16 @@ raw_atomic64_dec(atomic64_t *v) #endif } +/** + * raw_atomic64_dec_return() - atomic decrement with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_dec_return(atomic64_t *v) { @@ -2073,6 +3346,16 @@ raw_atomic64_dec_return(atomic64_t *v) #endif } +/** + * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_dec_return_acquire(atomic64_t *v) { @@ -2089,6 +3372,16 @@ raw_atomic64_dec_return_acquire(atomic64_t *v) #endif } +/** + * raw_atomic64_dec_return_release() - atomic decrement with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_dec_return_release(atomic64_t *v) { @@ -2104,6 +3397,16 @@ raw_atomic64_dec_return_release(atomic64_t *v) #endif } +/** + * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline s64 raw_atomic64_dec_return_relaxed(atomic64_t *v) { @@ -2116,6 +3419,16 @@ raw_atomic64_dec_return_relaxed(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_dec() - atomic decrement with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_dec(atomic64_t *v) { @@ -2132,6 +3445,16 @@ raw_atomic64_fetch_dec(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_dec_acquire(atomic64_t *v) { @@ -2148,6 +3471,16 @@ raw_atomic64_fetch_dec_acquire(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_dec_release(atomic64_t *v) { @@ -2163,6 +3496,16 @@ raw_atomic64_fetch_dec_release(atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_dec_relaxed(atomic64_t *v) { @@ -2175,12 +3518,34 @@ raw_atomic64_fetch_dec_relaxed(atomic64_t *v) #endif } +/** + * raw_atomic64_and() - atomic bitwise AND with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_and() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_and(s64 i, atomic64_t *v) { arch_atomic64_and(i, v); } +/** + * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_and(s64 i, atomic64_t *v) { @@ -2197,6 +3562,17 @@ raw_atomic64_fetch_and(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { @@ -2213,6 +3589,17 @@ raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) { @@ -2228,6 +3615,17 @@ raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) { @@ -2240,6 +3638,17 @@ raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_andnot(s64 i, atomic64_t *v) { @@ -2250,6 +3659,17 @@ raw_atomic64_andnot(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) { @@ -2266,6 +3686,17 @@ raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { @@ -2282,6 +3713,17 @@ raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { @@ -2297,6 +3739,17 @@ raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { @@ -2309,12 +3762,34 @@ raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_or() - atomic bitwise OR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_or() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_or(s64 i, atomic64_t *v) { arch_atomic64_or(i, v); } +/** + * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_or(s64 i, atomic64_t *v) { @@ -2331,6 +3806,17 @@ raw_atomic64_fetch_or(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { @@ -2347,6 +3833,17 @@ raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) { @@ -2362,6 +3859,17 @@ raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) { @@ -2374,12 +3882,34 @@ raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_xor() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic64_xor(s64 i, atomic64_t *v) { arch_atomic64_xor(i, v); } +/** + * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_xor(s64 i, atomic64_t *v) { @@ -2396,6 +3926,17 @@ raw_atomic64_fetch_xor(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { @@ -2412,6 +3953,17 @@ raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) { @@ -2427,6 +3979,17 @@ raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) { @@ -2439,6 +4002,17 @@ raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_xchg() - atomic exchange with full ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_xchg(atomic64_t *v, s64 new) { @@ -2455,6 +4029,17 @@ raw_atomic64_xchg(atomic64_t *v, s64 new) #endif } +/** + * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) { @@ -2471,6 +4056,17 @@ raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) #endif } +/** + * raw_atomic64_xchg_release() - atomic exchange with release ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_xchg_release(atomic64_t *v, s64 new) { @@ -2486,6 +4082,17 @@ raw_atomic64_xchg_release(atomic64_t *v, s64 new) #endif } +/** + * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) { @@ -2498,6 +4105,18 @@ raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) #endif } +/** + * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { @@ -2514,6 +4133,18 @@ raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) #endif } +/** + * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { @@ -2530,6 +4161,18 @@ raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) #endif } +/** + * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { @@ -2545,6 +4188,18 @@ raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) #endif } +/** + * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { @@ -2557,6 +4212,19 @@ raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) #endif } +/** + * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { @@ -2577,6 +4245,19 @@ raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) #endif } +/** + * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { @@ -2597,6 +4278,19 @@ raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) #endif } +/** + * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { @@ -2616,6 +4310,19 @@ raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) #endif } +/** + * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { @@ -2632,6 +4339,17 @@ raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) #endif } +/** + * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic64_sub_and_test(s64 i, atomic64_t *v) { @@ -2642,6 +4360,16 @@ raw_atomic64_sub_and_test(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic64_dec_and_test(atomic64_t *v) { @@ -2652,6 +4380,16 @@ raw_atomic64_dec_and_test(atomic64_t *v) #endif } +/** + * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic64_inc_and_test(atomic64_t *v) { @@ -2662,6 +4400,17 @@ raw_atomic64_inc_and_test(atomic64_t *v) #endif } +/** + * raw_atomic64_add_negative() - atomic add and test if negative with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic64_add_negative(s64 i, atomic64_t *v) { @@ -2678,6 +4427,17 @@ raw_atomic64_add_negative(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) { @@ -2694,6 +4454,17 @@ raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic64_add_negative_release(s64 i, atomic64_t *v) { @@ -2709,6 +4480,17 @@ raw_atomic64_add_negative_release(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) { @@ -2721,6 +4503,18 @@ raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) #endif } +/** + * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic64_t + * @a: s64 value to add + * @u: s64 value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline s64 raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { @@ -2738,6 +4532,18 @@ raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) #endif } +/** + * raw_atomic64_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic64_t + * @a: s64 value to add + * @u: s64 value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { @@ -2748,6 +4554,16 @@ raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) #endif } +/** + * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering + * @v: pointer to atomic64_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic64_inc_not_zero(atomic64_t *v) { @@ -2758,6 +4574,16 @@ raw_atomic64_inc_not_zero(atomic64_t *v) #endif } +/** + * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering + * @v: pointer to atomic64_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic64_inc_unless_negative(atomic64_t *v) { @@ -2775,6 +4601,16 @@ raw_atomic64_inc_unless_negative(atomic64_t *v) #endif } +/** + * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering + * @v: pointer to atomic64_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic64_dec_unless_positive(atomic64_t *v) { @@ -2792,6 +4628,16 @@ raw_atomic64_dec_unless_positive(atomic64_t *v) #endif } +/** + * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering + * @v: pointer to atomic64_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline s64 raw_atomic64_dec_if_positive(atomic64_t *v) { @@ -2811,4 +4657,4 @@ raw_atomic64_dec_if_positive(atomic64_t *v) } #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 205e090382132f1fc85e48b46e722865f9c81309 +// 3916f02c038baa3f5190d275f68b9211667fcc9d diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 5491c89dc03a..ebfc795f921b 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -16,6 +16,16 @@ #include #include +/** + * atomic_read() - atomic load with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically loads the value of @v with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_read() there. + * + * Return: The value loaded from @v. + */ static __always_inline int atomic_read(const atomic_t *v) { @@ -23,6 +33,16 @@ atomic_read(const atomic_t *v) return raw_atomic_read(v); } +/** + * atomic_read_acquire() - atomic load with acquire ordering + * @v: pointer to atomic_t + * + * Atomically loads the value of @v with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_read_acquire() there. + * + * Return: The value loaded from @v. + */ static __always_inline int atomic_read_acquire(const atomic_t *v) { @@ -30,6 +50,17 @@ atomic_read_acquire(const atomic_t *v) return raw_atomic_read_acquire(v); } +/** + * atomic_set() - atomic set with relaxed ordering + * @v: pointer to atomic_t + * @i: int value to assign + * + * Atomically sets @v to @i with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_set() there. + * + * Return: Nothing. + */ static __always_inline void atomic_set(atomic_t *v, int i) { @@ -37,6 +68,17 @@ atomic_set(atomic_t *v, int i) raw_atomic_set(v, i); } +/** + * atomic_set_release() - atomic set with release ordering + * @v: pointer to atomic_t + * @i: int value to assign + * + * Atomically sets @v to @i with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_set_release() there. + * + * Return: Nothing. + */ static __always_inline void atomic_set_release(atomic_t *v, int i) { @@ -45,6 +87,17 @@ atomic_set_release(atomic_t *v, int i) raw_atomic_set_release(v, i); } +/** + * atomic_add() - atomic add with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add() there. + * + * Return: Nothing. + */ static __always_inline void atomic_add(int i, atomic_t *v) { @@ -52,6 +105,17 @@ atomic_add(int i, atomic_t *v) raw_atomic_add(i, v); } +/** + * atomic_add_return() - atomic add with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_return() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_add_return(int i, atomic_t *v) { @@ -60,6 +124,17 @@ atomic_add_return(int i, atomic_t *v) return raw_atomic_add_return(i, v); } +/** + * atomic_add_return_acquire() - atomic add with acquire ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_add_return_acquire(int i, atomic_t *v) { @@ -67,6 +142,17 @@ atomic_add_return_acquire(int i, atomic_t *v) return raw_atomic_add_return_acquire(i, v); } +/** + * atomic_add_return_release() - atomic add with release ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_add_return_release(int i, atomic_t *v) { @@ -75,6 +161,17 @@ atomic_add_return_release(int i, atomic_t *v) return raw_atomic_add_return_release(i, v); } +/** + * atomic_add_return_relaxed() - atomic add with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_add_return_relaxed(int i, atomic_t *v) { @@ -82,6 +179,17 @@ atomic_add_return_relaxed(int i, atomic_t *v) return raw_atomic_add_return_relaxed(i, v); } +/** + * atomic_fetch_add() - atomic add with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_add() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_add(int i, atomic_t *v) { @@ -90,6 +198,17 @@ atomic_fetch_add(int i, atomic_t *v) return raw_atomic_fetch_add(i, v); } +/** + * atomic_fetch_add_acquire() - atomic add with acquire ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_add_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_add_acquire(int i, atomic_t *v) { @@ -97,6 +216,17 @@ atomic_fetch_add_acquire(int i, atomic_t *v) return raw_atomic_fetch_add_acquire(i, v); } +/** + * atomic_fetch_add_release() - atomic add with release ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_add_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_add_release(int i, atomic_t *v) { @@ -105,6 +235,17 @@ atomic_fetch_add_release(int i, atomic_t *v) return raw_atomic_fetch_add_release(i, v); } +/** + * atomic_fetch_add_relaxed() - atomic add with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_add_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_add_relaxed(int i, atomic_t *v) { @@ -112,6 +253,17 @@ atomic_fetch_add_relaxed(int i, atomic_t *v) return raw_atomic_fetch_add_relaxed(i, v); } +/** + * atomic_sub() - atomic subtract with relaxed ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_sub() there. + * + * Return: Nothing. + */ static __always_inline void atomic_sub(int i, atomic_t *v) { @@ -119,6 +271,17 @@ atomic_sub(int i, atomic_t *v) raw_atomic_sub(i, v); } +/** + * atomic_sub_return() - atomic subtract with full ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_sub_return() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_sub_return(int i, atomic_t *v) { @@ -127,6 +290,17 @@ atomic_sub_return(int i, atomic_t *v) return raw_atomic_sub_return(i, v); } +/** + * atomic_sub_return_acquire() - atomic subtract with acquire ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_sub_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_sub_return_acquire(int i, atomic_t *v) { @@ -134,6 +308,17 @@ atomic_sub_return_acquire(int i, atomic_t *v) return raw_atomic_sub_return_acquire(i, v); } +/** + * atomic_sub_return_release() - atomic subtract with release ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_sub_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_sub_return_release(int i, atomic_t *v) { @@ -142,6 +327,17 @@ atomic_sub_return_release(int i, atomic_t *v) return raw_atomic_sub_return_release(i, v); } +/** + * atomic_sub_return_relaxed() - atomic subtract with relaxed ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_sub_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_sub_return_relaxed(int i, atomic_t *v) { @@ -149,6 +345,17 @@ atomic_sub_return_relaxed(int i, atomic_t *v) return raw_atomic_sub_return_relaxed(i, v); } +/** + * atomic_fetch_sub() - atomic subtract with full ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_sub() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_sub(int i, atomic_t *v) { @@ -157,6 +364,17 @@ atomic_fetch_sub(int i, atomic_t *v) return raw_atomic_fetch_sub(i, v); } +/** + * atomic_fetch_sub_acquire() - atomic subtract with acquire ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_sub_acquire(int i, atomic_t *v) { @@ -164,6 +382,17 @@ atomic_fetch_sub_acquire(int i, atomic_t *v) return raw_atomic_fetch_sub_acquire(i, v); } +/** + * atomic_fetch_sub_release() - atomic subtract with release ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_sub_release(int i, atomic_t *v) { @@ -172,6 +401,17 @@ atomic_fetch_sub_release(int i, atomic_t *v) return raw_atomic_fetch_sub_release(i, v); } +/** + * atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering + * @i: int value to subtract + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_sub_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_sub_relaxed(int i, atomic_t *v) { @@ -179,6 +419,16 @@ atomic_fetch_sub_relaxed(int i, atomic_t *v) return raw_atomic_fetch_sub_relaxed(i, v); } +/** + * atomic_inc() - atomic increment with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc() there. + * + * Return: Nothing. + */ static __always_inline void atomic_inc(atomic_t *v) { @@ -186,6 +436,16 @@ atomic_inc(atomic_t *v) raw_atomic_inc(v); } +/** + * atomic_inc_return() - atomic increment with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_return() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_inc_return(atomic_t *v) { @@ -194,6 +454,16 @@ atomic_inc_return(atomic_t *v) return raw_atomic_inc_return(v); } +/** + * atomic_inc_return_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_inc_return_acquire(atomic_t *v) { @@ -201,6 +471,16 @@ atomic_inc_return_acquire(atomic_t *v) return raw_atomic_inc_return_acquire(v); } +/** + * atomic_inc_return_release() - atomic increment with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_inc_return_release(atomic_t *v) { @@ -209,6 +489,16 @@ atomic_inc_return_release(atomic_t *v) return raw_atomic_inc_return_release(v); } +/** + * atomic_inc_return_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_inc_return_relaxed(atomic_t *v) { @@ -216,6 +506,16 @@ atomic_inc_return_relaxed(atomic_t *v) return raw_atomic_inc_return_relaxed(v); } +/** + * atomic_fetch_inc() - atomic increment with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_inc() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_inc(atomic_t *v) { @@ -224,6 +524,16 @@ atomic_fetch_inc(atomic_t *v) return raw_atomic_fetch_inc(v); } +/** + * atomic_fetch_inc_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_inc_acquire(atomic_t *v) { @@ -231,6 +541,16 @@ atomic_fetch_inc_acquire(atomic_t *v) return raw_atomic_fetch_inc_acquire(v); } +/** + * atomic_fetch_inc_release() - atomic increment with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_inc_release(atomic_t *v) { @@ -239,6 +559,16 @@ atomic_fetch_inc_release(atomic_t *v) return raw_atomic_fetch_inc_release(v); } +/** + * atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_inc_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_inc_relaxed(atomic_t *v) { @@ -246,6 +576,16 @@ atomic_fetch_inc_relaxed(atomic_t *v) return raw_atomic_fetch_inc_relaxed(v); } +/** + * atomic_dec() - atomic decrement with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec() there. + * + * Return: Nothing. + */ static __always_inline void atomic_dec(atomic_t *v) { @@ -253,6 +593,16 @@ atomic_dec(atomic_t *v) raw_atomic_dec(v); } +/** + * atomic_dec_return() - atomic decrement with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_return() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_dec_return(atomic_t *v) { @@ -261,6 +611,16 @@ atomic_dec_return(atomic_t *v) return raw_atomic_dec_return(v); } +/** + * atomic_dec_return_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_dec_return_acquire(atomic_t *v) { @@ -268,6 +628,16 @@ atomic_dec_return_acquire(atomic_t *v) return raw_atomic_dec_return_acquire(v); } +/** + * atomic_dec_return_release() - atomic decrement with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_dec_return_release(atomic_t *v) { @@ -276,6 +646,16 @@ atomic_dec_return_release(atomic_t *v) return raw_atomic_dec_return_release(v); } +/** + * atomic_dec_return_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline int atomic_dec_return_relaxed(atomic_t *v) { @@ -283,6 +663,16 @@ atomic_dec_return_relaxed(atomic_t *v) return raw_atomic_dec_return_relaxed(v); } +/** + * atomic_fetch_dec() - atomic decrement with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_dec() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_dec(atomic_t *v) { @@ -291,6 +681,16 @@ atomic_fetch_dec(atomic_t *v) return raw_atomic_fetch_dec(v); } +/** + * atomic_fetch_dec_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_dec_acquire(atomic_t *v) { @@ -298,6 +698,16 @@ atomic_fetch_dec_acquire(atomic_t *v) return raw_atomic_fetch_dec_acquire(v); } +/** + * atomic_fetch_dec_release() - atomic decrement with release ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_dec_release(atomic_t *v) { @@ -306,6 +716,16 @@ atomic_fetch_dec_release(atomic_t *v) return raw_atomic_fetch_dec_release(v); } +/** + * atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_dec_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_dec_relaxed(atomic_t *v) { @@ -313,6 +733,17 @@ atomic_fetch_dec_relaxed(atomic_t *v) return raw_atomic_fetch_dec_relaxed(v); } +/** + * atomic_and() - atomic bitwise AND with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_and() there. + * + * Return: Nothing. + */ static __always_inline void atomic_and(int i, atomic_t *v) { @@ -320,6 +751,17 @@ atomic_and(int i, atomic_t *v) raw_atomic_and(i, v); } +/** + * atomic_fetch_and() - atomic bitwise AND with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_and() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_and(int i, atomic_t *v) { @@ -328,6 +770,17 @@ atomic_fetch_and(int i, atomic_t *v) return raw_atomic_fetch_and(i, v); } +/** + * atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_and_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_and_acquire(int i, atomic_t *v) { @@ -335,6 +788,17 @@ atomic_fetch_and_acquire(int i, atomic_t *v) return raw_atomic_fetch_and_acquire(i, v); } +/** + * atomic_fetch_and_release() - atomic bitwise AND with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_and_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_and_release(int i, atomic_t *v) { @@ -343,6 +807,17 @@ atomic_fetch_and_release(int i, atomic_t *v) return raw_atomic_fetch_and_release(i, v); } +/** + * atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_and_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_and_relaxed(int i, atomic_t *v) { @@ -350,6 +825,17 @@ atomic_fetch_and_relaxed(int i, atomic_t *v) return raw_atomic_fetch_and_relaxed(i, v); } +/** + * atomic_andnot() - atomic bitwise AND NOT with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_andnot() there. + * + * Return: Nothing. + */ static __always_inline void atomic_andnot(int i, atomic_t *v) { @@ -357,6 +843,17 @@ atomic_andnot(int i, atomic_t *v) raw_atomic_andnot(i, v); } +/** + * atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_andnot(int i, atomic_t *v) { @@ -365,6 +862,17 @@ atomic_fetch_andnot(int i, atomic_t *v) return raw_atomic_fetch_andnot(i, v); } +/** + * atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) { @@ -372,6 +880,17 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) return raw_atomic_fetch_andnot_acquire(i, v); } +/** + * atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_andnot_release(int i, atomic_t *v) { @@ -380,6 +899,17 @@ atomic_fetch_andnot_release(int i, atomic_t *v) return raw_atomic_fetch_andnot_release(i, v); } +/** + * atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_andnot_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) { @@ -387,6 +917,17 @@ atomic_fetch_andnot_relaxed(int i, atomic_t *v) return raw_atomic_fetch_andnot_relaxed(i, v); } +/** + * atomic_or() - atomic bitwise OR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_or() there. + * + * Return: Nothing. + */ static __always_inline void atomic_or(int i, atomic_t *v) { @@ -394,6 +935,17 @@ atomic_or(int i, atomic_t *v) raw_atomic_or(i, v); } +/** + * atomic_fetch_or() - atomic bitwise OR with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_or() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_or(int i, atomic_t *v) { @@ -402,6 +954,17 @@ atomic_fetch_or(int i, atomic_t *v) return raw_atomic_fetch_or(i, v); } +/** + * atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_or_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_or_acquire(int i, atomic_t *v) { @@ -409,6 +972,17 @@ atomic_fetch_or_acquire(int i, atomic_t *v) return raw_atomic_fetch_or_acquire(i, v); } +/** + * atomic_fetch_or_release() - atomic bitwise OR with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_or_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_or_release(int i, atomic_t *v) { @@ -417,6 +991,17 @@ atomic_fetch_or_release(int i, atomic_t *v) return raw_atomic_fetch_or_release(i, v); } +/** + * atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_or_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_or_relaxed(int i, atomic_t *v) { @@ -424,6 +1009,17 @@ atomic_fetch_or_relaxed(int i, atomic_t *v) return raw_atomic_fetch_or_relaxed(i, v); } +/** + * atomic_xor() - atomic bitwise XOR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_xor() there. + * + * Return: Nothing. + */ static __always_inline void atomic_xor(int i, atomic_t *v) { @@ -431,6 +1027,17 @@ atomic_xor(int i, atomic_t *v) raw_atomic_xor(i, v); } +/** + * atomic_fetch_xor() - atomic bitwise XOR with full ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_xor() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_xor(int i, atomic_t *v) { @@ -439,6 +1046,17 @@ atomic_fetch_xor(int i, atomic_t *v) return raw_atomic_fetch_xor(i, v); } +/** + * atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_xor_acquire(int i, atomic_t *v) { @@ -446,6 +1064,17 @@ atomic_fetch_xor_acquire(int i, atomic_t *v) return raw_atomic_fetch_xor_acquire(i, v); } +/** + * atomic_fetch_xor_release() - atomic bitwise XOR with release ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_xor_release(int i, atomic_t *v) { @@ -454,6 +1083,17 @@ atomic_fetch_xor_release(int i, atomic_t *v) return raw_atomic_fetch_xor_release(i, v); } +/** + * atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering + * @i: int value + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_xor_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_xor_relaxed(int i, atomic_t *v) { @@ -461,6 +1101,17 @@ atomic_fetch_xor_relaxed(int i, atomic_t *v) return raw_atomic_fetch_xor_relaxed(i, v); } +/** + * atomic_xchg() - atomic exchange with full ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_xchg() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_xchg(atomic_t *v, int new) { @@ -469,6 +1120,17 @@ atomic_xchg(atomic_t *v, int new) return raw_atomic_xchg(v, new); } +/** + * atomic_xchg_acquire() - atomic exchange with acquire ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_xchg_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_xchg_acquire(atomic_t *v, int new) { @@ -476,6 +1138,17 @@ atomic_xchg_acquire(atomic_t *v, int new) return raw_atomic_xchg_acquire(v, new); } +/** + * atomic_xchg_release() - atomic exchange with release ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_xchg_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_xchg_release(atomic_t *v, int new) { @@ -484,6 +1157,17 @@ atomic_xchg_release(atomic_t *v, int new) return raw_atomic_xchg_release(v, new); } +/** + * atomic_xchg_relaxed() - atomic exchange with relaxed ordering + * @v: pointer to atomic_t + * @new: int value to assign + * + * Atomically updates @v to @new with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_xchg_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_xchg_relaxed(atomic_t *v, int new) { @@ -491,6 +1175,18 @@ atomic_xchg_relaxed(atomic_t *v, int new) return raw_atomic_xchg_relaxed(v, new); } +/** + * atomic_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_cmpxchg() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) { @@ -499,6 +1195,18 @@ atomic_cmpxchg(atomic_t *v, int old, int new) return raw_atomic_cmpxchg(v, old, new); } +/** + * atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { @@ -506,6 +1214,18 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) return raw_atomic_cmpxchg_acquire(v, old, new); } +/** + * atomic_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_release() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_cmpxchg_release(atomic_t *v, int old, int new) { @@ -514,6 +1234,18 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) return raw_atomic_cmpxchg_release(v, old, new); } +/** + * atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_t + * @old: int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_cmpxchg_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { @@ -521,6 +1253,19 @@ atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) return raw_atomic_cmpxchg_relaxed(v, old, new); } +/** + * atomic_try_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) { @@ -530,6 +1275,19 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) return raw_atomic_try_cmpxchg(v, old, new); } +/** + * atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { @@ -538,6 +1296,19 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) return raw_atomic_try_cmpxchg_acquire(v, old, new); } +/** + * atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { @@ -547,6 +1318,19 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) return raw_atomic_try_cmpxchg_release(v, old, new); } +/** + * atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_t + * @old: pointer to int value to compare with + * @new: int value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { @@ -555,6 +1339,17 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) return raw_atomic_try_cmpxchg_relaxed(v, old, new); } +/** + * atomic_sub_and_test() - atomic subtract and test if zero with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_sub_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) { @@ -563,6 +1358,16 @@ atomic_sub_and_test(int i, atomic_t *v) return raw_atomic_sub_and_test(i, v); } +/** + * atomic_dec_and_test() - atomic decrement and test if zero with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic_dec_and_test(atomic_t *v) { @@ -571,6 +1376,16 @@ atomic_dec_and_test(atomic_t *v) return raw_atomic_dec_and_test(v); } +/** + * atomic_inc_and_test() - atomic increment and test if zero with full ordering + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic_inc_and_test(atomic_t *v) { @@ -579,6 +1394,17 @@ atomic_inc_and_test(atomic_t *v) return raw_atomic_inc_and_test(v); } +/** + * atomic_add_negative() - atomic add and test if negative with full ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_negative() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_add_negative(int i, atomic_t *v) { @@ -587,6 +1413,17 @@ atomic_add_negative(int i, atomic_t *v) return raw_atomic_add_negative(i, v); } +/** + * atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_negative_acquire() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_add_negative_acquire(int i, atomic_t *v) { @@ -594,6 +1431,17 @@ atomic_add_negative_acquire(int i, atomic_t *v) return raw_atomic_add_negative_acquire(i, v); } +/** + * atomic_add_negative_release() - atomic add and test if negative with release ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_negative_release() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_add_negative_release(int i, atomic_t *v) { @@ -602,6 +1450,17 @@ atomic_add_negative_release(int i, atomic_t *v) return raw_atomic_add_negative_release(i, v); } +/** + * atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering + * @i: int value to add + * @v: pointer to atomic_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_negative_relaxed() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_add_negative_relaxed(int i, atomic_t *v) { @@ -609,6 +1468,18 @@ atomic_add_negative_relaxed(int i, atomic_t *v) return raw_atomic_add_negative_relaxed(i, v); } +/** + * atomic_fetch_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_t + * @a: int value to add + * @u: int value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_fetch_add_unless() there. + * + * Return: The original value of @v. + */ static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { @@ -617,6 +1488,18 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) return raw_atomic_fetch_add_unless(v, a, u); } +/** + * atomic_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_t + * @a: int value to add + * @u: int value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_add_unless() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_add_unless(atomic_t *v, int a, int u) { @@ -625,6 +1508,16 @@ atomic_add_unless(atomic_t *v, int a, int u) return raw_atomic_add_unless(v, a, u); } +/** + * atomic_inc_not_zero() - atomic increment unless zero with full ordering + * @v: pointer to atomic_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_not_zero() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_inc_not_zero(atomic_t *v) { @@ -633,6 +1526,16 @@ atomic_inc_not_zero(atomic_t *v) return raw_atomic_inc_not_zero(v); } +/** + * atomic_inc_unless_negative() - atomic increment unless negative with full ordering + * @v: pointer to atomic_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_inc_unless_negative() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_inc_unless_negative(atomic_t *v) { @@ -641,6 +1544,16 @@ atomic_inc_unless_negative(atomic_t *v) return raw_atomic_inc_unless_negative(v); } +/** + * atomic_dec_unless_positive() - atomic decrement unless positive with full ordering + * @v: pointer to atomic_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_unless_positive() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_dec_unless_positive(atomic_t *v) { @@ -649,6 +1562,16 @@ atomic_dec_unless_positive(atomic_t *v) return raw_atomic_dec_unless_positive(v); } +/** + * atomic_dec_if_positive() - atomic decrement if positive with full ordering + * @v: pointer to atomic_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline int atomic_dec_if_positive(atomic_t *v) { @@ -657,6 +1580,16 @@ atomic_dec_if_positive(atomic_t *v) return raw_atomic_dec_if_positive(v); } +/** + * atomic64_read() - atomic load with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically loads the value of @v with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_read() there. + * + * Return: The value loaded from @v. + */ static __always_inline s64 atomic64_read(const atomic64_t *v) { @@ -664,6 +1597,16 @@ atomic64_read(const atomic64_t *v) return raw_atomic64_read(v); } +/** + * atomic64_read_acquire() - atomic load with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically loads the value of @v with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_read_acquire() there. + * + * Return: The value loaded from @v. + */ static __always_inline s64 atomic64_read_acquire(const atomic64_t *v) { @@ -671,6 +1614,17 @@ atomic64_read_acquire(const atomic64_t *v) return raw_atomic64_read_acquire(v); } +/** + * atomic64_set() - atomic set with relaxed ordering + * @v: pointer to atomic64_t + * @i: s64 value to assign + * + * Atomically sets @v to @i with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_set() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_set(atomic64_t *v, s64 i) { @@ -678,6 +1632,17 @@ atomic64_set(atomic64_t *v, s64 i) raw_atomic64_set(v, i); } +/** + * atomic64_set_release() - atomic set with release ordering + * @v: pointer to atomic64_t + * @i: s64 value to assign + * + * Atomically sets @v to @i with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_set_release() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_set_release(atomic64_t *v, s64 i) { @@ -686,6 +1651,17 @@ atomic64_set_release(atomic64_t *v, s64 i) raw_atomic64_set_release(v, i); } +/** + * atomic64_add() - atomic add with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_add(s64 i, atomic64_t *v) { @@ -693,6 +1669,17 @@ atomic64_add(s64 i, atomic64_t *v) raw_atomic64_add(i, v); } +/** + * atomic64_add_return() - atomic add with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_return() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) { @@ -701,6 +1688,17 @@ atomic64_add_return(s64 i, atomic64_t *v) return raw_atomic64_add_return(i, v); } +/** + * atomic64_add_return_acquire() - atomic add with acquire ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_add_return_acquire(s64 i, atomic64_t *v) { @@ -708,6 +1706,17 @@ atomic64_add_return_acquire(s64 i, atomic64_t *v) return raw_atomic64_add_return_acquire(i, v); } +/** + * atomic64_add_return_release() - atomic add with release ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_add_return_release(s64 i, atomic64_t *v) { @@ -716,6 +1725,17 @@ atomic64_add_return_release(s64 i, atomic64_t *v) return raw_atomic64_add_return_release(i, v); } +/** + * atomic64_add_return_relaxed() - atomic add with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_add_return_relaxed(s64 i, atomic64_t *v) { @@ -723,6 +1743,17 @@ atomic64_add_return_relaxed(s64 i, atomic64_t *v) return raw_atomic64_add_return_relaxed(i, v); } +/** + * atomic64_fetch_add() - atomic add with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_add() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) { @@ -731,6 +1762,17 @@ atomic64_fetch_add(s64 i, atomic64_t *v) return raw_atomic64_fetch_add(i, v); } +/** + * atomic64_fetch_add_acquire() - atomic add with acquire ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { @@ -738,6 +1780,17 @@ atomic64_fetch_add_acquire(s64 i, atomic64_t *v) return raw_atomic64_fetch_add_acquire(i, v); } +/** + * atomic64_fetch_add_release() - atomic add with release ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_add_release(s64 i, atomic64_t *v) { @@ -746,6 +1799,17 @@ atomic64_fetch_add_release(s64 i, atomic64_t *v) return raw_atomic64_fetch_add_release(i, v); } +/** + * atomic64_fetch_add_relaxed() - atomic add with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) { @@ -753,6 +1817,17 @@ atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) return raw_atomic64_fetch_add_relaxed(i, v); } +/** + * atomic64_sub() - atomic subtract with relaxed ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_sub() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_sub(s64 i, atomic64_t *v) { @@ -760,6 +1835,17 @@ atomic64_sub(s64 i, atomic64_t *v) raw_atomic64_sub(i, v); } +/** + * atomic64_sub_return() - atomic subtract with full ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_sub_return() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) { @@ -768,6 +1854,17 @@ atomic64_sub_return(s64 i, atomic64_t *v) return raw_atomic64_sub_return(i, v); } +/** + * atomic64_sub_return_acquire() - atomic subtract with acquire ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_sub_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_sub_return_acquire(s64 i, atomic64_t *v) { @@ -775,6 +1872,17 @@ atomic64_sub_return_acquire(s64 i, atomic64_t *v) return raw_atomic64_sub_return_acquire(i, v); } +/** + * atomic64_sub_return_release() - atomic subtract with release ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_sub_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_sub_return_release(s64 i, atomic64_t *v) { @@ -783,6 +1891,17 @@ atomic64_sub_return_release(s64 i, atomic64_t *v) return raw_atomic64_sub_return_release(i, v); } +/** + * atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_sub_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) { @@ -790,6 +1909,17 @@ atomic64_sub_return_relaxed(s64 i, atomic64_t *v) return raw_atomic64_sub_return_relaxed(i, v); } +/** + * atomic64_fetch_sub() - atomic subtract with full ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) { @@ -798,6 +1928,17 @@ atomic64_fetch_sub(s64 i, atomic64_t *v) return raw_atomic64_fetch_sub(i, v); } +/** + * atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { @@ -805,6 +1946,17 @@ atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) return raw_atomic64_fetch_sub_acquire(i, v); } +/** + * atomic64_fetch_sub_release() - atomic subtract with release ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_sub_release(s64 i, atomic64_t *v) { @@ -813,6 +1965,17 @@ atomic64_fetch_sub_release(s64 i, atomic64_t *v) return raw_atomic64_fetch_sub_release(i, v); } +/** + * atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering + * @i: s64 value to subtract + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_sub_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) { @@ -820,6 +1983,16 @@ atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) return raw_atomic64_fetch_sub_relaxed(i, v); } +/** + * atomic64_inc() - atomic increment with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_inc(atomic64_t *v) { @@ -827,6 +2000,16 @@ atomic64_inc(atomic64_t *v) raw_atomic64_inc(v); } +/** + * atomic64_inc_return() - atomic increment with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_return() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_inc_return(atomic64_t *v) { @@ -835,6 +2018,16 @@ atomic64_inc_return(atomic64_t *v) return raw_atomic64_inc_return(v); } +/** + * atomic64_inc_return_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_inc_return_acquire(atomic64_t *v) { @@ -842,6 +2035,16 @@ atomic64_inc_return_acquire(atomic64_t *v) return raw_atomic64_inc_return_acquire(v); } +/** + * atomic64_inc_return_release() - atomic increment with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_inc_return_release(atomic64_t *v) { @@ -850,6 +2053,16 @@ atomic64_inc_return_release(atomic64_t *v) return raw_atomic64_inc_return_release(v); } +/** + * atomic64_inc_return_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_inc_return_relaxed(atomic64_t *v) { @@ -857,6 +2070,16 @@ atomic64_inc_return_relaxed(atomic64_t *v) return raw_atomic64_inc_return_relaxed(v); } +/** + * atomic64_fetch_inc() - atomic increment with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_inc(atomic64_t *v) { @@ -865,6 +2088,16 @@ atomic64_fetch_inc(atomic64_t *v) return raw_atomic64_fetch_inc(v); } +/** + * atomic64_fetch_inc_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_inc_acquire(atomic64_t *v) { @@ -872,6 +2105,16 @@ atomic64_fetch_inc_acquire(atomic64_t *v) return raw_atomic64_fetch_inc_acquire(v); } +/** + * atomic64_fetch_inc_release() - atomic increment with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_inc_release(atomic64_t *v) { @@ -880,6 +2123,16 @@ atomic64_fetch_inc_release(atomic64_t *v) return raw_atomic64_fetch_inc_release(v); } +/** + * atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_inc_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_inc_relaxed(atomic64_t *v) { @@ -887,6 +2140,16 @@ atomic64_fetch_inc_relaxed(atomic64_t *v) return raw_atomic64_fetch_inc_relaxed(v); } +/** + * atomic64_dec() - atomic decrement with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_dec(atomic64_t *v) { @@ -894,6 +2157,16 @@ atomic64_dec(atomic64_t *v) raw_atomic64_dec(v); } +/** + * atomic64_dec_return() - atomic decrement with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_return() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_dec_return(atomic64_t *v) { @@ -902,6 +2175,16 @@ atomic64_dec_return(atomic64_t *v) return raw_atomic64_dec_return(v); } +/** + * atomic64_dec_return_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_dec_return_acquire(atomic64_t *v) { @@ -909,6 +2192,16 @@ atomic64_dec_return_acquire(atomic64_t *v) return raw_atomic64_dec_return_acquire(v); } +/** + * atomic64_dec_return_release() - atomic decrement with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_dec_return_release(atomic64_t *v) { @@ -917,6 +2210,16 @@ atomic64_dec_return_release(atomic64_t *v) return raw_atomic64_dec_return_release(v); } +/** + * atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline s64 atomic64_dec_return_relaxed(atomic64_t *v) { @@ -924,6 +2227,16 @@ atomic64_dec_return_relaxed(atomic64_t *v) return raw_atomic64_dec_return_relaxed(v); } +/** + * atomic64_fetch_dec() - atomic decrement with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_dec(atomic64_t *v) { @@ -932,6 +2245,16 @@ atomic64_fetch_dec(atomic64_t *v) return raw_atomic64_fetch_dec(v); } +/** + * atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_dec_acquire(atomic64_t *v) { @@ -939,6 +2262,16 @@ atomic64_fetch_dec_acquire(atomic64_t *v) return raw_atomic64_fetch_dec_acquire(v); } +/** + * atomic64_fetch_dec_release() - atomic decrement with release ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_dec_release(atomic64_t *v) { @@ -947,6 +2280,16 @@ atomic64_fetch_dec_release(atomic64_t *v) return raw_atomic64_fetch_dec_release(v); } +/** + * atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_dec_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_dec_relaxed(atomic64_t *v) { @@ -954,6 +2297,17 @@ atomic64_fetch_dec_relaxed(atomic64_t *v) return raw_atomic64_fetch_dec_relaxed(v); } +/** + * atomic64_and() - atomic bitwise AND with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_and() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_and(s64 i, atomic64_t *v) { @@ -961,6 +2315,17 @@ atomic64_and(s64 i, atomic64_t *v) raw_atomic64_and(i, v); } +/** + * atomic64_fetch_and() - atomic bitwise AND with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_and() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) { @@ -969,6 +2334,17 @@ atomic64_fetch_and(s64 i, atomic64_t *v) return raw_atomic64_fetch_and(i, v); } +/** + * atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { @@ -976,6 +2352,17 @@ atomic64_fetch_and_acquire(s64 i, atomic64_t *v) return raw_atomic64_fetch_and_acquire(i, v); } +/** + * atomic64_fetch_and_release() - atomic bitwise AND with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_and_release(s64 i, atomic64_t *v) { @@ -984,6 +2371,17 @@ atomic64_fetch_and_release(s64 i, atomic64_t *v) return raw_atomic64_fetch_and_release(i, v); } +/** + * atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_and_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) { @@ -991,6 +2389,17 @@ atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) return raw_atomic64_fetch_and_relaxed(i, v); } +/** + * atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_andnot() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_andnot(s64 i, atomic64_t *v) { @@ -998,6 +2407,17 @@ atomic64_andnot(s64 i, atomic64_t *v) raw_atomic64_andnot(i, v); } +/** + * atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_andnot(s64 i, atomic64_t *v) { @@ -1006,6 +2426,17 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) return raw_atomic64_fetch_andnot(i, v); } +/** + * atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { @@ -1013,6 +2444,17 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) return raw_atomic64_fetch_andnot_acquire(i, v); } +/** + * atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { @@ -1021,6 +2463,17 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) return raw_atomic64_fetch_andnot_release(i, v); } +/** + * atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_andnot_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { @@ -1028,6 +2481,17 @@ atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) return raw_atomic64_fetch_andnot_relaxed(i, v); } +/** + * atomic64_or() - atomic bitwise OR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_or() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_or(s64 i, atomic64_t *v) { @@ -1035,6 +2499,17 @@ atomic64_or(s64 i, atomic64_t *v) raw_atomic64_or(i, v); } +/** + * atomic64_fetch_or() - atomic bitwise OR with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_or() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) { @@ -1043,6 +2518,17 @@ atomic64_fetch_or(s64 i, atomic64_t *v) return raw_atomic64_fetch_or(i, v); } +/** + * atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { @@ -1050,6 +2536,17 @@ atomic64_fetch_or_acquire(s64 i, atomic64_t *v) return raw_atomic64_fetch_or_acquire(i, v); } +/** + * atomic64_fetch_or_release() - atomic bitwise OR with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_or_release(s64 i, atomic64_t *v) { @@ -1058,6 +2555,17 @@ atomic64_fetch_or_release(s64 i, atomic64_t *v) return raw_atomic64_fetch_or_release(i, v); } +/** + * atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_or_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) { @@ -1065,6 +2573,17 @@ atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) return raw_atomic64_fetch_or_relaxed(i, v); } +/** + * atomic64_xor() - atomic bitwise XOR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_xor() there. + * + * Return: Nothing. + */ static __always_inline void atomic64_xor(s64 i, atomic64_t *v) { @@ -1072,6 +2591,17 @@ atomic64_xor(s64 i, atomic64_t *v) raw_atomic64_xor(i, v); } +/** + * atomic64_fetch_xor() - atomic bitwise XOR with full ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) { @@ -1080,6 +2610,17 @@ atomic64_fetch_xor(s64 i, atomic64_t *v) return raw_atomic64_fetch_xor(i, v); } +/** + * atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { @@ -1087,6 +2628,17 @@ atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) return raw_atomic64_fetch_xor_acquire(i, v); } +/** + * atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_xor_release(s64 i, atomic64_t *v) { @@ -1095,6 +2647,17 @@ atomic64_fetch_xor_release(s64 i, atomic64_t *v) return raw_atomic64_fetch_xor_release(i, v); } +/** + * atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering + * @i: s64 value + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_xor_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) { @@ -1102,6 +2665,17 @@ atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) return raw_atomic64_fetch_xor_relaxed(i, v); } +/** + * atomic64_xchg() - atomic exchange with full ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_xchg() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 new) { @@ -1110,6 +2684,17 @@ atomic64_xchg(atomic64_t *v, s64 new) return raw_atomic64_xchg(v, new); } +/** + * atomic64_xchg_acquire() - atomic exchange with acquire ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_xchg_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_xchg_acquire(atomic64_t *v, s64 new) { @@ -1117,6 +2702,17 @@ atomic64_xchg_acquire(atomic64_t *v, s64 new) return raw_atomic64_xchg_acquire(v, new); } +/** + * atomic64_xchg_release() - atomic exchange with release ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_xchg_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_xchg_release(atomic64_t *v, s64 new) { @@ -1125,6 +2721,17 @@ atomic64_xchg_release(atomic64_t *v, s64 new) return raw_atomic64_xchg_release(v, new); } +/** + * atomic64_xchg_relaxed() - atomic exchange with relaxed ordering + * @v: pointer to atomic64_t + * @new: s64 value to assign + * + * Atomically updates @v to @new with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_xchg_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_xchg_relaxed(atomic64_t *v, s64 new) { @@ -1132,6 +2739,18 @@ atomic64_xchg_relaxed(atomic64_t *v, s64 new) return raw_atomic64_xchg_relaxed(v, new); } +/** + * atomic64_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { @@ -1140,6 +2759,18 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) return raw_atomic64_cmpxchg(v, old, new); } +/** + * atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { @@ -1147,6 +2778,18 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) return raw_atomic64_cmpxchg_acquire(v, old, new); } +/** + * atomic64_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_release() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { @@ -1155,6 +2798,18 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) return raw_atomic64_cmpxchg_release(v, old, new); } +/** + * atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic64_t + * @old: s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_cmpxchg_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { @@ -1162,6 +2817,19 @@ atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) return raw_atomic64_cmpxchg_relaxed(v, old, new); } +/** + * atomic64_try_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { @@ -1171,6 +2839,19 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) return raw_atomic64_try_cmpxchg(v, old, new); } +/** + * atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { @@ -1179,6 +2860,19 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) return raw_atomic64_try_cmpxchg_acquire(v, old, new); } +/** + * atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { @@ -1188,6 +2882,19 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) return raw_atomic64_try_cmpxchg_release(v, old, new); } +/** + * atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic64_t + * @old: pointer to s64 value to compare with + * @new: s64 value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { @@ -1196,6 +2903,17 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) return raw_atomic64_try_cmpxchg_relaxed(v, old, new); } +/** + * atomic64_sub_and_test() - atomic subtract and test if zero with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_sub_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) { @@ -1204,6 +2922,16 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) return raw_atomic64_sub_and_test(i, v); } +/** + * atomic64_dec_and_test() - atomic decrement and test if zero with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic64_dec_and_test(atomic64_t *v) { @@ -1212,6 +2940,16 @@ atomic64_dec_and_test(atomic64_t *v) return raw_atomic64_dec_and_test(v); } +/** + * atomic64_inc_and_test() - atomic increment and test if zero with full ordering + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic64_inc_and_test(atomic64_t *v) { @@ -1220,6 +2958,17 @@ atomic64_inc_and_test(atomic64_t *v) return raw_atomic64_inc_and_test(v); } +/** + * atomic64_add_negative() - atomic add and test if negative with full ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_negative() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) { @@ -1228,6 +2977,17 @@ atomic64_add_negative(s64 i, atomic64_t *v) return raw_atomic64_add_negative(i, v); } +/** + * atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_negative_acquire() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic64_add_negative_acquire(s64 i, atomic64_t *v) { @@ -1235,6 +2995,17 @@ atomic64_add_negative_acquire(s64 i, atomic64_t *v) return raw_atomic64_add_negative_acquire(i, v); } +/** + * atomic64_add_negative_release() - atomic add and test if negative with release ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_negative_release() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic64_add_negative_release(s64 i, atomic64_t *v) { @@ -1243,6 +3014,17 @@ atomic64_add_negative_release(s64 i, atomic64_t *v) return raw_atomic64_add_negative_release(i, v); } +/** + * atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering + * @i: s64 value to add + * @v: pointer to atomic64_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_negative_relaxed() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic64_add_negative_relaxed(s64 i, atomic64_t *v) { @@ -1250,6 +3032,18 @@ atomic64_add_negative_relaxed(s64 i, atomic64_t *v) return raw_atomic64_add_negative_relaxed(i, v); } +/** + * atomic64_fetch_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic64_t + * @a: s64 value to add + * @u: s64 value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_fetch_add_unless() there. + * + * Return: The original value of @v. + */ static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { @@ -1258,6 +3052,18 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) return raw_atomic64_fetch_add_unless(v, a, u); } +/** + * atomic64_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic64_t + * @a: s64 value to add + * @u: s64 value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_add_unless() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { @@ -1266,6 +3072,16 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) return raw_atomic64_add_unless(v, a, u); } +/** + * atomic64_inc_not_zero() - atomic increment unless zero with full ordering + * @v: pointer to atomic64_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_not_zero() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) { @@ -1274,6 +3090,16 @@ atomic64_inc_not_zero(atomic64_t *v) return raw_atomic64_inc_not_zero(v); } +/** + * atomic64_inc_unless_negative() - atomic increment unless negative with full ordering + * @v: pointer to atomic64_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_inc_unless_negative() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic64_inc_unless_negative(atomic64_t *v) { @@ -1282,6 +3108,16 @@ atomic64_inc_unless_negative(atomic64_t *v) return raw_atomic64_inc_unless_negative(v); } +/** + * atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering + * @v: pointer to atomic64_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_unless_positive() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic64_dec_unless_positive(atomic64_t *v) { @@ -1290,6 +3126,16 @@ atomic64_dec_unless_positive(atomic64_t *v) return raw_atomic64_dec_unless_positive(v); } +/** + * atomic64_dec_if_positive() - atomic decrement if positive with full ordering + * @v: pointer to atomic64_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) { @@ -1298,6 +3144,16 @@ atomic64_dec_if_positive(atomic64_t *v) return raw_atomic64_dec_if_positive(v); } +/** + * atomic_long_read() - atomic load with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically loads the value of @v with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_read() there. + * + * Return: The value loaded from @v. + */ static __always_inline long atomic_long_read(const atomic_long_t *v) { @@ -1305,6 +3161,16 @@ atomic_long_read(const atomic_long_t *v) return raw_atomic_long_read(v); } +/** + * atomic_long_read_acquire() - atomic load with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically loads the value of @v with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_read_acquire() there. + * + * Return: The value loaded from @v. + */ static __always_inline long atomic_long_read_acquire(const atomic_long_t *v) { @@ -1312,6 +3178,17 @@ atomic_long_read_acquire(const atomic_long_t *v) return raw_atomic_long_read_acquire(v); } +/** + * atomic_long_set() - atomic set with relaxed ordering + * @v: pointer to atomic_long_t + * @i: long value to assign + * + * Atomically sets @v to @i with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_set() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_set(atomic_long_t *v, long i) { @@ -1319,6 +3196,17 @@ atomic_long_set(atomic_long_t *v, long i) raw_atomic_long_set(v, i); } +/** + * atomic_long_set_release() - atomic set with release ordering + * @v: pointer to atomic_long_t + * @i: long value to assign + * + * Atomically sets @v to @i with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_set_release() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_set_release(atomic_long_t *v, long i) { @@ -1327,6 +3215,17 @@ atomic_long_set_release(atomic_long_t *v, long i) raw_atomic_long_set_release(v, i); } +/** + * atomic_long_add() - atomic add with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_add(long i, atomic_long_t *v) { @@ -1334,6 +3233,17 @@ atomic_long_add(long i, atomic_long_t *v) raw_atomic_long_add(i, v); } +/** + * atomic_long_add_return() - atomic add with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_return() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_add_return(long i, atomic_long_t *v) { @@ -1342,6 +3252,17 @@ atomic_long_add_return(long i, atomic_long_t *v) return raw_atomic_long_add_return(i, v); } +/** + * atomic_long_add_return_acquire() - atomic add with acquire ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_add_return_acquire(long i, atomic_long_t *v) { @@ -1349,6 +3270,17 @@ atomic_long_add_return_acquire(long i, atomic_long_t *v) return raw_atomic_long_add_return_acquire(i, v); } +/** + * atomic_long_add_return_release() - atomic add with release ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_add_return_release(long i, atomic_long_t *v) { @@ -1357,6 +3289,17 @@ atomic_long_add_return_release(long i, atomic_long_t *v) return raw_atomic_long_add_return_release(i, v); } +/** + * atomic_long_add_return_relaxed() - atomic add with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_add_return_relaxed(long i, atomic_long_t *v) { @@ -1364,6 +3307,17 @@ atomic_long_add_return_relaxed(long i, atomic_long_t *v) return raw_atomic_long_add_return_relaxed(i, v); } +/** + * atomic_long_fetch_add() - atomic add with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_add(long i, atomic_long_t *v) { @@ -1372,6 +3326,17 @@ atomic_long_fetch_add(long i, atomic_long_t *v) return raw_atomic_long_fetch_add(i, v); } +/** + * atomic_long_fetch_add_acquire() - atomic add with acquire ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { @@ -1379,6 +3344,17 @@ atomic_long_fetch_add_acquire(long i, atomic_long_t *v) return raw_atomic_long_fetch_add_acquire(i, v); } +/** + * atomic_long_fetch_add_release() - atomic add with release ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_add_release(long i, atomic_long_t *v) { @@ -1387,6 +3363,17 @@ atomic_long_fetch_add_release(long i, atomic_long_t *v) return raw_atomic_long_fetch_add_release(i, v); } +/** + * atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { @@ -1394,6 +3381,17 @@ atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) return raw_atomic_long_fetch_add_relaxed(i, v); } +/** + * atomic_long_sub() - atomic subtract with relaxed ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_sub() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_sub(long i, atomic_long_t *v) { @@ -1401,6 +3399,17 @@ atomic_long_sub(long i, atomic_long_t *v) raw_atomic_long_sub(i, v); } +/** + * atomic_long_sub_return() - atomic subtract with full ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_sub_return() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_sub_return(long i, atomic_long_t *v) { @@ -1409,6 +3418,17 @@ atomic_long_sub_return(long i, atomic_long_t *v) return raw_atomic_long_sub_return(i, v); } +/** + * atomic_long_sub_return_acquire() - atomic subtract with acquire ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_sub_return_acquire(long i, atomic_long_t *v) { @@ -1416,6 +3436,17 @@ atomic_long_sub_return_acquire(long i, atomic_long_t *v) return raw_atomic_long_sub_return_acquire(i, v); } +/** + * atomic_long_sub_return_release() - atomic subtract with release ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_sub_return_release(long i, atomic_long_t *v) { @@ -1424,6 +3455,17 @@ atomic_long_sub_return_release(long i, atomic_long_t *v) return raw_atomic_long_sub_return_release(i, v); } +/** + * atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_sub_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { @@ -1431,6 +3473,17 @@ atomic_long_sub_return_relaxed(long i, atomic_long_t *v) return raw_atomic_long_sub_return_relaxed(i, v); } +/** + * atomic_long_fetch_sub() - atomic subtract with full ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_sub(long i, atomic_long_t *v) { @@ -1439,6 +3492,17 @@ atomic_long_fetch_sub(long i, atomic_long_t *v) return raw_atomic_long_fetch_sub(i, v); } +/** + * atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { @@ -1446,6 +3510,17 @@ atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) return raw_atomic_long_fetch_sub_acquire(i, v); } +/** + * atomic_long_fetch_sub_release() - atomic subtract with release ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_sub_release(long i, atomic_long_t *v) { @@ -1454,6 +3529,17 @@ atomic_long_fetch_sub_release(long i, atomic_long_t *v) return raw_atomic_long_fetch_sub_release(i, v); } +/** + * atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_sub_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { @@ -1461,6 +3547,16 @@ atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) return raw_atomic_long_fetch_sub_relaxed(i, v); } +/** + * atomic_long_inc() - atomic increment with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_inc(atomic_long_t *v) { @@ -1468,6 +3564,16 @@ atomic_long_inc(atomic_long_t *v) raw_atomic_long_inc(v); } +/** + * atomic_long_inc_return() - atomic increment with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_return() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_inc_return(atomic_long_t *v) { @@ -1476,6 +3582,16 @@ atomic_long_inc_return(atomic_long_t *v) return raw_atomic_long_inc_return(v); } +/** + * atomic_long_inc_return_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_inc_return_acquire(atomic_long_t *v) { @@ -1483,6 +3599,16 @@ atomic_long_inc_return_acquire(atomic_long_t *v) return raw_atomic_long_inc_return_acquire(v); } +/** + * atomic_long_inc_return_release() - atomic increment with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_inc_return_release(atomic_long_t *v) { @@ -1491,6 +3617,16 @@ atomic_long_inc_return_release(atomic_long_t *v) return raw_atomic_long_inc_return_release(v); } +/** + * atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_inc_return_relaxed(atomic_long_t *v) { @@ -1498,6 +3634,16 @@ atomic_long_inc_return_relaxed(atomic_long_t *v) return raw_atomic_long_inc_return_relaxed(v); } +/** + * atomic_long_fetch_inc() - atomic increment with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_inc(atomic_long_t *v) { @@ -1506,6 +3652,16 @@ atomic_long_fetch_inc(atomic_long_t *v) return raw_atomic_long_fetch_inc(v); } +/** + * atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_inc_acquire(atomic_long_t *v) { @@ -1513,6 +3669,16 @@ atomic_long_fetch_inc_acquire(atomic_long_t *v) return raw_atomic_long_fetch_inc_acquire(v); } +/** + * atomic_long_fetch_inc_release() - atomic increment with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_inc_release(atomic_long_t *v) { @@ -1521,6 +3687,16 @@ atomic_long_fetch_inc_release(atomic_long_t *v) return raw_atomic_long_fetch_inc_release(v); } +/** + * atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_inc_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_inc_relaxed(atomic_long_t *v) { @@ -1528,6 +3704,16 @@ atomic_long_fetch_inc_relaxed(atomic_long_t *v) return raw_atomic_long_fetch_inc_relaxed(v); } +/** + * atomic_long_dec() - atomic decrement with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_dec(atomic_long_t *v) { @@ -1535,6 +3721,16 @@ atomic_long_dec(atomic_long_t *v) raw_atomic_long_dec(v); } +/** + * atomic_long_dec_return() - atomic decrement with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_return() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_dec_return(atomic_long_t *v) { @@ -1543,6 +3739,16 @@ atomic_long_dec_return(atomic_long_t *v) return raw_atomic_long_dec_return(v); } +/** + * atomic_long_dec_return_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_acquire() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_dec_return_acquire(atomic_long_t *v) { @@ -1550,6 +3756,16 @@ atomic_long_dec_return_acquire(atomic_long_t *v) return raw_atomic_long_dec_return_acquire(v); } +/** + * atomic_long_dec_return_release() - atomic decrement with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_release() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_dec_return_release(atomic_long_t *v) { @@ -1558,6 +3774,16 @@ atomic_long_dec_return_release(atomic_long_t *v) return raw_atomic_long_dec_return_release(v); } +/** + * atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_return_relaxed() there. + * + * Return: The updated value of @v. + */ static __always_inline long atomic_long_dec_return_relaxed(atomic_long_t *v) { @@ -1565,6 +3791,16 @@ atomic_long_dec_return_relaxed(atomic_long_t *v) return raw_atomic_long_dec_return_relaxed(v); } +/** + * atomic_long_fetch_dec() - atomic decrement with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_dec(atomic_long_t *v) { @@ -1573,6 +3809,16 @@ atomic_long_fetch_dec(atomic_long_t *v) return raw_atomic_long_fetch_dec(v); } +/** + * atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_dec_acquire(atomic_long_t *v) { @@ -1580,6 +3826,16 @@ atomic_long_fetch_dec_acquire(atomic_long_t *v) return raw_atomic_long_fetch_dec_acquire(v); } +/** + * atomic_long_fetch_dec_release() - atomic decrement with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_dec_release(atomic_long_t *v) { @@ -1588,6 +3844,16 @@ atomic_long_fetch_dec_release(atomic_long_t *v) return raw_atomic_long_fetch_dec_release(v); } +/** + * atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_dec_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_dec_relaxed(atomic_long_t *v) { @@ -1595,6 +3861,17 @@ atomic_long_fetch_dec_relaxed(atomic_long_t *v) return raw_atomic_long_fetch_dec_relaxed(v); } +/** + * atomic_long_and() - atomic bitwise AND with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_and() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_and(long i, atomic_long_t *v) { @@ -1602,6 +3879,17 @@ atomic_long_and(long i, atomic_long_t *v) raw_atomic_long_and(i, v); } +/** + * atomic_long_fetch_and() - atomic bitwise AND with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_and(long i, atomic_long_t *v) { @@ -1610,6 +3898,17 @@ atomic_long_fetch_and(long i, atomic_long_t *v) return raw_atomic_long_fetch_and(i, v); } +/** + * atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { @@ -1617,6 +3916,17 @@ atomic_long_fetch_and_acquire(long i, atomic_long_t *v) return raw_atomic_long_fetch_and_acquire(i, v); } +/** + * atomic_long_fetch_and_release() - atomic bitwise AND with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_and_release(long i, atomic_long_t *v) { @@ -1625,6 +3935,17 @@ atomic_long_fetch_and_release(long i, atomic_long_t *v) return raw_atomic_long_fetch_and_release(i, v); } +/** + * atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_and_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { @@ -1632,6 +3953,17 @@ atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) return raw_atomic_long_fetch_and_relaxed(i, v); } +/** + * atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_andnot() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_andnot(long i, atomic_long_t *v) { @@ -1639,6 +3971,17 @@ atomic_long_andnot(long i, atomic_long_t *v) raw_atomic_long_andnot(i, v); } +/** + * atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_andnot(long i, atomic_long_t *v) { @@ -1647,6 +3990,17 @@ atomic_long_fetch_andnot(long i, atomic_long_t *v) return raw_atomic_long_fetch_andnot(i, v); } +/** + * atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { @@ -1654,6 +4008,17 @@ atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) return raw_atomic_long_fetch_andnot_acquire(i, v); } +/** + * atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { @@ -1662,6 +4027,17 @@ atomic_long_fetch_andnot_release(long i, atomic_long_t *v) return raw_atomic_long_fetch_andnot_release(i, v); } +/** + * atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_andnot_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { @@ -1669,6 +4045,17 @@ atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) return raw_atomic_long_fetch_andnot_relaxed(i, v); } +/** + * atomic_long_or() - atomic bitwise OR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_or() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_or(long i, atomic_long_t *v) { @@ -1676,6 +4063,17 @@ atomic_long_or(long i, atomic_long_t *v) raw_atomic_long_or(i, v); } +/** + * atomic_long_fetch_or() - atomic bitwise OR with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_or(long i, atomic_long_t *v) { @@ -1684,6 +4082,17 @@ atomic_long_fetch_or(long i, atomic_long_t *v) return raw_atomic_long_fetch_or(i, v); } +/** + * atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { @@ -1691,6 +4100,17 @@ atomic_long_fetch_or_acquire(long i, atomic_long_t *v) return raw_atomic_long_fetch_or_acquire(i, v); } +/** + * atomic_long_fetch_or_release() - atomic bitwise OR with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_or_release(long i, atomic_long_t *v) { @@ -1699,6 +4119,17 @@ atomic_long_fetch_or_release(long i, atomic_long_t *v) return raw_atomic_long_fetch_or_release(i, v); } +/** + * atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_or_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { @@ -1706,6 +4137,17 @@ atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) return raw_atomic_long_fetch_or_relaxed(i, v); } +/** + * atomic_long_xor() - atomic bitwise XOR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_xor() there. + * + * Return: Nothing. + */ static __always_inline void atomic_long_xor(long i, atomic_long_t *v) { @@ -1713,6 +4155,17 @@ atomic_long_xor(long i, atomic_long_t *v) raw_atomic_long_xor(i, v); } +/** + * atomic_long_fetch_xor() - atomic bitwise XOR with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_xor(long i, atomic_long_t *v) { @@ -1721,6 +4174,17 @@ atomic_long_fetch_xor(long i, atomic_long_t *v) return raw_atomic_long_fetch_xor(i, v); } +/** + * atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { @@ -1728,6 +4192,17 @@ atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) return raw_atomic_long_fetch_xor_acquire(i, v); } +/** + * atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_xor_release(long i, atomic_long_t *v) { @@ -1736,6 +4211,17 @@ atomic_long_fetch_xor_release(long i, atomic_long_t *v) return raw_atomic_long_fetch_xor_release(i, v); } +/** + * atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_xor_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { @@ -1743,6 +4229,17 @@ atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) return raw_atomic_long_fetch_xor_relaxed(i, v); } +/** + * atomic_long_xchg() - atomic exchange with full ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_xchg() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_xchg(atomic_long_t *v, long new) { @@ -1751,6 +4248,17 @@ atomic_long_xchg(atomic_long_t *v, long new) return raw_atomic_long_xchg(v, new); } +/** + * atomic_long_xchg_acquire() - atomic exchange with acquire ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_xchg_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_xchg_acquire(atomic_long_t *v, long new) { @@ -1758,6 +4266,17 @@ atomic_long_xchg_acquire(atomic_long_t *v, long new) return raw_atomic_long_xchg_acquire(v, new); } +/** + * atomic_long_xchg_release() - atomic exchange with release ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_xchg_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_xchg_release(atomic_long_t *v, long new) { @@ -1766,6 +4285,17 @@ atomic_long_xchg_release(atomic_long_t *v, long new) return raw_atomic_long_xchg_release(v, new); } +/** + * atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_xchg_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_xchg_relaxed(atomic_long_t *v, long new) { @@ -1773,6 +4303,18 @@ atomic_long_xchg_relaxed(atomic_long_t *v, long new) return raw_atomic_long_xchg_relaxed(v, new); } +/** + * atomic_long_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { @@ -1781,6 +4323,18 @@ atomic_long_cmpxchg(atomic_long_t *v, long old, long new) return raw_atomic_long_cmpxchg(v, old, new); } +/** + * atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_acquire() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { @@ -1788,6 +4342,18 @@ atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) return raw_atomic_long_cmpxchg_acquire(v, old, new); } +/** + * atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_release() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { @@ -1796,6 +4362,18 @@ atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) return raw_atomic_long_cmpxchg_release(v, old, new); } +/** + * atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_cmpxchg_relaxed() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { @@ -1803,6 +4381,19 @@ atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) return raw_atomic_long_cmpxchg_relaxed(v, old, new); } +/** + * atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { @@ -1812,6 +4403,19 @@ atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) return raw_atomic_long_try_cmpxchg(v, old, new); } +/** + * atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { @@ -1820,6 +4424,19 @@ atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) return raw_atomic_long_try_cmpxchg_acquire(v, old, new); } +/** + * atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { @@ -1829,6 +4446,19 @@ atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) return raw_atomic_long_try_cmpxchg_release(v, old, new); } +/** + * atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * Otherwise, updates @old to the current value of @v. + * + * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { @@ -1837,6 +4467,17 @@ atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) return raw_atomic_long_try_cmpxchg_relaxed(v, old, new); } +/** + * atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_sub_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic_long_sub_and_test(long i, atomic_long_t *v) { @@ -1845,6 +4486,16 @@ atomic_long_sub_and_test(long i, atomic_long_t *v) return raw_atomic_long_sub_and_test(i, v); } +/** + * atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic_long_dec_and_test(atomic_long_t *v) { @@ -1853,6 +4504,16 @@ atomic_long_dec_and_test(atomic_long_t *v) return raw_atomic_long_dec_and_test(v); } +/** + * atomic_long_inc_and_test() - atomic increment and test if zero with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_and_test() there. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool atomic_long_inc_and_test(atomic_long_t *v) { @@ -1861,6 +4522,17 @@ atomic_long_inc_and_test(atomic_long_t *v) return raw_atomic_long_inc_and_test(v); } +/** + * atomic_long_add_negative() - atomic add and test if negative with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_negative() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_long_add_negative(long i, atomic_long_t *v) { @@ -1869,6 +4541,17 @@ atomic_long_add_negative(long i, atomic_long_t *v) return raw_atomic_long_add_negative(i, v); } +/** + * atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_acquire() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_long_add_negative_acquire(long i, atomic_long_t *v) { @@ -1876,6 +4559,17 @@ atomic_long_add_negative_acquire(long i, atomic_long_t *v) return raw_atomic_long_add_negative_acquire(i, v); } +/** + * atomic_long_add_negative_release() - atomic add and test if negative with release ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_release() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_long_add_negative_release(long i, atomic_long_t *v) { @@ -1884,6 +4578,17 @@ atomic_long_add_negative_release(long i, atomic_long_t *v) return raw_atomic_long_add_negative_release(i, v); } +/** + * atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_negative_relaxed() there. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool atomic_long_add_negative_relaxed(long i, atomic_long_t *v) { @@ -1891,6 +4596,18 @@ atomic_long_add_negative_relaxed(long i, atomic_long_t *v) return raw_atomic_long_add_negative_relaxed(i, v); } +/** + * atomic_long_fetch_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_long_t + * @a: long value to add + * @u: long value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_fetch_add_unless() there. + * + * Return: The original value of @v. + */ static __always_inline long atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { @@ -1899,6 +4616,18 @@ atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) return raw_atomic_long_fetch_add_unless(v, a, u); } +/** + * atomic_long_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_long_t + * @a: long value to add + * @u: long value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_add_unless() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_long_add_unless(atomic_long_t *v, long a, long u) { @@ -1907,6 +4636,16 @@ atomic_long_add_unless(atomic_long_t *v, long a, long u) return raw_atomic_long_add_unless(v, a, u); } +/** + * atomic_long_inc_not_zero() - atomic increment unless zero with full ordering + * @v: pointer to atomic_long_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_not_zero() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_long_inc_not_zero(atomic_long_t *v) { @@ -1915,6 +4654,16 @@ atomic_long_inc_not_zero(atomic_long_t *v) return raw_atomic_long_inc_not_zero(v); } +/** + * atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering + * @v: pointer to atomic_long_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_inc_unless_negative() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_long_inc_unless_negative(atomic_long_t *v) { @@ -1923,6 +4672,16 @@ atomic_long_inc_unless_negative(atomic_long_t *v) return raw_atomic_long_inc_unless_negative(v); } +/** + * atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering + * @v: pointer to atomic_long_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_unless_positive() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool atomic_long_dec_unless_positive(atomic_long_t *v) { @@ -1931,6 +4690,16 @@ atomic_long_dec_unless_positive(atomic_long_t *v) return raw_atomic_long_dec_unless_positive(v); } +/** + * atomic_long_dec_if_positive() - atomic decrement if positive with full ordering + * @v: pointer to atomic_long_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. + * + * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline long atomic_long_dec_if_positive(atomic_long_t *v) { @@ -2231,4 +5000,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// a4c3d2b229f907654cc53cb5d40e80f7fed1ec9c +// 06cec02e676a484857aee38b0071a1d846ec9457 diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h index f564f71ff8af..f6df2adadf99 100644 --- a/include/linux/atomic/atomic-long.h +++ b/include/linux/atomic/atomic-long.h @@ -21,6 +21,16 @@ typedef atomic_t atomic_long_t; #define atomic_long_cond_read_relaxed atomic_cond_read_relaxed #endif +/** + * raw_atomic_long_read() - atomic load with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically loads the value of @v with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_read() elsewhere. + * + * Return: The value loaded from @v. + */ static __always_inline long raw_atomic_long_read(const atomic_long_t *v) { @@ -31,6 +41,16 @@ raw_atomic_long_read(const atomic_long_t *v) #endif } +/** + * raw_atomic_long_read_acquire() - atomic load with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically loads the value of @v with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_read_acquire() elsewhere. + * + * Return: The value loaded from @v. + */ static __always_inline long raw_atomic_long_read_acquire(const atomic_long_t *v) { @@ -41,6 +61,17 @@ raw_atomic_long_read_acquire(const atomic_long_t *v) #endif } +/** + * raw_atomic_long_set() - atomic set with relaxed ordering + * @v: pointer to atomic_long_t + * @i: long value to assign + * + * Atomically sets @v to @i with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_set() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_set(atomic_long_t *v, long i) { @@ -51,6 +82,17 @@ raw_atomic_long_set(atomic_long_t *v, long i) #endif } +/** + * raw_atomic_long_set_release() - atomic set with release ordering + * @v: pointer to atomic_long_t + * @i: long value to assign + * + * Atomically sets @v to @i with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_set_release() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_set_release(atomic_long_t *v, long i) { @@ -61,6 +103,17 @@ raw_atomic_long_set_release(atomic_long_t *v, long i) #endif } +/** + * raw_atomic_long_add() - atomic add with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_add(long i, atomic_long_t *v) { @@ -71,6 +124,17 @@ raw_atomic_long_add(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_return() - atomic add with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_add_return(long i, atomic_long_t *v) { @@ -81,6 +145,17 @@ raw_atomic_long_add_return(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_return_acquire() - atomic add with acquire ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) { @@ -91,6 +166,17 @@ raw_atomic_long_add_return_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_return_release() - atomic add with release ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_add_return_release(long i, atomic_long_t *v) { @@ -101,6 +187,17 @@ raw_atomic_long_add_return_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_return_relaxed() - atomic add with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) { @@ -111,6 +208,17 @@ raw_atomic_long_add_return_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_add() - atomic add with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_add() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_add(long i, atomic_long_t *v) { @@ -121,6 +229,17 @@ raw_atomic_long_fetch_add(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_add_acquire() - atomic add with acquire ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_add_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { @@ -131,6 +250,17 @@ raw_atomic_long_fetch_add_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_add_release() - atomic add with release ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_add_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) { @@ -141,6 +271,17 @@ raw_atomic_long_fetch_add_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_add_relaxed() - atomic add with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_add_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { @@ -151,6 +292,17 @@ raw_atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_sub() - atomic subtract with relaxed ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_sub() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_sub(long i, atomic_long_t *v) { @@ -161,6 +313,17 @@ raw_atomic_long_sub(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_sub_return() - atomic subtract with full ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_sub_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_sub_return(long i, atomic_long_t *v) { @@ -171,6 +334,17 @@ raw_atomic_long_sub_return(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_sub_return_acquire() - atomic subtract with acquire ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_sub_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) { @@ -181,6 +355,17 @@ raw_atomic_long_sub_return_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_sub_return_release() - atomic subtract with release ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_sub_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_sub_return_release(long i, atomic_long_t *v) { @@ -191,6 +376,17 @@ raw_atomic_long_sub_return_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_sub_return_relaxed() - atomic subtract with relaxed ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_sub_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { @@ -201,6 +397,17 @@ raw_atomic_long_sub_return_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_sub() - atomic subtract with full ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_sub() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_sub(long i, atomic_long_t *v) { @@ -211,6 +418,17 @@ raw_atomic_long_fetch_sub(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_sub_acquire() - atomic subtract with acquire ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_sub_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { @@ -221,6 +439,17 @@ raw_atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_sub_release() - atomic subtract with release ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_sub_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) { @@ -231,6 +460,17 @@ raw_atomic_long_fetch_sub_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_sub_relaxed() - atomic subtract with relaxed ordering + * @i: long value to subtract + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_sub_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { @@ -241,6 +481,16 @@ raw_atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc() - atomic increment with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_inc(atomic_long_t *v) { @@ -251,6 +501,16 @@ raw_atomic_long_inc(atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc_return() - atomic increment with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_inc_return(atomic_long_t *v) { @@ -261,6 +521,16 @@ raw_atomic_long_inc_return(atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc_return_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_inc_return_acquire(atomic_long_t *v) { @@ -271,6 +541,16 @@ raw_atomic_long_inc_return_acquire(atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc_return_release() - atomic increment with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_inc_return_release(atomic_long_t *v) { @@ -281,6 +561,16 @@ raw_atomic_long_inc_return_release(atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc_return_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_inc_return_relaxed(atomic_long_t *v) { @@ -291,6 +581,16 @@ raw_atomic_long_inc_return_relaxed(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_inc() - atomic increment with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_inc() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_inc(atomic_long_t *v) { @@ -301,6 +601,16 @@ raw_atomic_long_fetch_inc(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_inc_acquire() - atomic increment with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_inc_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) { @@ -311,6 +621,16 @@ raw_atomic_long_fetch_inc_acquire(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_inc_release() - atomic increment with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_inc_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_inc_release(atomic_long_t *v) { @@ -321,6 +641,16 @@ raw_atomic_long_fetch_inc_release(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_inc_relaxed() - atomic increment with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_inc_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) { @@ -331,6 +661,16 @@ raw_atomic_long_fetch_inc_relaxed(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec() - atomic decrement with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_dec(atomic_long_t *v) { @@ -341,6 +681,16 @@ raw_atomic_long_dec(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_return() - atomic decrement with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_return() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_dec_return(atomic_long_t *v) { @@ -351,6 +701,16 @@ raw_atomic_long_dec_return(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_return_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_return_acquire() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_dec_return_acquire(atomic_long_t *v) { @@ -361,6 +721,16 @@ raw_atomic_long_dec_return_acquire(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_return_release() - atomic decrement with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_return_release() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_dec_return_release(atomic_long_t *v) { @@ -371,6 +741,16 @@ raw_atomic_long_dec_return_release(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_return_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_return_relaxed() elsewhere. + * + * Return: The updated value of @v. + */ static __always_inline long raw_atomic_long_dec_return_relaxed(atomic_long_t *v) { @@ -381,6 +761,16 @@ raw_atomic_long_dec_return_relaxed(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_dec() - atomic decrement with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_dec() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_dec(atomic_long_t *v) { @@ -391,6 +781,16 @@ raw_atomic_long_fetch_dec(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_dec_acquire() - atomic decrement with acquire ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_dec_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) { @@ -401,6 +801,16 @@ raw_atomic_long_fetch_dec_acquire(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_dec_release() - atomic decrement with release ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_dec_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_dec_release(atomic_long_t *v) { @@ -411,6 +821,16 @@ raw_atomic_long_fetch_dec_release(atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_dec_relaxed() - atomic decrement with relaxed ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_dec_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) { @@ -421,6 +841,17 @@ raw_atomic_long_fetch_dec_relaxed(atomic_long_t *v) #endif } +/** + * raw_atomic_long_and() - atomic bitwise AND with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_and() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_and(long i, atomic_long_t *v) { @@ -431,6 +862,17 @@ raw_atomic_long_and(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_and() - atomic bitwise AND with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_and() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_and(long i, atomic_long_t *v) { @@ -441,6 +883,17 @@ raw_atomic_long_fetch_and(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_and_acquire() - atomic bitwise AND with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_and_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { @@ -451,6 +904,17 @@ raw_atomic_long_fetch_and_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_and_release() - atomic bitwise AND with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_and_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) { @@ -461,6 +925,17 @@ raw_atomic_long_fetch_and_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_and_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { @@ -471,6 +946,17 @@ raw_atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_andnot() - atomic bitwise AND NOT with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_andnot() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_andnot(long i, atomic_long_t *v) { @@ -481,6 +967,17 @@ raw_atomic_long_andnot(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_andnot() - atomic bitwise AND NOT with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_andnot() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) { @@ -491,6 +988,17 @@ raw_atomic_long_fetch_andnot(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { @@ -501,6 +1009,17 @@ raw_atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_andnot_release() - atomic bitwise AND NOT with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { @@ -511,6 +1030,17 @@ raw_atomic_long_fetch_andnot_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v & ~@i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_andnot_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { @@ -521,6 +1051,17 @@ raw_atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_or() - atomic bitwise OR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_or() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_or(long i, atomic_long_t *v) { @@ -531,6 +1072,17 @@ raw_atomic_long_or(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_or() - atomic bitwise OR with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_or() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_or(long i, atomic_long_t *v) { @@ -541,6 +1093,17 @@ raw_atomic_long_fetch_or(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_or_acquire() - atomic bitwise OR with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_or_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { @@ -551,6 +1114,17 @@ raw_atomic_long_fetch_or_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_or_release() - atomic bitwise OR with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_or_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) { @@ -561,6 +1135,17 @@ raw_atomic_long_fetch_or_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v | @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_or_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { @@ -571,6 +1156,17 @@ raw_atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_xor() - atomic bitwise XOR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_xor() elsewhere. + * + * Return: Nothing. + */ static __always_inline void raw_atomic_long_xor(long i, atomic_long_t *v) { @@ -581,6 +1177,17 @@ raw_atomic_long_xor(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_xor() - atomic bitwise XOR with full ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_xor() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_xor(long i, atomic_long_t *v) { @@ -591,6 +1198,17 @@ raw_atomic_long_fetch_xor(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_xor_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { @@ -601,6 +1219,17 @@ raw_atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_xor_release() - atomic bitwise XOR with release ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_xor_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) { @@ -611,6 +1240,17 @@ raw_atomic_long_fetch_xor_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering + * @i: long value + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v ^ @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_xor_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { @@ -621,6 +1261,17 @@ raw_atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_xchg() - atomic exchange with full ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_xchg() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_xchg(atomic_long_t *v, long new) { @@ -631,6 +1282,17 @@ raw_atomic_long_xchg(atomic_long_t *v, long new) #endif } +/** + * raw_atomic_long_xchg_acquire() - atomic exchange with acquire ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_xchg_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_xchg_acquire(atomic_long_t *v, long new) { @@ -641,6 +1303,17 @@ raw_atomic_long_xchg_acquire(atomic_long_t *v, long new) #endif } +/** + * raw_atomic_long_xchg_release() - atomic exchange with release ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_xchg_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_xchg_release(atomic_long_t *v, long new) { @@ -651,6 +1324,17 @@ raw_atomic_long_xchg_release(atomic_long_t *v, long new) #endif } +/** + * raw_atomic_long_xchg_relaxed() - atomic exchange with relaxed ordering + * @v: pointer to atomic_long_t + * @new: long value to assign + * + * Atomically updates @v to @new with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_xchg_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new) { @@ -661,6 +1345,18 @@ raw_atomic_long_xchg_relaxed(atomic_long_t *v, long new) #endif } +/** + * raw_atomic_long_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_cmpxchg() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { @@ -671,6 +1367,18 @@ raw_atomic_long_cmpxchg(atomic_long_t *v, long old, long new) #endif } +/** + * raw_atomic_long_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_cmpxchg_acquire() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { @@ -681,6 +1389,18 @@ raw_atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) #endif } +/** + * raw_atomic_long_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_cmpxchg_release() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { @@ -691,6 +1411,18 @@ raw_atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) #endif } +/** + * raw_atomic_long_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_long_t + * @old: long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_cmpxchg_relaxed() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { @@ -701,6 +1433,19 @@ raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) #endif } +/** + * raw_atomic_long_try_cmpxchg() - atomic compare and exchange with full ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with full ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { @@ -711,6 +1456,19 @@ raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) #endif } +/** + * raw_atomic_long_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with acquire ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { @@ -721,6 +1479,19 @@ raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) #endif } +/** + * raw_atomic_long_try_cmpxchg_release() - atomic compare and exchange with release ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with release ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { @@ -731,6 +1502,19 @@ raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) #endif } +/** + * raw_atomic_long_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering + * @v: pointer to atomic_long_t + * @old: pointer to long value to compare with + * @new: long value to assign + * + * If (@v == @old), atomically updates @v to @new with relaxed ordering. + * Otherwise, updates @old to the current value of @v. + * + * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere. + * + * Return: @true if the exchange occured, @false otherwise. + */ static __always_inline bool raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { @@ -741,6 +1525,17 @@ raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) #endif } +/** + * raw_atomic_long_sub_and_test() - atomic subtract and test if zero with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_sub_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic_long_sub_and_test(long i, atomic_long_t *v) { @@ -751,6 +1546,16 @@ raw_atomic_long_sub_and_test(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_and_test() - atomic decrement and test if zero with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic_long_dec_and_test(atomic_long_t *v) { @@ -761,6 +1566,16 @@ raw_atomic_long_dec_and_test(atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc_and_test() - atomic increment and test if zero with full ordering + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_and_test() elsewhere. + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ static __always_inline bool raw_atomic_long_inc_and_test(atomic_long_t *v) { @@ -771,6 +1586,17 @@ raw_atomic_long_inc_and_test(atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_negative() - atomic add and test if negative with full ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_negative() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_long_add_negative(long i, atomic_long_t *v) { @@ -781,6 +1607,17 @@ raw_atomic_long_add_negative(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_negative_acquire() - atomic add and test if negative with acquire ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with acquire ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_negative_acquire() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) { @@ -791,6 +1628,17 @@ raw_atomic_long_add_negative_acquire(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_negative_release() - atomic add and test if negative with release ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with release ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_negative_release() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_long_add_negative_release(long i, atomic_long_t *v) { @@ -801,6 +1649,17 @@ raw_atomic_long_add_negative_release(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_add_negative_relaxed() - atomic add and test if negative with relaxed ordering + * @i: long value to add + * @v: pointer to atomic_long_t + * + * Atomically updates @v to (@v + @i) with relaxed ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_negative_relaxed() elsewhere. + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ static __always_inline bool raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) { @@ -811,6 +1670,18 @@ raw_atomic_long_add_negative_relaxed(long i, atomic_long_t *v) #endif } +/** + * raw_atomic_long_fetch_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_long_t + * @a: long value to add + * @u: long value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_fetch_add_unless() elsewhere. + * + * Return: The original value of @v. + */ static __always_inline long raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { @@ -821,6 +1692,18 @@ raw_atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) #endif } +/** + * raw_atomic_long_add_unless() - atomic add unless value with full ordering + * @v: pointer to atomic_long_t + * @a: long value to add + * @u: long value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_add_unless() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) { @@ -831,6 +1714,16 @@ raw_atomic_long_add_unless(atomic_long_t *v, long a, long u) #endif } +/** + * raw_atomic_long_inc_not_zero() - atomic increment unless zero with full ordering + * @v: pointer to atomic_long_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_not_zero() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_long_inc_not_zero(atomic_long_t *v) { @@ -841,6 +1734,16 @@ raw_atomic_long_inc_not_zero(atomic_long_t *v) #endif } +/** + * raw_atomic_long_inc_unless_negative() - atomic increment unless negative with full ordering + * @v: pointer to atomic_long_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_inc_unless_negative() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_long_inc_unless_negative(atomic_long_t *v) { @@ -851,6 +1754,16 @@ raw_atomic_long_inc_unless_negative(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_unless_positive() - atomic decrement unless positive with full ordering + * @v: pointer to atomic_long_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_unless_positive() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline bool raw_atomic_long_dec_unless_positive(atomic_long_t *v) { @@ -861,6 +1774,16 @@ raw_atomic_long_dec_unless_positive(atomic_long_t *v) #endif } +/** + * raw_atomic_long_dec_if_positive() - atomic decrement if positive with full ordering + * @v: pointer to atomic_long_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. + * + * Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere. + * + * Return: @true if @v was updated, @false otherwise. + */ static __always_inline long raw_atomic_long_dec_if_positive(atomic_long_t *v) { @@ -872,4 +1795,4 @@ raw_atomic_long_dec_if_positive(atomic_long_t *v) } #endif /* _LINUX_ATOMIC_LONG_H */ -// e785d25cc3f220b7d473d36aac9da85dd7eb13a8 +// 029d2e3a493086671e874a4c2e0e42084be42403 -- cgit v1.2.3 From 642af0f92cbe01c4b05eb38a0fe94867a3798b34 Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Thu, 1 Jun 2023 16:14:51 +0200 Subject: net: mdio: Introduce a regmap-based mdio driver There exists several examples today of devices that embed an ethernet PHY or PCS directly inside an SoC. In this situation, either the device is controlled through a vendor-specific register set, or sometimes exposes the standard 802.3 registers that are typically accessed over MDIO. As phylib and phylink are designed to use mdiodevices, this driver allows creating a virtual MDIO bus, that translates mdiodev register accesses to regmap accesses. The reason we use regmap is because there are at least 3 such devices known today, 2 of them are Altera TSE PCS's, memory-mapped, exposed with a 4-byte stride in stmmac's dwmac-socfpga variant, and a 2-byte stride in altera-tse. The other one (nxp,sja1110-base-tx-mdio) is exposed over SPI. Signed-off-by: Maxime Chevallier Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/mdio/mdio-regmap.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 include/linux/mdio/mdio-regmap.h (limited to 'include/linux') diff --git a/include/linux/mdio/mdio-regmap.h b/include/linux/mdio/mdio-regmap.h new file mode 100644 index 000000000000..679d9069846b --- /dev/null +++ b/include/linux/mdio/mdio-regmap.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Driver for MMIO-Mapped MDIO devices. Some IPs expose internal PHYs or PCS + * within the MMIO-mapped area + * + * Copyright (C) 2023 Maxime Chevallier + */ +#ifndef MDIO_REGMAP_H +#define MDIO_REGMAP_H + +#include + +struct device; +struct regmap; + +struct mdio_regmap_config { + struct device *parent; + struct regmap *regmap; + char name[MII_BUS_ID_SIZE]; + u8 valid_addr; + bool autoscan; +}; + +struct mii_bus *devm_mdio_regmap_register(struct device *dev, + const struct mdio_regmap_config *config); + +#endif -- cgit v1.2.3 From 196eec4062b006575e441ef00339c3ebcea26b8d Mon Sep 17 00:00:00 2001 From: Maxime Chevallier Date: Thu, 1 Jun 2023 16:14:53 +0200 Subject: net: pcs: Drop the TSE PCS driver Now that we can easily create a mdio-device that represents a memory-mapped device that exposes an MDIO-like register layout, we don't need the Altera TSE PCS anymore, since we can use the Lynx PCS instead. Reviewed-by: Simon Horman Signed-off-by: Maxime Chevallier Signed-off-by: David S. Miller --- include/linux/pcs-altera-tse.h | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 include/linux/pcs-altera-tse.h (limited to 'include/linux') diff --git a/include/linux/pcs-altera-tse.h b/include/linux/pcs-altera-tse.h deleted file mode 100644 index 92ab9f08e835..000000000000 --- a/include/linux/pcs-altera-tse.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2022 Bootlin - * - * Maxime Chevallier - */ - -#ifndef __LINUX_PCS_ALTERA_TSE_H -#define __LINUX_PCS_ALTERA_TSE_H - -struct phylink_pcs; -struct net_device; - -struct phylink_pcs *alt_tse_pcs_create(struct net_device *ndev, - void __iomem *pcs_base, int reg_width); - -#endif /* __LINUX_PCS_ALTERA_TSE_H */ -- cgit v1.2.3 From 4739b9f3d211b3c4ce9353fbfd9f22e2bcb64c17 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 2 Jun 2023 14:58:45 +0100 Subject: net: pcs: xpcs: remove xpcs_create() from public view There are now no callers of xpcs_create(), so let's remove it from public view to discourage future direct usage. Signed-off-by: Russell King (Oracle) Reviewed-by: Vladimir Oltean Signed-off-by: David S. Miller --- include/linux/pcs/pcs-xpcs.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pcs/pcs-xpcs.h b/include/linux/pcs/pcs-xpcs.h index a99972a6d046..914e387d5387 100644 --- a/include/linux/pcs/pcs-xpcs.h +++ b/include/linux/pcs/pcs-xpcs.h @@ -35,8 +35,6 @@ int xpcs_do_config(struct dw_xpcs *xpcs, phy_interface_t interface, void xpcs_get_interfaces(struct dw_xpcs *xpcs, unsigned long *interfaces); int xpcs_config_eee(struct dw_xpcs *xpcs, int mult_fact_100ns, int enable); -struct dw_xpcs *xpcs_create(struct mdio_device *mdiodev, - phy_interface_t interface); struct dw_xpcs *xpcs_create_mdiodev(struct mii_bus *bus, int addr, phy_interface_t interface); void xpcs_destroy(struct dw_xpcs *xpcs); -- cgit v1.2.3 From e15885689cf4bc92356e52ea6ef38379a749819a Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Mon, 15 May 2023 11:35:33 +0000 Subject: x86/amd_nb: Add MI200 PCI IDs The AMD MI200 series accelerators are data center GPUs. They include unified memory controllers and a data fabric similar to those used in AMD x86 CPU products. The memory controllers report errors using MCA, though these errors are generally handled through GPU drivers that directly manage the accelerator device. In some configurations, memory errors from these devices will be reported through MCA and managed by x86 CPUs. The OS is expected to handle these errors in similar fashion to MCA errors originating from memory controllers on the CPUs. In Linux, this flow includes passing MCA errors to a notifier chain with handlers in the EDAC subsystem. The AMD64 EDAC module requires information from the memory controllers and data fabric in order to provide detailed decoding of memory errors. The information is read from hardware registers accessed through interfaces in the data fabric. The accelerator data fabrics are visible to the host x86 CPUs as PCI devices just like x86 CPU data fabrics are already. However, the accelerator fabrics have new and unique PCI IDs. Add PCI IDs for the MI200 series of accelerator devices in order to enable EDAC support. The data fabrics of the accelerator devices will be enumerated as any other fabric already supported. System-specific implementation details will be handled within the AMD64 EDAC module. [ bp: Scrub off marketing speak. ] Signed-off-by: Yazen Ghannam Co-developed-by: Muralidhara M K Signed-off-by: Muralidhara M K Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20230515113537.1052146-2-muralimk@amd.com --- include/linux/pci_ids.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 95f33dadb2be..a99b1fcfc617 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -568,6 +568,7 @@ #define PCI_DEVICE_ID_AMD_19H_M60H_DF_F3 0x14e3 #define PCI_DEVICE_ID_AMD_19H_M70H_DF_F3 0x14f3 #define PCI_DEVICE_ID_AMD_19H_M78H_DF_F3 0x12fb +#define PCI_DEVICE_ID_AMD_MI200_DF_F3 0x14d3 #define PCI_DEVICE_ID_AMD_CNB17H_F3 0x1703 #define PCI_DEVICE_ID_AMD_LANCE 0x2000 #define PCI_DEVICE_ID_AMD_LANCE_HOME 0x2001 -- cgit v1.2.3 From d0e135408e196921da2c85ee424235382c9ed614 Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Fri, 2 Jun 2023 12:33:07 +0200 Subject: highmem: Rename put_and_unmap_page() to unmap_and_put_page() With commit 849ad04cf562a ("new helper: put_and_unmap_page()"), Al Viro introduced the put_and_unmap_page() to use in those many places where we have a common pattern consisting of calls to kunmap_local() + put_page(). Obviously, first we unmap and then we put pages. Instead, the original name of this helper seems to imply that we first put and then unmap. Therefore, rename the helper and change the only known upstreamed user (i.e., fs/sysv) before this helper enters common use and might become difficult to find all call sites and instead easy to break the builds. Cc: Al Viro Signed-off-by: Fabio M. De Francesco Reviewed-by: Eric Biggers Message-Id: <20230602103307.5637-1-fmdefrancesco@gmail.com> Signed-off-by: Christian Brauner --- include/linux/highmem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 4de1dbcd3ef6..68da30625a6c 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -507,7 +507,7 @@ static inline void folio_zero_range(struct folio *folio, zero_user_segments(&folio->page, start, start + length, 0, 0); } -static inline void put_and_unmap_page(struct page *page, void *addr) +static inline void unmap_and_put_page(struct page *page, void *addr) { kunmap_local(addr); put_page(page); -- cgit v1.2.3 From a7bc2e8ddf3c8e1f5bfeb401f7ee112956cea259 Mon Sep 17 00:00:00 2001 From: chenzhiyin Date: Thu, 1 Jun 2023 05:24:00 -0400 Subject: fs.h: Optimize file struct to prevent false sharing In the syscall test of UnixBench, performance regression occurred due to false sharing. The lock and atomic members, including file::f_lock, file::f_count and file::f_pos_lock are highly contended and frequently updated in the high-concurrency test scenarios. perf c2c indentified one affected read access, file::f_op. To prevent false sharing, the layout of file struct is changed as following (A) f_lock, f_count and f_pos_lock are put together to share the same cache line. (B) The read mostly members, including f_path, f_inode, f_op are put into a separate cache line. (C) f_mode is put together with f_count, since they are used frequently at the same time. Due to '__randomize_layout' attribute of file struct, the updated layout only can be effective when CONFIG_RANDSTRUCT_NONE is 'y'. The optimization has been validated in the syscall test of UnixBench. performance gain is 30~50%. Furthermore, to confirm the optimization effectiveness on the other codes path, the results of fsdisk, fsbuffer and fstime are also shown. Here are the detailed test results of unixbench. Command: numactl -C 3-18 ./Run -c 16 syscall fsbuffer fstime fsdisk Without Patch ------------------------------------------------------------------------ File Copy 1024 bufsize 2000 maxblocks 875052.1 KBps (30.0 s, 2 samples) File Copy 256 bufsize 500 maxblocks 235484.0 KBps (30.0 s, 2 samples) File Copy 4096 bufsize 8000 maxblocks 2815153.5 KBps (30.0 s, 2 samples) System Call Overhead 5772268.3 lps (10.0 s, 7 samples) System Benchmarks Partial Index BASELINE RESULT INDEX File Copy 1024 bufsize 2000 maxblocks 3960.0 875052.1 2209.7 File Copy 256 bufsize 500 maxblocks 1655.0 235484.0 1422.9 File Copy 4096 bufsize 8000 maxblocks 5800.0 2815153.5 4853.7 System Call Overhead 15000.0 5772268.3 3848.2 ======== System Benchmarks Index Score (Partial Only) 2768.3 With Patch ------------------------------------------------------------------------ File Copy 1024 bufsize 2000 maxblocks 1009977.2 KBps (30.0 s, 2 samples) File Copy 256 bufsize 500 maxblocks 264765.9 KBps (30.0 s, 2 samples) File Copy 4096 bufsize 8000 maxblocks 3052236.0 KBps (30.0 s, 2 samples) System Call Overhead 8237404.4 lps (10.0 s, 7 samples) System Benchmarks Partial Index BASELINE RESULT INDEX File Copy 1024 bufsize 2000 maxblocks 3960.0 1009977.2 2550.4 File Copy 256 bufsize 500 maxblocks 1655.0 264765.9 1599.8 File Copy 4096 bufsize 8000 maxblocks 5800.0 3052236.0 5262.5 System Call Overhead 15000.0 8237404.4 5491.6 ======== System Benchmarks Index Score (Partial Only) 3295.3 Signed-off-by: chenzhiyin Message-Id: <20230601092400.27162-1-zhiyin.chen@intel.com> Signed-off-by: Christian Brauner --- include/linux/fs.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 21a981680856..9c2671b285a4 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -956,29 +956,35 @@ static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index) index < ra->start + ra->size); } +/* + * f_{lock,count,pos_lock} members can be highly contended and share + * the same cacheline. f_{lock,mode} are very frequently used together + * and so share the same cacheline as well. The read-mostly + * f_{path,inode,op} are kept on a separate cacheline. + */ struct file { union { struct llist_node f_llist; struct rcu_head f_rcuhead; unsigned int f_iocb_flags; }; - struct path f_path; - struct inode *f_inode; /* cached value */ - const struct file_operations *f_op; /* * Protects f_ep, f_flags. * Must not be taken from IRQ context. */ spinlock_t f_lock; - atomic_long_t f_count; - unsigned int f_flags; fmode_t f_mode; + atomic_long_t f_count; struct mutex f_pos_lock; loff_t f_pos; + unsigned int f_flags; struct fown_struct f_owner; const struct cred *f_cred; struct file_ra_state f_ra; + struct path f_path; + struct inode *f_inode; /* cached value */ + const struct file_operations *f_op; u64 f_version; #ifdef CONFIG_SECURITY -- cgit v1.2.3 From 82078b9895bd46ce69b4a73e2da40e7e2202fdb5 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 18 May 2023 13:45:36 -0400 Subject: NFSD: Ensure that xdr_write_pages updates rq_next_page All other NFSv[23] procedures manage to keep page_ptr and rq_next_page in lock step. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 762d7231e574..3b10636c51a9 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -508,6 +508,27 @@ static inline void svcxdr_init_encode(struct svc_rqst *rqstp) xdr->rqst = NULL; } +/** + * svcxdr_encode_opaque_pages - Insert pages into an xdr_stream + * @xdr: xdr_stream to be updated + * @pages: array of pages to insert + * @base: starting offset of first data byte in @pages + * @len: number of data bytes in @pages to insert + * + * After the @pages are added, the tail iovec is instantiated pointing + * to end of the head buffer, and the stream is set up to encode + * subsequent items into the tail. + */ +static inline void svcxdr_encode_opaque_pages(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + struct page **pages, + unsigned int base, + unsigned int len) +{ + xdr_write_pages(xdr, pages, base, len); + xdr->page_ptr = rqstp->rq_next_page - 1; +} + /** * svcxdr_set_auth_slack - * @rqstp: RPC transaction -- cgit v1.2.3 From b407460ee99033503993ac7437d593451fcdfe44 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 2 Jun 2023 10:50:36 +0200 Subject: iopoll: Call cpu_relax() in busy loops It is considered good practice to call cpu_relax() in busy loops, see Documentation/process/volatile-considered-harmful.rst. This can not only lower CPU power consumption or yield to a hyperthreaded twin processor, but also allows an architecture to mitigate hardware issues (e.g. ARM Erratum 754327 for Cortex-A9 prior to r2p0) in the architecture-specific cpu_relax() implementation. In addition, cpu_relax() is also a compiler barrier. It is not immediately obvious that the @op argument "function" will result in an actual function call (e.g. in case of inlining). Where a function call is a C sequence point, this is lost on inlining. Therefore, with agressive enough optimization it might be possible for the compiler to hoist the: (val) = op(args); "load" out of the loop because it doesn't see the value changing. The addition of cpu_relax() would inhibit this. As the iopoll helpers lack calls to cpu_relax(), people are sometimes reluctant to use them, and may fall back to open-coded polling loops (including cpu_relax() calls) instead. Fix this by adding calls to cpu_relax() to the iopoll helpers: - For the non-atomic case, it is sufficient to call cpu_relax() in case of a zero sleep-between-reads value, as a call to usleep_range() is a safe barrier otherwise. However, it doesn't hurt to add the call regardless, for simplicity, and for similarity with the atomic case below. - For the atomic case, cpu_relax() must be called regardless of the sleep-between-reads value, as there is no guarantee all architecture-specific implementations of udelay() handle this. Signed-off-by: Geert Uytterhoeven Acked-by: Peter Zijlstra (Intel) Acked-by: Arnd Bergmann Reviewed-by: Tony Lindgren Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/45c87bec3397fdd704376807f0eec5cc71be440f.1685692810.git.geert+renesas@glider.be --- include/linux/iopoll.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h index 2c8860e406bd..0417360a6db9 100644 --- a/include/linux/iopoll.h +++ b/include/linux/iopoll.h @@ -53,6 +53,7 @@ } \ if (__sleep_us) \ usleep_range((__sleep_us >> 2) + 1, __sleep_us); \ + cpu_relax(); \ } \ (cond) ? 0 : -ETIMEDOUT; \ }) @@ -95,6 +96,7 @@ } \ if (__delay_us) \ udelay(__delay_us); \ + cpu_relax(); \ } \ (cond) ? 0 : -ETIMEDOUT; \ }) -- cgit v1.2.3 From 7349a69cf3125e92d48e442d9f400ba446fa314f Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 2 Jun 2023 10:50:37 +0200 Subject: iopoll: Do not use timekeeping in read_poll_timeout_atomic() read_poll_timeout_atomic() uses ktime_get() to implement the timeout feature, just like its non-atomic counterpart. However, there are several issues with this, due to its use in atomic contexts: 1. When called in the s2ram path (as typically done by clock or PM domain drivers), timekeeping may be suspended, triggering the WARN_ON(timekeeping_suspended) in ktime_get(): WARNING: CPU: 0 PID: 654 at kernel/time/timekeeping.c:843 ktime_get+0x28/0x78 Calling ktime_get_mono_fast_ns() instead of ktime_get() would get rid of that warning. However, that would break timeout handling, as (at least on systems with an ARM architectured timer), the time returned by ktime_get_mono_fast_ns() does not advance while timekeeping is suspended. Interestingly, (on the same ARM systems) the time returned by ktime_get() does advance while timekeeping is suspended, despite the warning. 2. Depending on the actual clock source, and especially before a high-resolution clocksource (e.g. the ARM architectured timer) becomes available, time may not advance in atomic contexts, thus breaking timeout handling. Fix this by abandoning the idea that one can rely on timekeeping to implement timeout handling in all atomic contexts, and switch from a global time-based to a locally-estimated timeout handling. In most (all?) cases the timeout condition is exceptional and an error condition, hence any additional delays due to underestimating wall clock time are irrelevant. Signed-off-by: Geert Uytterhoeven Acked-by: Arnd Bergmann Reviewed-by: Tony Lindgren Reviewed-by: Ulf Hansson Link: https://lore.kernel.org/r/3d2a2f4e553489392d871108797c3be08f88300b.1685692810.git.geert+renesas@glider.be --- include/linux/iopoll.h | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h index 0417360a6db9..19a7b00baff4 100644 --- a/include/linux/iopoll.h +++ b/include/linux/iopoll.h @@ -74,6 +74,10 @@ * Returns 0 on success and -ETIMEDOUT upon a timeout. In either * case, the last read value at @args is stored in @val. * + * This macro does not rely on timekeeping. Hence it is safe to call even when + * timekeeping is suspended, at the expense of an underestimation of wall clock + * time, which is rather minimal with a non-zero delay_us. + * * When available, you'll probably want to use one of the specialized * macros defined below rather than this macro directly. */ @@ -81,22 +85,30 @@ delay_before_read, args...) \ ({ \ u64 __timeout_us = (timeout_us); \ + s64 __left_ns = __timeout_us * NSEC_PER_USEC; \ unsigned long __delay_us = (delay_us); \ - ktime_t __timeout = ktime_add_us(ktime_get(), __timeout_us); \ - if (delay_before_read && __delay_us) \ + u64 __delay_ns = __delay_us * NSEC_PER_USEC; \ + if (delay_before_read && __delay_us) { \ udelay(__delay_us); \ + if (__timeout_us) \ + __left_ns -= __delay_ns; \ + } \ for (;;) { \ (val) = op(args); \ if (cond) \ break; \ - if (__timeout_us && \ - ktime_compare(ktime_get(), __timeout) > 0) { \ + if (__timeout_us && __left_ns < 0) { \ (val) = op(args); \ break; \ } \ - if (__delay_us) \ + if (__delay_us) { \ udelay(__delay_us); \ + if (__timeout_us) \ + __left_ns -= __delay_ns; \ + } \ cpu_relax(); \ + if (__timeout_us) \ + __left_ns--; \ } \ (cond) ? 0 : -ETIMEDOUT; \ }) -- cgit v1.2.3 From 8be3593b9efa8903d2ee7bb9cdf57a8e56c66f36 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 28 May 2023 09:02:05 +0100 Subject: drivers/perf: apple_m1: Force 63bit counters for M2 CPUs Sidharth reports that on M2, the PMU never generates any interrupt when using 'perf record', which is a annoying as you get no sample. I'm temped to say "no sample, no problem", but others may have a different opinion. Upon investigation, it appears that the counters on M2 are significantly different from the ones on M1, as they count on 64 bits instead of 48. Which of course, in the fine M1 tradition, means that we can only use 63 bits, as the top bit is used to signal the interrupt... This results in having to introduce yet another flag to indicate yet another odd counter width. Who knows what the next crazy implementation will do... With this, perf can work out the correct offset, and 'perf record' works as intended. Tested on M2 and M2-Pro CPUs. Cc: Janne Grunau Cc: Hector Martin Cc: Mark Rutland Cc: Will Deacon Fixes: 7d0bfb7c9977 ("drivers/perf: apple_m1: Add Apple M2 support") Reported-by: Sidharth Kshatriya Tested-by: Sidharth Kshatriya Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230528080205.288446-1-maz@kernel.org Signed-off-by: Will Deacon --- include/linux/perf/arm_pmu.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 525b5d64e394..c0e4baf940dc 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -26,9 +26,11 @@ */ #define ARMPMU_EVT_64BIT 0x00001 /* Event uses a 64bit counter */ #define ARMPMU_EVT_47BIT 0x00002 /* Event uses a 47bit counter */ +#define ARMPMU_EVT_63BIT 0x00004 /* Event uses a 63bit counter */ static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_64BIT) == ARMPMU_EVT_64BIT); static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_47BIT) == ARMPMU_EVT_47BIT); +static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_63BIT) == ARMPMU_EVT_63BIT); #define HW_OP_UNSUPPORTED 0xFFFF #define C(_x) PERF_COUNT_HW_CACHE_##_x -- cgit v1.2.3 From 4f4e7112666b5aa1f179b4046299f85c09b46821 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Mon, 5 Jun 2023 16:47:57 +0200 Subject: ALSA: usb-audio: Use __le16 for 16bit USB descriptor fields Use proper notion for 16bit values for fixing the sparse warnings. Fixes: f8ddb0fb3289 ("ALSA: usb-audio: Define USB MIDI 2.0 specs") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202305260528.wcqjXso8-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202305270534.odwHL9F0-lkp@intel.com/ Link: https://lore.kernel.org/r/20230605144758.6677-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- include/linux/usb/midi-v2.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/midi-v2.h b/include/linux/usb/midi-v2.h index ebbffcae0417..16f09d959a2d 100644 --- a/include/linux/usb/midi-v2.h +++ b/include/linux/usb/midi-v2.h @@ -73,7 +73,7 @@ struct usb_ms20_gr_trm_block_header_descriptor { __u8 bLength; /* 5 */ __u8 bDescriptorType; /* USB_DT_CS_GR_TRM_BLOCK */ __u8 bDescriptorSubtype; /* USB_MS_GR_TRM_BLOCK_HEADER */ - __u16 wTotalLength; /* Total number of bytes */ + __le16 wTotalLength; /* Total number of bytes */ } __packed; /* 5.4.2.1 Group Terminal Block Descriptor */ @@ -87,8 +87,8 @@ struct usb_ms20_gr_trm_block_descriptor { __u8 nNumGroupTrm; /* Number of member Group Terminals spanned */ __u8 iBlockItem; /* String ID of Block item */ __u8 bMIDIProtocol; /* Default MIDI protocol */ - __u16 wMaxInputBandwidth; /* Max input bandwidth capability in 4kB/s */ - __u16 wMaxOutputBandwidth; /* Max output bandwidth capability in 4kB/s */ + __le16 wMaxInputBandwidth; /* Max input bandwidth capability in 4kB/s */ + __le16 wMaxOutputBandwidth; /* Max output bandwidth capability in 4kB/s */ } __packed; #endif /* __LINUX_USB_MIDI_V2_H */ -- cgit v1.2.3 From 0718afd47f70cf46877c39c25d06b786e1a3f36c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 11:44:52 +0200 Subject: block: introduce holder ops Add a new blk_holder_ops structure, which is passed to blkdev_get_by_* and installed in the block_device for exclusive claims. It will be used to allow the block layer to call back into the user of the block device for thing like notification of a removed device or a device resize. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Acked-by: Dave Chinner Reviewed-by: Dave Chinner Link: https://lore.kernel.org/r/20230601094459.1350643-10-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blk_types.h | 2 ++ include/linux/blkdev.h | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8ef209e3aa96..deb69eeab6bd 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -55,6 +55,8 @@ struct block_device { struct super_block * bd_super; void * bd_claiming; void * bd_holder; + const struct blk_holder_ops *bd_holder_ops; + struct mutex bd_holder_lock; /* The counter of freeze processes */ int bd_fsfreeze_count; int bd_holders; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d89c2da14698..44f2a8bc57e8 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1470,10 +1470,15 @@ void blkdev_show(struct seq_file *seqf, off_t offset); #define BLKDEV_MAJOR_MAX 0 #endif +struct blk_holder_ops { +}; + +struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder, + const struct blk_holder_ops *hops); struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, - void *holder); -struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder); -int bd_prepare_to_claim(struct block_device *bdev, void *holder); + void *holder, const struct blk_holder_ops *hops); +int bd_prepare_to_claim(struct block_device *bdev, void *holder, + const struct blk_holder_ops *hops); void bd_abort_claiming(struct block_device *bdev, void *holder); void blkdev_put(struct block_device *bdev, fmode_t mode); -- cgit v1.2.3 From f55e017c642051ddc01d77a89ab18f5ee71d6276 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 11:44:53 +0200 Subject: block: add a mark_dead holder operation Add a mark_dead method to blk_holder_ops that is called from blk_mark_disk_dead to notify the holder that the block device it is using has been marked dead. Signed-off-by: Christoph Hellwig Reviewed-by: Jan Kara Acked-by: Christian Brauner Acked-by: Dave Chinner Reviewed-by: Dave Chinner Link: https://lore.kernel.org/r/20230601094459.1350643-11-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 44f2a8bc57e8..9e9a9e4edee9 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1471,6 +1471,7 @@ void blkdev_show(struct seq_file *seqf, off_t offset); #endif struct blk_holder_ops { + void (*mark_dead)(struct block_device *bdev); }; struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder, -- cgit v1.2.3 From 87efb39075be6a288cd7f23858f15bd01c83028a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 11:44:54 +0200 Subject: fs: add a method to shut down the file system Add a new ->shutdown super operation that can be used to tell the file system to shut down, and call it from newly created holder ops when the block device under a file system shuts down. This only covers the main block device for "simple" file systems using get_tree_bdev / mount_bdev. File systems their own get_tree method or opening additional devices will need to set up their own blk_holder_ops. Signed-off-by: Christoph Hellwig Reviewed-by: Christian Brauner Reviewed-by: Jan Kara Reviewed-by: Darrick J. Wong Acked-by: Dave Chinner Reviewed-by: Dave Chinner Link: https://lore.kernel.org/r/20230601094459.1350643-12-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/fs.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 08ba2ae1d3ce..7b2053649820 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1932,6 +1932,7 @@ struct super_operations { struct shrink_control *); long (*free_cached_objects)(struct super_block *, struct shrink_control *); + void (*shutdown)(struct super_block *sb); }; /* -- cgit v1.2.3 From aa5f6ed8c21ec1aa5fd688118d8d5cd87c5ffc1d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:12 +0200 Subject: driver core: return bool from driver_probe_done bool is the most sensible return value for a yes/no return. Also add __init as this funtion is only called from the early boot code. Signed-off-by: Christoph Hellwig Acked-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20230531125535.676098-2-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/device/driver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index c244267a6744..7738f458995f 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -126,7 +126,7 @@ int __must_check driver_register(struct device_driver *drv); void driver_unregister(struct device_driver *drv); struct device_driver *driver_find(const char *name, const struct bus_type *bus); -int driver_probe_done(void); +bool __init driver_probe_done(void); void wait_for_device_probe(void); void __init wait_for_init_devices_probe(void); -- cgit v1.2.3 From f5524c3fadba35c075a5131bad74e3041507a694 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:16 +0200 Subject: init: remove pointless Root_* values Remove all unused defines, and just use the expanded versions for the SCSI disk majors. I've decided to keep Root_RAM0 even if it could be expanded as there is a lot of special casing for it in the init code. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230531125535.676098-6-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/root_dev.h | 8 -------- 1 file changed, 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/root_dev.h b/include/linux/root_dev.h index 4e78651371ba..ed3ea8da6429 100644 --- a/include/linux/root_dev.h +++ b/include/linux/root_dev.h @@ -10,14 +10,6 @@ enum { Root_NFS = MKDEV(UNNAMED_MAJOR, 255), Root_CIFS = MKDEV(UNNAMED_MAJOR, 254), Root_RAM0 = MKDEV(RAMDISK_MAJOR, 0), - Root_RAM1 = MKDEV(RAMDISK_MAJOR, 1), - Root_FD0 = MKDEV(FLOPPY_MAJOR, 0), - Root_HDA1 = MKDEV(IDE0_MAJOR, 1), - Root_HDA2 = MKDEV(IDE0_MAJOR, 2), - Root_SDA1 = MKDEV(SCSI_DISK0_MAJOR, 1), - Root_SDA2 = MKDEV(SCSI_DISK0_MAJOR, 2), - Root_HDC1 = MKDEV(IDE1_MAJOR, 1), - Root_SR0 = MKDEV(SCSI_CDROM_MAJOR, 0), }; extern dev_t ROOT_DEV; -- cgit v1.2.3 From 07d63cbb67cdb5e2a7720fdd8579b3be979c2d66 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:21 +0200 Subject: init: handle ubi/mtd root mounting like all other root types Assign a Root_Generic magic value for UBI/MTD root and handle the root mounting in mount_root like all other root types. Besides making the code more clear this also means that UBI/MTD root can be used together with an initrd (not that anyone should care). Also factor parsing of the root name into a helper now that it can be easily done and will get more complicated with subsequent patches. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230531125535.676098-11-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/root_dev.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/root_dev.h b/include/linux/root_dev.h index ed3ea8da6429..847c9a06101b 100644 --- a/include/linux/root_dev.h +++ b/include/linux/root_dev.h @@ -9,6 +9,7 @@ enum { Root_NFS = MKDEV(UNNAMED_MAJOR, 255), Root_CIFS = MKDEV(UNNAMED_MAJOR, 254), + Root_Generic = MKDEV(UNNAMED_MAJOR, 253), Root_RAM0 = MKDEV(RAMDISK_MAJOR, 0), }; -- cgit v1.2.3 From cf056a43121559d3642419917d405c3237ded90a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:24 +0200 Subject: init: improve the name_to_dev_t interface name_to_dev_t has a very misleading name, that doesn't make clear it should only be used by the early init code, and also has a bad calling convention that doesn't allow returning different kinds of errors. Rename it to early_lookup_bdev to make the use case clear, and return an errno, where -EINVAL means the string could not be parsed, and -ENODEV means it the string was valid, but there was no device found for it. Also stub out the whole call for !CONFIG_BLOCK as all the non-block root cases are always covered in the caller. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230531125535.676098-14-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 5 +++++ include/linux/mount.h | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 9e9a9e4edee9..d682e233fd66 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1501,6 +1501,7 @@ int sync_blockdev_nowait(struct block_device *bdev); void sync_bdevs(bool wait); void bdev_statx_dioalign(struct inode *inode, struct kstat *stat); void printk_all_partitions(void); +int early_lookup_bdev(const char *pathname, dev_t *dev); #else static inline void invalidate_bdev(struct block_device *bdev) { @@ -1522,6 +1523,10 @@ static inline void bdev_statx_dioalign(struct inode *inode, struct kstat *stat) static inline void printk_all_partitions(void) { } +static inline int early_lookup_bdev(const char *pathname, dev_t *dev) +{ + return -EINVAL; +} #endif /* CONFIG_BLOCK */ int fsync_bdev(struct block_device *bdev); diff --git a/include/linux/mount.h b/include/linux/mount.h index 1ea326c368f7..4b81ea90440e 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -107,7 +107,6 @@ extern struct vfsmount *vfs_submount(const struct dentry *mountpoint, extern void mnt_set_expiry(struct vfsmount *mnt, struct list_head *expiry_list); extern void mark_mounts_for_expiry(struct list_head *mounts); -extern dev_t name_to_dev_t(const char *name); extern bool path_is_mountpoint(const struct path *path); extern bool our_mnt(struct vfsmount *mnt); -- cgit v1.2.3 From 7cadcaf1d82618852745e7206fffa2c72c17ce4b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:27 +0200 Subject: block: move more code to early-lookup.c blk_lookup_devt is only used by code in early-lookup.c, so move it there. printk_all_partitions and it's helper bdevt_str are only used by the early init code in init/do_mounts.c, so they should go there as well. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230531125535.676098-17-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d682e233fd66..52718176d1b4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -838,7 +838,6 @@ static inline void bd_unlink_disk_holder(struct block_device *bdev, dev_t part_devt(struct gendisk *disk, u8 partno); void inc_diskseq(struct gendisk *disk); -dev_t blk_lookup_devt(const char *name, int partno); void blk_request_module(dev_t devt); extern int blk_register_queue(struct gendisk *disk); -- cgit v1.2.3 From d4a28d7defe79006e59293a4b43d518ba8483fb0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:30 +0200 Subject: dm: remove dm_get_dev_t Open code dm_get_dev_t in the only remaining caller, and propagate the exact error code from lookup_bdev and early_lookup_bdev. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230531125535.676098-20-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/device-mapper.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index a52d2b9a6846..c27b84002d83 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -170,8 +170,6 @@ struct dm_dev { char name[16]; }; -dev_t dm_get_dev_t(const char *path); - /* * Constructors should call these functions to ensure destination devices * are opened/closed correctly. -- cgit v1.2.3 From 2577f53f42947d8ca01666e3444bb7307319ea38 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 31 May 2023 14:55:35 +0200 Subject: block: mark early_lookup_bdev as __init early_lookup_bdev is now only used during the early boot code as it should, so mark it __init to not waste run time memory on it. Signed-off-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230531125535.676098-25-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 52718176d1b4..f4c339d9dd03 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1500,7 +1500,7 @@ int sync_blockdev_nowait(struct block_device *bdev); void sync_bdevs(bool wait); void bdev_statx_dioalign(struct inode *inode, struct kstat *stat); void printk_all_partitions(void); -int early_lookup_bdev(const char *pathname, dev_t *dev); +int __init early_lookup_bdev(const char *pathname, dev_t *dev); #else static inline void invalidate_bdev(struct block_device *bdev) { -- cgit v1.2.3 From d5e1586617be7093ea3419e3fa9387ed833cdbb1 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 2 Jun 2023 10:42:53 +0200 Subject: sched: Unconditionally use full-fat wait_task_inactive() While modifying wait_task_inactive() for PREEMPT_RT; the build robot noted that UP got broken. This led to audit and consideration of the UP implementation of wait_task_inactive(). It looks like the UP implementation is also broken for PREEMPT; consider task_current_syscall() getting preempted between the two calls to wait_task_inactive(). Therefore move the wait_task_inactive() implementation out of CONFIG_SMP and unconditionally use it. Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20230602103731.GA630648%40hirez.programming.kicks-ass.net --- include/linux/sched.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index eed5d65b8d1f..1292d38d66cc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2006,15 +2006,12 @@ static __always_inline void scheduler_ipi(void) */ preempt_fold_need_resched(); } -extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state); #else static inline void scheduler_ipi(void) { } -static inline unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) -{ - return 1; -} #endif +extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state); + /* * Set thread flags in other task's structures. * See asm/thread_info.h for TIF_xxxx flags available: -- cgit v1.2.3 From d16317de9b412aa7bd3598c607112298e36b4352 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 19 May 2023 12:20:59 +0200 Subject: seqlock/latch: Provide raw_read_seqcount_latch_retry() The read side of seqcount_latch consists of: do { seq = raw_read_seqcount_latch(&latch->seq); ... } while (read_seqcount_latch_retry(&latch->seq, seq)); which is asymmetric in the raw_ department, and sure enough, read_seqcount_latch_retry() includes (explicit) instrumentation where raw_read_seqcount_latch() does not. This inconsistency becomes a problem when trying to use it from noinstr code. As such, fix it by renaming and re-implementing raw_read_seqcount_latch_retry() without the instrumentation. Specifically the instrumentation in question is kcsan_atomic_next(0) in do___read_seqcount_retry(). Loosing this annotation is not a problem because raw_read_seqcount_latch() does not pass through kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX). Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Thomas Gleixner Reviewed-by: Petr Mladek Tested-by: Michael Kelley # Hyper-V Link: https://lore.kernel.org/r/20230519102715.233598176@infradead.org --- include/linux/rbtree_latch.h | 2 +- include/linux/seqlock.h | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rbtree_latch.h b/include/linux/rbtree_latch.h index 3d1a9e716b80..6a0999c26c7c 100644 --- a/include/linux/rbtree_latch.h +++ b/include/linux/rbtree_latch.h @@ -206,7 +206,7 @@ latch_tree_find(void *key, struct latch_tree_root *root, do { seq = raw_read_seqcount_latch(&root->seq); node = __lt_find(key, root, seq & 1, ops->comp); - } while (read_seqcount_latch_retry(&root->seq, seq)); + } while (raw_read_seqcount_latch_retry(&root->seq, seq)); return node; } diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 3926e9027947..987a59d977c5 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -671,9 +671,9 @@ typedef struct { * * Return: sequence counter raw value. Use the lowest bit as an index for * picking which data copy to read. The full counter must then be checked - * with read_seqcount_latch_retry(). + * with raw_read_seqcount_latch_retry(). */ -static inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s) +static __always_inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s) { /* * Pairs with the first smp_wmb() in raw_write_seqcount_latch(). @@ -683,16 +683,17 @@ static inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s) } /** - * read_seqcount_latch_retry() - end a seqcount_latch_t read section + * raw_read_seqcount_latch_retry() - end a seqcount_latch_t read section * @s: Pointer to seqcount_latch_t * @start: count, from raw_read_seqcount_latch() * * Return: true if a read section retry is required, else false */ -static inline int -read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) +static __always_inline int +raw_read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) { - return read_seqcount_retry(&s->seqcount, start); + smp_rmb(); + return unlikely(READ_ONCE(s->seqcount.sequence) != start); } /** @@ -752,7 +753,7 @@ read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start) * entry = data_query(latch->data[idx], ...); * * // This includes needed smp_rmb() - * } while (read_seqcount_latch_retry(&latch->seq, seq)); + * } while (raw_read_seqcount_latch_retry(&latch->seq, seq)); * * return entry; * } -- cgit v1.2.3 From fc4a0db4149afcdae2527f0d8c376accca34adc9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 19 May 2023 12:21:05 +0200 Subject: math64: Always inline u128 version of mul_u64_u64_shr() In order to prevent the following complaint from happening, always inline the u128 variant of mul_u64_u64_shr() -- which is what x86_64 will use. vmlinux.o: warning: objtool: read_hv_sched_clock_tsc+0x5a: call to mul_u64_u64_shr.constprop.0() leaves .noinstr.text section It should compile into something like: asm("mul %[mul];" "shrd %rdx, %rax, %cl" : "+&a" (a) : "c" shift, [mul] "r" (mul) : "d"); Which is silly not to inline, but it happens. Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley # Hyper-V Link: https://lore.kernel.org/r/20230519102715.637420396@infradead.org --- include/linux/math64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/math64.h b/include/linux/math64.h index 8b9191a2849e..bf74478926d4 100644 --- a/include/linux/math64.h +++ b/include/linux/math64.h @@ -168,7 +168,7 @@ static __always_inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift) #endif /* mul_u64_u32_shr */ #ifndef mul_u64_u64_shr -static inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift) +static __always_inline u64 mul_u64_u64_shr(u64 a, u64 mul, unsigned int shift) { return (u64)(((unsigned __int128)a * mul) >> shift); } -- cgit v1.2.3 From fb7d4948c4da2dbd26da4b7ec76bbd2f19ff862a Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 19 May 2023 12:21:10 +0200 Subject: sched/clock: Provide local_clock_noinstr() Now that all ARCH_WANTS_NO_INSTR architectures (arm64, loongarch, s390, x86) provide sched_clock_noinstr(), use this to provide local_clock_noinstr(). This local_clock_noinstr() will be safe to use from noinstr code with the assumption that any such noinstr code is non-preemptible (it had better be, entry code will have IRQs disabled while __cpuidle must have preemption disabled). Specifically, preempt_enable_notrace(), a common part of many a sched_clock() implementation calls out to schedule() -- even though, per the above, it will never trigger -- which frustrates noinstr validation. vmlinux.o: warning: objtool: local_clock+0xb5: call to preempt_schedule_notrace_thunk() leaves .noinstr.text section Signed-off-by: Peter Zijlstra (Intel) Tested-by: Michael Kelley # Hyper-V Link: https://lore.kernel.org/r/20230519102715.978624636@infradead.org --- include/linux/sched/clock.h | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched/clock.h b/include/linux/sched/clock.h index ca008f7d3615..196f0ca351a2 100644 --- a/include/linux/sched/clock.h +++ b/include/linux/sched/clock.h @@ -12,7 +12,16 @@ * * Please use one of the three interfaces below. */ -extern unsigned long long notrace sched_clock(void); +extern u64 sched_clock(void); + +#if defined(CONFIG_ARCH_WANTS_NO_INSTR) || defined(CONFIG_GENERIC_SCHED_CLOCK) +extern u64 sched_clock_noinstr(void); +#else +static __always_inline u64 sched_clock_noinstr(void) +{ + return sched_clock(); +} +#endif /* * See the comment in kernel/sched/clock.c @@ -45,6 +54,11 @@ static inline u64 cpu_clock(int cpu) return sched_clock(); } +static __always_inline u64 local_clock_noinstr(void) +{ + return sched_clock_noinstr(); +} + static __always_inline u64 local_clock(void) { return sched_clock(); @@ -79,6 +93,7 @@ static inline u64 cpu_clock(int cpu) return sched_clock_cpu(cpu); } +extern u64 local_clock_noinstr(void); extern u64 local_clock(void); #endif -- cgit v1.2.3 From 7a113ff6355944283402fb617dc97122f68d5a41 Mon Sep 17 00:00:00 2001 From: Andrzej Hajda Date: Fri, 2 Jun 2023 12:21:33 +0200 Subject: lib/ref_tracker: add unlocked leak print helper To have reliable detection of leaks, caller must be able to check under the same lock both: tracked counter and the leaks. dir.lock is natural candidate for such lock and unlocked print helper can be called with this lock taken. As a bonus we can reuse this helper in ref_tracker_dir_exit. Signed-off-by: Andrzej Hajda Reviewed-by: Andi Shyti Reviewed-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/ref_tracker.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ref_tracker.h b/include/linux/ref_tracker.h index 9ca353ab712b..87a92f2bec1b 100644 --- a/include/linux/ref_tracker.h +++ b/include/linux/ref_tracker.h @@ -36,6 +36,9 @@ static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, void ref_tracker_dir_exit(struct ref_tracker_dir *dir); +void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, + unsigned int display_limit); + void ref_tracker_dir_print(struct ref_tracker_dir *dir, unsigned int display_limit); @@ -56,6 +59,11 @@ static inline void ref_tracker_dir_exit(struct ref_tracker_dir *dir) { } +static inline void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, + unsigned int display_limit) +{ +} + static inline void ref_tracker_dir_print(struct ref_tracker_dir *dir, unsigned int display_limit) { -- cgit v1.2.3 From b6d7c0eb2dcbd238fa233a3a1737654e380e784a Mon Sep 17 00:00:00 2001 From: Andrzej Hajda Date: Fri, 2 Jun 2023 12:21:34 +0200 Subject: lib/ref_tracker: improve printing stats In case the library is tracking busy subsystem, simply printing stack for every active reference will spam log with long, hard to read, redundant stack traces. To improve readabilty following changes have been made: - reports are printed per stack_handle - log is more compact, - added display name for ref_tracker_dir - it will differentiate multiple subsystems, - stack trace is printed indented, in the same printk call, - info about dropped references is printed as well. Signed-off-by: Andrzej Hajda Reviewed-by: Andi Shyti Reviewed-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/ref_tracker.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ref_tracker.h b/include/linux/ref_tracker.h index 87a92f2bec1b..19a69e7809d6 100644 --- a/include/linux/ref_tracker.h +++ b/include/linux/ref_tracker.h @@ -17,12 +17,15 @@ struct ref_tracker_dir { bool dead; struct list_head list; /* List of active trackers */ struct list_head quarantine; /* List of dead trackers */ + char name[32]; #endif }; #ifdef CONFIG_REF_TRACKER + static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, - unsigned int quarantine_count) + unsigned int quarantine_count, + const char *name) { INIT_LIST_HEAD(&dir->list); INIT_LIST_HEAD(&dir->quarantine); @@ -31,6 +34,7 @@ static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, dir->dead = false; refcount_set(&dir->untracked, 1); refcount_set(&dir->no_tracker, 1); + strscpy(dir->name, name, sizeof(dir->name)); stack_depot_init(); } @@ -51,7 +55,8 @@ int ref_tracker_free(struct ref_tracker_dir *dir, #else /* CONFIG_REF_TRACKER */ static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, - unsigned int quarantine_count) + unsigned int quarantine_count, + const char *name) { } -- cgit v1.2.3 From 227c6c832303cec3941166d3335ecbccd980d615 Mon Sep 17 00:00:00 2001 From: Andrzej Hajda Date: Fri, 2 Jun 2023 12:21:35 +0200 Subject: lib/ref_tracker: add printing to memory buffer Similar to stack_(depot|trace)_snprint the patch adds helper to printing stats to memory buffer. It will be helpful in case of debugfs. Signed-off-by: Andrzej Hajda Reviewed-by: Andi Shyti Reviewed-by: Eric Dumazet Signed-off-by: Jakub Kicinski --- include/linux/ref_tracker.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ref_tracker.h b/include/linux/ref_tracker.h index 19a69e7809d6..8eac4f3d5254 100644 --- a/include/linux/ref_tracker.h +++ b/include/linux/ref_tracker.h @@ -46,6 +46,8 @@ void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir, void ref_tracker_dir_print(struct ref_tracker_dir *dir, unsigned int display_limit); +int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size); + int ref_tracker_alloc(struct ref_tracker_dir *dir, struct ref_tracker **trackerp, gfp_t gfp); @@ -74,6 +76,12 @@ static inline void ref_tracker_dir_print(struct ref_tracker_dir *dir, { } +static inline int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, + char *buf, size_t size) +{ + return 0; +} + static inline int ref_tracker_alloc(struct ref_tracker_dir *dir, struct ref_tracker **trackerp, gfp_t gfp) -- cgit v1.2.3 From d01a77afd6bef1b3a2ed15e8ca6887ca7da0cddc Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 5 Jun 2023 20:05:52 +0300 Subject: lib/string_helpers: Change returned value of the strreplace() It's more useful to return the pointer to the string itself with strreplace(), so it may be used like attr->name = strreplace(name, '/', '_'); While at it, amend the kernel documentation. Signed-off-by: Andy Shevchenko Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20230605170553.7835-3-andriy.shevchenko@linux.intel.com --- include/linux/string.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/string.h b/include/linux/string.h index c062c581a98b..dbfc66400050 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -169,7 +169,7 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) #endif void *memchr_inv(const void *s, int c, size_t n); -char *strreplace(char *s, char old, char new); +char *strreplace(char *str, char old, char new); extern void kfree_const(const void *x); -- cgit v1.2.3 From 943211c87427f25bd22e0e63849fb486bb5f87fa Mon Sep 17 00:00:00 2001 From: Siddh Raman Pant Date: Mon, 5 Jun 2023 20:06:16 +0530 Subject: watch_queue: prevent dangling pipe pointer NULL the dangling pipe reference while clearing watch_queue. If not done, a reference to a freed pipe remains in the watch_queue, as this function is called before freeing a pipe in free_pipe_info() (see line 834 of fs/pipe.c). The sole use of wqueue->defunct is for checking if the watch queue has been cleared, but wqueue->pipe is also NULLed while clearing. Thus, wqueue->defunct is superfluous, as wqueue->pipe can be checked for NULL. Hence, the former can be removed. Tested with keyutils testsuite. Cc: stable@vger.kernel.org # 6.1 Signed-off-by: Siddh Raman Pant Acked-by: David Howells Message-Id: <20230605143616.640517-1-code@siddh.me> Signed-off-by: Christian Brauner --- include/linux/watch_queue.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/watch_queue.h b/include/linux/watch_queue.h index fc6bba20273b..45cd42f55d49 100644 --- a/include/linux/watch_queue.h +++ b/include/linux/watch_queue.h @@ -38,7 +38,7 @@ struct watch_filter { struct watch_queue { struct rcu_head rcu; struct watch_filter __rcu *filter; - struct pipe_inode_info *pipe; /* The pipe we're using as a buffer */ + struct pipe_inode_info *pipe; /* Pipe we use as a buffer, NULL if queue closed */ struct hlist_head watches; /* Contributory watches */ struct page **notes; /* Preallocated notifications */ unsigned long *notes_bitmap; /* Allocation bitmap for notes */ @@ -46,7 +46,6 @@ struct watch_queue { spinlock_t lock; unsigned int nr_notes; /* Number of notes */ unsigned int nr_pages; /* Number of pages in notes[] */ - bool defunct; /* T when queues closed */ }; /* -- cgit v1.2.3 From 08dbff230048ec2812c33e78f81855635a3c1734 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Thu, 4 May 2023 16:45:08 +0300 Subject: wifi: mac80211: skip EHT BSS membership selector Skip the EHT BSS membership selector for getting rates. While at it, add the definitions for GLK and EPS, and sort the list. Signed-off-by: Johannes Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230504134511.828474-9-gregory.greenman@intel.com Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index c4cf296e7eaf..c271184a3968 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -1349,8 +1349,11 @@ struct ieee80211_mgmt { /* Supported rates membership selectors */ #define BSS_MEMBERSHIP_SELECTOR_HT_PHY 127 #define BSS_MEMBERSHIP_SELECTOR_VHT_PHY 126 -#define BSS_MEMBERSHIP_SELECTOR_HE_PHY 122 +#define BSS_MEMBERSHIP_SELECTOR_GLK 125 +#define BSS_MEMBERSHIP_SELECTOR_EPS 124 #define BSS_MEMBERSHIP_SELECTOR_SAE_H2E 123 +#define BSS_MEMBERSHIP_SELECTOR_HE_PHY 122 +#define BSS_MEMBERSHIP_SELECTOR_EHT_PHY 121 /* mgmt header + 1 byte category code */ #define IEEE80211_MIN_ACTION_SIZE offsetof(struct ieee80211_mgmt, u.action.u) -- cgit v1.2.3 From ce2bb3b66273d7d122c5dbf8f1e58e8ebc82c5fb Mon Sep 17 00:00:00 2001 From: Emmanuel Grumbach Date: Thu, 4 May 2023 16:45:10 +0300 Subject: wifi: mac80211: fetch and store the EML capability information We need to teach the low level driver about the EML capability which includes information for EMLSR / EMLMR operation. Signed-off-by: Emmanuel Grumbach Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230504134511.828474-11-gregory.greenman@intel.com Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index c271184a3968..fba4c44da832 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4635,6 +4635,41 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) return sizeof(*mle) + common + mle->variable[0]; } +/** + * ieee80211_mle_get_eml_cap - returns the EML capability + * @data: pointer to the multi link EHT IE + * + * The element is assumed to be big enough. This must be checked by + * ieee80211_mle_size_ok(). + * If the EML capability can't be found (the type is not basic, or + * the EML capability presence bit is clear), 0 will be returned. + */ +static inline u16 ieee80211_mle_get_eml_cap(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) != + IEEE80211_ML_CONTROL_TYPE_BASIC) + return 0; + + /* common points now at the beginning of ieee80211_mle_basic_common_info */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_EML_CAPA)) + return 0; + + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY) + common += 2; + + return get_unaligned_le16(common); +} + /** * ieee80211_mle_size_ok - validate multi-link element size * @data: pointer to the element data -- cgit v1.2.3 From 29c6e2dc3d12a188a48f2a45759e8da44840546b Mon Sep 17 00:00:00 2001 From: Emmanuel Grumbach Date: Sun, 4 Jun 2023 12:11:17 +0300 Subject: wifi: mac80211: provide a helper to fetch the medium synchronization delay There are drivers which need this information. Signed-off-by: Emmanuel Grumbach Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230604120651.b1043f3126e2.Iad3806f8bf8df07f52ef0a02cc3d0373c44a8c93@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index fba4c44da832..516cd32d6196 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4635,6 +4635,41 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) return sizeof(*mle) + common + mle->variable[0]; } +/** + * ieee80211_mle_get_eml_sync_delay - returns the medium sync delay + * @data: pointer to the multi link EHT IE + * + * The element is assumed to be big enough. This must be checked by + * ieee80211_mle_size_ok(). + * If the medium synchronization can't be found (the type is not basic, or + * the medium sync presence bit is clear), 0 will be returned. + */ +static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) != + IEEE80211_ML_CONTROL_TYPE_BASIC) + return 0; + + /* common points now at the beginning of + * ieee80211_mle_basic_common_info + */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)) + return 0; + + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) + common += 1; + + return get_unaligned_le16(common); +} + /** * ieee80211_mle_get_eml_cap - returns the EML capability * @data: pointer to the multi link EHT IE -- cgit v1.2.3 From 758cd5fc13b20a5874d33b7d381e78408743f587 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Wed, 31 May 2023 16:20:38 +0100 Subject: firmware: arm_scmi: Add Powercap protocol enable support SCMI powercap protocol v3.2 supports disabling the powercap on a zone by zone basis by providing a zero valued powercap. Expose new operations to enable/disable powercapping on a per-zone base. Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20230531152039.2363181-3-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- include/linux/scmi_protocol.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux') diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 0ce5746a4470..e6fe4f73ffe6 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -629,11 +629,25 @@ struct scmi_powercap_info { * @num_domains_get: get the count of powercap domains provided by SCMI. * @info_get: get the information for the specified domain. * @cap_get: get the current CAP value for the specified domain. + * On SCMI platforms supporting powercap zone disabling, this could + * report a zero value for a zone where powercapping is disabled. * @cap_set: set the CAP value for the specified domain to the provided value; * if the domain supports setting the CAP with an asynchronous command * this request will finally trigger an asynchronous transfer, but, if * @ignore_dresp here is set to true, this call will anyway return * immediately without waiting for the related delayed response. + * Note that the powercap requested value must NOT be zero, even if + * the platform supports disabling a powercap by setting its cap to + * zero (since SCMI v3.2): there are dedicated operations that should + * be used for that. (@cap_enable_set/get) + * @cap_enable_set: enable or disable the powercapping on the specified domain, + * if supported by the SCMI platform implementation. + * Note that, by the SCMI specification, the platform can + * silently ignore our disable request and decide to enforce + * anyway some other powercap value requested by another agent + * on the system: for this reason @cap_get and @cap_enable_get + * will always report the final platform view of the powercaps. + * @cap_enable_get: get the current CAP enable status for the specified domain. * @pai_get: get the current PAI value for the specified domain. * @pai_set: set the PAI value for the specified domain to the provided value. * @measurements_get: retrieve the current average power measurements for the @@ -662,6 +676,10 @@ struct scmi_powercap_proto_ops { u32 *power_cap); int (*cap_set)(const struct scmi_protocol_handle *ph, u32 domain_id, u32 power_cap, bool ignore_dresp); + int (*cap_enable_set)(const struct scmi_protocol_handle *ph, + u32 domain_id, bool enable); + int (*cap_enable_get)(const struct scmi_protocol_handle *ph, + u32 domain_id, bool *enable); int (*pai_get)(const struct scmi_protocol_handle *ph, u32 domain_id, u32 *pai); int (*pai_set)(const struct scmi_protocol_handle *ph, u32 domain_id, -- cgit v1.2.3 From dcdfdd40fa82b6704d2841938e5c8ec3051eb0d6 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Tue, 6 Jun 2023 17:26:29 +0300 Subject: mm: Add support for unaccepted memory UEFI Specification version 2.9 introduces the concept of memory acceptance. Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, require memory to be accepted before it can be used by the guest. Accepting happens via a protocol specific to the Virtual Machine platform. There are several ways the kernel can deal with unaccepted memory: 1. Accept all the memory during boot. It is easy to implement and it doesn't have runtime cost once the system is booted. The downside is very long boot time. Accept can be parallelized to multiple CPUs to keep it manageable (i.e. via DEFERRED_STRUCT_PAGE_INIT), but it tends to saturate memory bandwidth and does not scale beyond the point. 2. Accept a block of memory on the first use. It requires more infrastructure and changes in page allocator to make it work, but it provides good boot time. On-demand memory accept means latency spikes every time kernel steps onto a new memory block. The spikes will go away once workload data set size gets stabilized or all memory gets accepted. 3. Accept all memory in background. Introduce a thread (or multiple) that gets memory accepted proactively. It will minimize time the system experience latency spikes on memory allocation while keeping low boot time. This approach cannot function on its own. It is an extension of #2: background memory acceptance requires functional scheduler, but the page allocator may need to tap into unaccepted memory before that. The downside of the approach is that these threads also steal CPU cycles and memory bandwidth from the user's workload and may hurt user experience. Implement #1 and #2 for now. #2 is the default. Some workloads may want to use #1 with accept_memory=eager in kernel command line. #3 can be implemented later based on user's demands. Support of unaccepted memory requires a few changes in core-mm code: - memblock accepts memory on allocation. It serves early boot memory allocations and doesn't limit them to pre-accepted pool of memory. - page allocator accepts memory on the first allocation of the page. When kernel runs out of accepted memory, it accepts memory until the high watermark is reached. It helps to minimize fragmentation. EFI code will provide two helpers if the platform supports unaccepted memory: - accept_memory() makes a range of physical addresses accepted. - range_contains_unaccepted_memory() checks anything within the range of physical addresses requires acceptance. Signed-off-by: Kirill A. Shutemov Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Vlastimil Babka Acked-by: Mike Rapoport # memblock Link: https://lore.kernel.org/r/20230606142637.5171-2-kirill.shutemov@linux.intel.com --- include/linux/mm.h | 19 +++++++++++++++++++ include/linux/mmzone.h | 8 ++++++++ 2 files changed, 27 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 27ce77080c79..d9174d464348 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3816,4 +3816,23 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start, } #endif +#ifdef CONFIG_UNACCEPTED_MEMORY + +bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end); +void accept_memory(phys_addr_t start, phys_addr_t end); + +#else + +static inline bool range_contains_unaccepted_memory(phys_addr_t start, + phys_addr_t end) +{ + return false; +} + +static inline void accept_memory(phys_addr_t start, phys_addr_t end) +{ +} + +#endif + #endif /* _LINUX_MM_H */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a4889c9d4055..6c1c2fc13017 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -143,6 +143,9 @@ enum zone_stat_item { NR_ZSPAGES, /* allocated in zsmalloc */ #endif NR_FREE_CMA_PAGES, +#ifdef CONFIG_UNACCEPTED_MEMORY + NR_UNACCEPTED, +#endif NR_VM_ZONE_STAT_ITEMS }; enum node_stat_item { @@ -910,6 +913,11 @@ struct zone { /* free areas of different sizes */ struct free_area free_area[MAX_ORDER + 1]; +#ifdef CONFIG_UNACCEPTED_MEMORY + /* Pages to be accepted. All pages on the list are MAX_ORDER */ + struct list_head unaccepted_pages; +#endif + /* zone flags, see below */ unsigned long flags; -- cgit v1.2.3 From 745e3ed85f71a6382a239b03d9278a8025f2beae Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Tue, 6 Jun 2023 17:26:31 +0300 Subject: efi/libstub: Implement support for unaccepted memory UEFI Specification version 2.9 introduces the concept of memory acceptance: Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, requiring memory to be accepted before it can be used by the guest. Accepting happens via a protocol specific for the Virtual Machine platform. Accepting memory is costly and it makes VMM allocate memory for the accepted guest physical address range. It's better to postpone memory acceptance until memory is needed. It lowers boot time and reduces memory overhead. The kernel needs to know what memory has been accepted. Firmware communicates this information via memory map: a new memory type -- EFI_UNACCEPTED_MEMORY -- indicates such memory. Range-based tracking works fine for firmware, but it gets bulky for the kernel: e820 (or whatever the arch uses) has to be modified on every page acceptance. It leads to table fragmentation and there's a limited number of entries in the e820 table. Another option is to mark such memory as usable in e820 and track if the range has been accepted in a bitmap. One bit in the bitmap represents a naturally aligned power-2-sized region of address space -- unit. For x86, unit size is 2MiB: 4k of the bitmap is enough to track 64GiB or physical address space. In the worst-case scenario -- a huge hole in the middle of the address space -- It needs 256MiB to handle 4PiB of the address space. Any unaccepted memory that is not aligned to unit_size gets accepted upfront. The bitmap is allocated and constructed in the EFI stub and passed down to the kernel via EFI configuration table. allocate_e820() allocates the bitmap if unaccepted memory is present, according to the size of unaccepted region. Signed-off-by: Kirill A. Shutemov Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230606142637.5171-4-kirill.shutemov@linux.intel.com --- include/linux/efi.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/efi.h b/include/linux/efi.h index 571d1a6e1b74..8ffe451a6a2f 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -108,7 +108,8 @@ typedef struct { #define EFI_MEMORY_MAPPED_IO_PORT_SPACE 12 #define EFI_PAL_CODE 13 #define EFI_PERSISTENT_MEMORY 14 -#define EFI_MAX_MEMORY_TYPE 15 +#define EFI_UNACCEPTED_MEMORY 15 +#define EFI_MAX_MEMORY_TYPE 16 /* Attribute values: */ #define EFI_MEMORY_UC ((u64)0x0000000000000001ULL) /* uncached */ @@ -417,6 +418,7 @@ void efi_native_runtime_setup(void); #define LINUX_EFI_MOK_VARIABLE_TABLE_GUID EFI_GUID(0xc451ed2b, 0x9694, 0x45d3, 0xba, 0xba, 0xed, 0x9f, 0x89, 0x88, 0xa3, 0x89) #define LINUX_EFI_COCO_SECRET_AREA_GUID EFI_GUID(0xadf956ad, 0xe98c, 0x484c, 0xae, 0x11, 0xb5, 0x1c, 0x7d, 0x33, 0x64, 0x47) #define LINUX_EFI_BOOT_MEMMAP_GUID EFI_GUID(0x800f683f, 0xd08b, 0x423a, 0xa2, 0x93, 0x96, 0x5c, 0x3c, 0x6f, 0xe2, 0xb4) +#define LINUX_EFI_UNACCEPTED_MEM_TABLE_GUID EFI_GUID(0xd5d1de3c, 0x105c, 0x44f9, 0x9e, 0xa9, 0xbc, 0xef, 0x98, 0x12, 0x00, 0x31) #define RISCV_EFI_BOOT_PROTOCOL_GUID EFI_GUID(0xccd15fec, 0x6f73, 0x4eec, 0x83, 0x95, 0x3e, 0x69, 0xe4, 0xb9, 0x40, 0xbf) @@ -534,6 +536,14 @@ struct efi_boot_memmap { efi_memory_desc_t map[]; }; +struct efi_unaccepted_memory { + u32 version; + u32 unit_size; + u64 phys_base; + u64 size; + unsigned long bitmap[]; +}; + /* * Architecture independent structure for describing a memory map for the * benefit of efi_memmap_init_early(), and for passing context between -- cgit v1.2.3 From aa6182707a53c5e4df7b3da7ba4faa7e29dc71a0 Mon Sep 17 00:00:00 2001 From: Ruiqi Gong Date: Tue, 6 Jun 2023 10:10:47 +0800 Subject: bpf: Cleanup unused function declaration All usage and the definition of `bpf_prog_free_linfo()` has been removed in commit e16301fbe183 ("bpf: Simplify freeing logic in linfo and jited_linfo"). Clean up its declaration in the header file. Signed-off-by: Ruiqi Gong Signed-off-by: Daniel Borkmann Acked-by: Stanislav Fomichev Link: https://lore.kernel.org/all/20230602030842.279262-1-gongruiqi@huaweicloud.com/ Link: https://lore.kernel.org/bpf/20230606021047.170667-1-gongruiqi@huaweicloud.com --- include/linux/filter.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index bbce89937fde..f69114083ec7 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -874,7 +874,6 @@ void bpf_prog_free(struct bpf_prog *fp); bool bpf_opcode_in_insntable(u8 code); -void bpf_prog_free_linfo(struct bpf_prog *prog); void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, const u32 *insn_to_jit_off); int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog); -- cgit v1.2.3 From 2053bc57f36763febced0b5cd91821698bcf6b3d Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Tue, 6 Jun 2023 17:26:33 +0300 Subject: efi: Add unaccepted memory support efi_config_parse_tables() reserves memory that holds unaccepted memory configuration table so it won't be reused by page allocator. Core-mm requires few helpers to support unaccepted memory: - accept_memory() checks the range of addresses against the bitmap and accept memory if needed. - range_contains_unaccepted_memory() checks if anything within the range requires acceptance. Architectural code has to provide efi_get_unaccepted_table() that returns pointer to the unaccepted memory configuration table. arch_accept_memory() handles arch-specific part of memory acceptance. Signed-off-by: Kirill A. Shutemov Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Ard Biesheuvel Reviewed-by: Tom Lendacky Link: https://lore.kernel.org/r/20230606142637.5171-6-kirill.shutemov@linux.intel.com --- include/linux/efi.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/efi.h b/include/linux/efi.h index 8ffe451a6a2f..67cb72d7a764 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -646,6 +646,7 @@ extern struct efi { unsigned long tpm_final_log; /* TPM2 Final Events Log table */ unsigned long mokvar_table; /* MOK variable config table */ unsigned long coco_secret; /* Confidential computing secret table */ + unsigned long unaccepted; /* Unaccepted memory table */ efi_get_time_t *get_time; efi_set_time_t *set_time; -- cgit v1.2.3 From c0461bd16666351f0de11578b1e02dcdae4db736 Mon Sep 17 00:00:00 2001 From: Dionna Glaze Date: Tue, 6 Jun 2023 09:51:27 -0500 Subject: x86/efi: Safely enable unaccepted memory in UEFI The UEFI v2.9 specification includes a new memory type to be used in environments where the OS must accept memory that is provided from its host. Before the introduction of this memory type, all memory was accepted eagerly in the firmware. In order for the firmware to safely stop accepting memory on the OS's behalf, the OS must affirmatively indicate support to the firmware. This is only a problem for AMD SEV-SNP, since Linux has had support for it since 5.19. The other technology that can make use of unaccepted memory, Intel TDX, does not yet have Linux support, so it can strictly require unaccepted memory support as a dependency of CONFIG_TDX and not require communication with the firmware. Enabling unaccepted memory requires calling a 0-argument enablement protocol before ExitBootServices. This call is only made if the kernel is compiled with UNACCEPTED_MEMORY=y This protocol will be removed after the end of life of the first LTS that includes it, in order to give firmware implementations an expiration date for it. When the protocol is removed, firmware will strictly infer that a SEV-SNP VM is running an OS that supports the unaccepted memory type. At the earliest convenience, when unaccepted memory support is added to Linux, SEV-SNP may take strict dependence in it. After the firmware removes support for the protocol, this should be reverted. [tl: address some checkscript warnings] Signed-off-by: Dionna Glaze Signed-off-by: Tom Lendacky Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/0d5f3d9a20b5cf361945b7ab1263c36586a78a42.1686063086.git.thomas.lendacky@amd.com --- include/linux/efi.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/efi.h b/include/linux/efi.h index 67cb72d7a764..18d83a613635 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -437,6 +437,9 @@ void efi_native_runtime_setup(void); #define DELLEMC_EFI_RCI2_TABLE_GUID EFI_GUID(0x2d9f28a2, 0xa886, 0x456a, 0x97, 0xa8, 0xf1, 0x1e, 0xf2, 0x4f, 0xf4, 0x55) #define AMD_SEV_MEM_ENCRYPT_GUID EFI_GUID(0x0cf29b71, 0x9e51, 0x433a, 0xa3, 0xb7, 0x81, 0xf3, 0xab, 0x16, 0xb8, 0x75) +/* OVMF protocol GUIDs */ +#define OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID EFI_GUID(0xc5a010fe, 0x38a7, 0x4531, 0x8a, 0x4a, 0x05, 0x00, 0xd2, 0xfd, 0x16, 0x49) + typedef struct { efi_guid_t guid; u64 table; -- cgit v1.2.3 From 6d6e57594957ee9131bc3802dfc8657ca6f78fee Mon Sep 17 00:00:00 2001 From: Vijaya Krishna Nivarthi Date: Wed, 17 May 2023 17:48:13 +0530 Subject: soc: qcom: geni-se: Add interfaces geni_se_tx_init_dma() and geni_se_rx_init_dma() The geni_se_xx_dma_prep() interfaces necessarily do DMA mapping before initiating DMA transfers. This is not suitable for spi where framework is expected to handle map/unmap. Expose new interfaces geni_se_xx_init_dma() which do only DMA transfer. Signed-off-by: Vijaya Krishna Nivarthi Reviewed-by: Douglas Anderson Reviewed-by: Konrad Dybcio Link: https://lore.kernel.org/r/1684325894-30252-2-git-send-email-quic_vnivarth@quicinc.com Signed-off-by: Mark Brown --- include/linux/soc/qcom/geni-se.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index c55a0bc8cb0e..821a19135bb6 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -490,9 +490,13 @@ int geni_se_clk_freq_match(struct geni_se *se, unsigned long req_freq, unsigned int *index, unsigned long *res_freq, bool exact); +void geni_se_tx_init_dma(struct geni_se *se, dma_addr_t iova, size_t len); + int geni_se_tx_dma_prep(struct geni_se *se, void *buf, size_t len, dma_addr_t *iova); +void geni_se_rx_init_dma(struct geni_se *se, dma_addr_t iova, size_t len); + int geni_se_rx_dma_prep(struct geni_se *se, void *buf, size_t len, dma_addr_t *iova); -- cgit v1.2.3 From f11f1a92c17385ff4d6e2bc8002d59aed70b98c4 Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Tue, 6 Jun 2023 11:16:25 -0700 Subject: Input: gameport - provide default trigger() and read() Instead of constantly checking pointer(s) for non-NULL-ness provide default implementations of trigger() and read() and instantiate them during pore registration if driver-specific versions were not provided. Link: https://lore.kernel.org/r/ZGvoqP5PAAsJuky4@google.com Signed-off-by: Dmitry Torokhov --- include/linux/gameport.h | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gameport.h b/include/linux/gameport.h index 8c2f00018e89..0a221e768ea4 100644 --- a/include/linux/gameport.h +++ b/include/linux/gameport.h @@ -5,7 +5,6 @@ #ifndef _GAMEPORT_H #define _GAMEPORT_H -#include #include #include #include @@ -165,18 +164,12 @@ void gameport_unregister_driver(struct gameport_driver *drv); static inline void gameport_trigger(struct gameport *gameport) { - if (gameport->trigger) - gameport->trigger(gameport); - else - outb(0xff, gameport->io); + gameport->trigger(gameport); } static inline unsigned char gameport_read(struct gameport *gameport) { - if (gameport->read) - return gameport->read(gameport); - else - return inb(gameport->io); + return gameport->read(gameport); } static inline int gameport_cooked_read(struct gameport *gameport, int *axes, int *buttons) -- cgit v1.2.3 From 28bd137a3c8e105587ba8c55b68ef43b519b270f Mon Sep 17 00:00:00 2001 From: Yanteng Si Date: Wed, 7 Jun 2023 17:21:49 +0800 Subject: ALSA: hda: Add Loongson LS7A HD-Audio support Add the new PCI ID 0x0014 0x7a07 and the new PCI ID 0x0014 0x7a37 Loongson HDA controller. Signed-off-by: Yanteng Si Acked-by: Huacai Chen Link: https://lore.kernel.org/r/993587483b9509796b29a416f257fcfb4b15c6ea.1686128807.git.siyanteng@loongson.cn Signed-off-by: Takashi Iwai --- include/linux/pci_ids.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 95f33dadb2be..c0c4ca8e2851 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -158,6 +158,9 @@ #define PCI_VENDOR_ID_LOONGSON 0x0014 +#define PCI_DEVICE_ID_LOONGSON_HDA 0x7a07 +#define PCI_DEVICE_ID_LOONGSON_HDMI 0x7a37 + #define PCI_VENDOR_ID_TTTECH 0x0357 #define PCI_DEVICE_ID_TTTECH_MC322 0x000a -- cgit v1.2.3 From efa76afdde16f195f8faff0e8dbe58ec18aad70c Mon Sep 17 00:00:00 2001 From: Petr Tesarik Date: Tue, 6 Jun 2023 14:46:25 +0200 Subject: swiotlb: remove unused field "used" from struct io_tlb_mem Commit 20347fca71a3 ("swiotlb: split up the global swiotlb lock") moved the number of used slots to struct io_tlb_area, but it did not remove the field from struct io_tlb_mem. Signed-off-by: Petr Tesarik Signed-off-by: Christoph Hellwig --- include/linux/swiotlb.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 7af2673b47ba..4e52cd5e0bdc 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -76,7 +76,6 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, * @nslabs: The number of IO TLB blocks (in groups of 64) between @start and * @end. For default swiotlb, this is command line adjustable via * setup_io_tlb_npages. - * @used: The number of used IO TLB block. * @list: The free list describing the number of free entries available * from each index. * @orig_addr: The original address corresponding to a mapped entry. @@ -98,7 +97,6 @@ struct io_tlb_mem { phys_addr_t end; void *vaddr; unsigned long nslabs; - unsigned long used; struct dentry *debugfs; bool late_alloc; bool force_bounce; -- cgit v1.2.3 From 3a41db531e5124adaa3a9ab9ca0c724aee85b10c Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 10 Mar 2023 18:45:41 +0200 Subject: pktcdvd: Get rid of custom printing macros We may use traditional dev_*() macros instead of custom ones provided by the driver. Signed-off-by: Andy Shevchenko Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20230310164549.22133-2-andriy.shevchenko@linux.intel.com Signed-off-by: Jens Axboe --- include/linux/pktcdvd.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pktcdvd.h b/include/linux/pktcdvd.h index f9c5ac80d59b..80cb00db42a4 100644 --- a/include/linux/pktcdvd.h +++ b/include/linux/pktcdvd.h @@ -156,7 +156,6 @@ struct pktcdvd_device { struct block_device *bdev; /* dev attached */ dev_t pkt_dev; /* our dev */ - char name[20]; struct packet_settings settings; struct packet_stats stats; int refcnt; /* Open count */ -- cgit v1.2.3 From 222dd185833e464faad2d175c14bca584b6b6dad Mon Sep 17 00:00:00 2001 From: Shay Drory Date: Tue, 6 Jun 2023 00:12:06 -0700 Subject: {net/RDMA}/mlx5: introduce lag_for_each_peer Introduce a generic APIs to iterate over all the devices which are part of the LAG. This API replace mlx5_lag_get_peer_mdev() which retrieve only a single peer device from the lag. Signed-off-by: Shay Drory Reviewed-by: Mark Bloch Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 94d2be5848ae..9a744c48eec2 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1174,7 +1174,13 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, u64 *values, int num_counters, size_t *offsets); -struct mlx5_core_dev *mlx5_lag_get_peer_mdev(struct mlx5_core_dev *dev); +struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int *i); + +#define mlx5_lag_for_each_peer_mdev(dev, peer, i) \ + for (i = 0, peer = mlx5_lag_get_next_peer_mdev(dev, &i); \ + peer; \ + peer = mlx5_lag_get_next_peer_mdev(dev, &i)) + u8 mlx5_lag_get_num_ports(struct mlx5_core_dev *dev); struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev); void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up); -- cgit v1.2.3 From a33682e4e78e249155abbe5e8ee880d5760b5e28 Mon Sep 17 00:00:00 2001 From: Lama Kayal Date: Tue, 6 Jun 2023 00:12:14 -0700 Subject: net/mlx5e: Expose catastrophic steering error counters Add generated_pkt_steering_fail and handled_pkt_steering_fail to devlink heatlth reporter. generated_pkt_steering_fail indicates the number of packets dropped due to illegal steering operation within the vport steering domain. handled_pkt_steering_fail indicates the number of packets dropped due to illegal steering operation, originated by the vport. Also, update devlink reporter functionality documentation with the newly exposed counters. Signed-off-by: Lama Kayal Reviewed-by: Rahul Rameshbabu Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index b89778d0d326..af3a92ad2e6b 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1755,7 +1755,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_328[0x2]; u8 relaxed_ordering_read[0x1]; u8 log_max_pd[0x5]; - u8 reserved_at_330[0x9]; + u8 reserved_at_330[0x7]; + u8 vnic_env_cnt_steering_fail[0x1]; + u8 reserved_at_338[0x1]; u8 q_counter_aggregation[0x1]; u8 q_counter_other_vport[0x1]; u8 log_max_xrcd[0x5]; @@ -3673,7 +3675,13 @@ struct mlx5_ifc_vnic_diagnostic_statistics_bits { u8 eth_wqe_too_small[0x20]; - u8 reserved_at_220[0xdc0]; + u8 reserved_at_220[0xc0]; + + u8 generated_pkt_steering_fail[0x40]; + + u8 handled_pkt_steering_fail[0x40]; + + u8 reserved_at_360[0xc80]; }; struct mlx5_ifc_traffic_counter_bits { -- cgit v1.2.3 From fcea0ccf4fd7f5e0b978d3c18923eea4e431118d Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 6 Jun 2023 10:35:31 +0100 Subject: ACPI: bus: Consolidate all arm specific initialisation into acpi_arm_init() Move all of the ARM-specific initialization into one function namely acpi_arm_init(), so it is not necessary to modify/update bus.c every time a new piece of it is added. Cc: Lorenzo Pieralisi Cc: Rafael J. Wysocki Suggested-by: Rafael J. Wysocki Reviewed-by: Robin Murphy Reviewed-by: Hanjun Guo Link: https://lore.kernel.org/r/CAJZ5v0iBZRZmV_oU+VurqxnVMbFN_ttqrL=cLh0sUH+=u0PYsw@mail.gmail.com Signed-off-by: Sudeep Holla Reviewed-by: Lorenzo Pieralisi Acked-by: Rafael J. Wysocki Reviewed-by: Shaoqin Huang Link: https://lore.kernel.org/r/20230606093531.2746732-1-sudeep.holla@arm.com Signed-off-by: Catalin Marinas --- include/linux/acpi.h | 6 ++++++ include/linux/acpi_agdi.h | 13 ------------- include/linux/acpi_apmt.h | 19 ------------------- include/linux/acpi_iort.h | 2 -- 4 files changed, 6 insertions(+), 34 deletions(-) delete mode 100644 include/linux/acpi_agdi.h delete mode 100644 include/linux/acpi_apmt.h (limited to 'include/linux') diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7b71dd74baeb..5ef126a0a50f 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1507,6 +1507,12 @@ static inline int find_acpi_cpu_topology_hetero_id(unsigned int cpu) } #endif +#ifdef CONFIG_ARM64 +void acpi_arm_init(void); +#else +static inline void acpi_arm_init(void) { } +#endif + #ifdef CONFIG_ACPI_PCC void acpi_init_pcc(void); #else diff --git a/include/linux/acpi_agdi.h b/include/linux/acpi_agdi.h deleted file mode 100644 index f477f0b452fa..000000000000 --- a/include/linux/acpi_agdi.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ - -#ifndef __ACPI_AGDI_H__ -#define __ACPI_AGDI_H__ - -#include - -#ifdef CONFIG_ACPI_AGDI -void __init acpi_agdi_init(void); -#else -static inline void acpi_agdi_init(void) {} -#endif -#endif /* __ACPI_AGDI_H__ */ diff --git a/include/linux/acpi_apmt.h b/include/linux/acpi_apmt.h deleted file mode 100644 index 40bd634d082f..000000000000 --- a/include/linux/acpi_apmt.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * ARM CoreSight PMU driver. - * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. - * - */ - -#ifndef __ACPI_APMT_H__ -#define __ACPI_APMT_H__ - -#include - -#ifdef CONFIG_ACPI_APMT -void acpi_apmt_init(void); -#else -static inline void acpi_apmt_init(void) { } -#endif /* CONFIG_ACPI_APMT */ - -#endif /* __ACPI_APMT_H__ */ diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index b43be0987b19..e4e7bb6fa720 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -27,7 +27,6 @@ int iort_register_domain_token(int trans_id, phys_addr_t base, void iort_deregister_domain_token(int trans_id); struct fwnode_handle *iort_find_domain_token(int trans_id); #ifdef CONFIG_ACPI_IORT -void acpi_iort_init(void); u32 iort_msi_map_id(struct device *dev, u32 id); struct irq_domain *iort_get_device_domain(struct device *dev, u32 id, enum irq_domain_bus_token bus_token); @@ -43,7 +42,6 @@ int iort_iommu_configure_id(struct device *dev, const u32 *id_in); void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head); phys_addr_t acpi_iort_dma_get_max_cpu_address(void); #else -static inline void acpi_iort_init(void) { } static inline u32 iort_msi_map_id(struct device *dev, u32 id) { return id; } static inline struct irq_domain *iort_get_device_domain( -- cgit v1.2.3 From 9ca73f2645706230249c4ec2a2b0cab9515987c8 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 17 Apr 2023 19:04:49 +0000 Subject: mm/slab: add a missing semicolon on SLAB_TYPESAFE_BY_RCU example code An example code snippet for SLAB_TYPESAFE_BY_RCU is missing a semicolon. Add it. Signed-off-by: SeongJae Park Reviewed-by: Paul E. McKenney Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index 6b3e155b70bf..5eeedbfffcd2 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -53,7 +53,7 @@ * stays valid, the trick to using this is relying on an independent * object validation pass. Something like: * - * rcu_read_lock() + * rcu_read_lock(); * again: * obj = lockless_lookup(key); * if (obj) { -- cgit v1.2.3 From 1143c9d9d7602f20ba7bb3cef0d07b10f23cbef7 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 17 Apr 2023 19:04:50 +0000 Subject: mm/slab: break up RCU readers on SLAB_TYPESAFE_BY_RCU example code The SLAB_TYPESAFE_BY_RCU example code snippet uses a single RCU read-side critical section for retries. 'Documentation/RCU/rculist_nulls.rst' has similar example code snippet, and commit da82af04352b ("doc: Update and wordsmith rculist_nulls.rst") broke it up. Apply the change to SLAB_TYPESAFE_BY_RCU example code snippet, too. Signed-off-by: SeongJae Park Reviewed-by: Paul E. McKenney Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slab.h b/include/linux/slab.h index 5eeedbfffcd2..c6bc05765bdb 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -53,16 +53,18 @@ * stays valid, the trick to using this is relying on an independent * object validation pass. Something like: * + * begin: * rcu_read_lock(); - * again: * obj = lockless_lookup(key); * if (obj) { * if (!try_get_ref(obj)) // might fail for free objects - * goto again; + * rcu_read_unlock(); + * goto begin; * * if (obj->key != key) { // not the object we expected * put_ref(obj); - * goto again; + * rcu_read_unlock(); + * goto begin; * } * } * rcu_read_unlock(); -- cgit v1.2.3 From af8de1e307bf1ecbd17d220122832cd093f7a3f8 Mon Sep 17 00:00:00 2001 From: Jiawen Wu Date: Tue, 6 Jun 2023 17:21:05 +0800 Subject: net: pcs: Add 10GBASE-R mode for Synopsys Designware XPCS Add basic support for XPCS using 10GBASE-R interface. This mode will be extended to use interrupt, so set pcs.poll false. And avoid soft reset so that the device using this mode is in the default configuration. Signed-off-by: Jiawen Wu Reviewed-by: Andrew Lunn Reviewed-by: Maciej Fijalkowski Signed-off-by: Paolo Abeni --- include/linux/pcs/pcs-xpcs.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pcs/pcs-xpcs.h b/include/linux/pcs/pcs-xpcs.h index 914e387d5387..ec8175b847cc 100644 --- a/include/linux/pcs/pcs-xpcs.h +++ b/include/linux/pcs/pcs-xpcs.h @@ -18,6 +18,7 @@ #define DW_AN_C37_SGMII 2 #define DW_2500BASEX 3 #define DW_AN_C37_1000BASEX 4 +#define DW_10GBASER 5 struct xpcs_id; -- cgit v1.2.3 From 0d7aeb68700ff87b4d2acafb789408a065225e1e Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 6 Jun 2023 14:08:47 +0100 Subject: Drop the netfs_ prefix from netfs_extract_iter_to_sg() Rename netfs_extract_iter_to_sg() and its auxiliary functions to drop the netfs_ prefix. Signed-off-by: David Howells cc: Jeff Layton cc: Steve French cc: Shyam Prasad N cc: Rohith Surabattula cc: Jens Axboe cc: Herbert Xu cc: "Matthew Wilcox (Oracle)" cc: "David S. Miller" cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: linux-crypto@vger.kernel.org cc: linux-cachefs@redhat.com cc: linux-cifs@vger.kernel.org cc: linux-fsdevel@vger.kernel.org cc: netdev@vger.kernel.org Signed-off-by: Paolo Abeni --- include/linux/netfs.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfs.h b/include/linux/netfs.h index a1f3522daa69..55e201c3a841 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -301,9 +301,9 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, struct iov_iter *new, iov_iter_extraction_t extraction_flags); struct sg_table; -ssize_t netfs_extract_iter_to_sg(struct iov_iter *iter, size_t len, - struct sg_table *sgtable, unsigned int sg_max, - iov_iter_extraction_t extraction_flags); +ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t len, + struct sg_table *sgtable, unsigned int sg_max, + iov_iter_extraction_t extraction_flags); /** * netfs_inode - Get the netfs inode context from the inode -- cgit v1.2.3 From f5f82cd18732d828bcd1ec308c4e8c55012e84b0 Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 6 Jun 2023 14:08:50 +0100 Subject: Move netfs_extract_iter_to_sg() to lib/scatterlist.c Move netfs_extract_iter_to_sg() to lib/scatterlist.c as it's going to be used by more than just network filesystems (AF_ALG, for example). Signed-off-by: David Howells cc: Jeff Layton cc: Steve French cc: Shyam Prasad N cc: Rohith Surabattula cc: Jens Axboe cc: Herbert Xu cc: "David S. Miller" cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Matthew Wilcox cc: linux-crypto@vger.kernel.org cc: linux-cachefs@redhat.com cc: linux-cifs@vger.kernel.org cc: linux-fsdevel@vger.kernel.org cc: netdev@vger.kernel.org Signed-off-by: Paolo Abeni --- include/linux/netfs.h | 4 ---- include/linux/uio.h | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 55e201c3a841..b11a84f6c32b 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -300,10 +300,6 @@ void netfs_stats_show(struct seq_file *); ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, struct iov_iter *new, iov_iter_extraction_t extraction_flags); -struct sg_table; -ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t len, - struct sg_table *sgtable, unsigned int sg_max, - iov_iter_extraction_t extraction_flags); /** * netfs_inode - Get the netfs inode context from the inode diff --git a/include/linux/uio.h b/include/linux/uio.h index 044c1d8c230c..0ccb983cf645 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -433,4 +433,9 @@ static inline bool iov_iter_extract_will_pin(const struct iov_iter *iter) return user_backed_iter(iter); } +struct sg_table; +ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t len, + struct sg_table *sgtable, unsigned int sg_max, + iov_iter_extraction_t extraction_flags); + #endif -- cgit v1.2.3 From fe6ac23777ef70c17aa7333400ede88e364fbd36 Mon Sep 17 00:00:00 2001 From: James Seo Date: Sun, 7 May 2023 08:22:17 -0700 Subject: hwmon: (core) Add missing beep-related standard attributes beep_enable, inX_beep, currX_beep, fanX_beep, and tempX_beep are standard attributes mentioned in the sysfs interface specification but not implemented in the hwmon core. Since these are not deprecated, implement them. Adding beep_mask is not necessary, as it is deprecated and the drivers already using it are manually defining it. Signed-off-by: James Seo Link: https://lore.kernel.org/r/20230507152216.1862653-1-james@equiv.tech Signed-off-by: Guenter Roeck --- include/linux/hwmon.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux') diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 492dd27a5dd8..8cd6a6b33593 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -44,6 +44,7 @@ enum hwmon_chip_attributes { hwmon_chip_in_samples, hwmon_chip_power_samples, hwmon_chip_temp_samples, + hwmon_chip_beep_enable, }; #define HWMON_C_TEMP_RESET_HISTORY BIT(hwmon_chip_temp_reset_history) @@ -58,6 +59,7 @@ enum hwmon_chip_attributes { #define HWMON_C_IN_SAMPLES BIT(hwmon_chip_in_samples) #define HWMON_C_POWER_SAMPLES BIT(hwmon_chip_power_samples) #define HWMON_C_TEMP_SAMPLES BIT(hwmon_chip_temp_samples) +#define HWMON_C_BEEP_ENABLE BIT(hwmon_chip_beep_enable) enum hwmon_temp_attributes { hwmon_temp_enable, @@ -87,6 +89,7 @@ enum hwmon_temp_attributes { hwmon_temp_reset_history, hwmon_temp_rated_min, hwmon_temp_rated_max, + hwmon_temp_beep, }; #define HWMON_T_ENABLE BIT(hwmon_temp_enable) @@ -116,6 +119,7 @@ enum hwmon_temp_attributes { #define HWMON_T_RESET_HISTORY BIT(hwmon_temp_reset_history) #define HWMON_T_RATED_MIN BIT(hwmon_temp_rated_min) #define HWMON_T_RATED_MAX BIT(hwmon_temp_rated_max) +#define HWMON_T_BEEP BIT(hwmon_temp_beep) enum hwmon_in_attributes { hwmon_in_enable, @@ -136,6 +140,7 @@ enum hwmon_in_attributes { hwmon_in_crit_alarm, hwmon_in_rated_min, hwmon_in_rated_max, + hwmon_in_beep, }; #define HWMON_I_ENABLE BIT(hwmon_in_enable) @@ -156,6 +161,7 @@ enum hwmon_in_attributes { #define HWMON_I_CRIT_ALARM BIT(hwmon_in_crit_alarm) #define HWMON_I_RATED_MIN BIT(hwmon_in_rated_min) #define HWMON_I_RATED_MAX BIT(hwmon_in_rated_max) +#define HWMON_I_BEEP BIT(hwmon_in_beep) enum hwmon_curr_attributes { hwmon_curr_enable, @@ -176,6 +182,7 @@ enum hwmon_curr_attributes { hwmon_curr_crit_alarm, hwmon_curr_rated_min, hwmon_curr_rated_max, + hwmon_curr_beep, }; #define HWMON_C_ENABLE BIT(hwmon_curr_enable) @@ -196,6 +203,7 @@ enum hwmon_curr_attributes { #define HWMON_C_CRIT_ALARM BIT(hwmon_curr_crit_alarm) #define HWMON_C_RATED_MIN BIT(hwmon_curr_rated_min) #define HWMON_C_RATED_MAX BIT(hwmon_curr_rated_max) +#define HWMON_C_BEEP BIT(hwmon_curr_beep) enum hwmon_power_attributes { hwmon_power_enable, @@ -312,6 +320,7 @@ enum hwmon_fan_attributes { hwmon_fan_min_alarm, hwmon_fan_max_alarm, hwmon_fan_fault, + hwmon_fan_beep, }; #define HWMON_F_ENABLE BIT(hwmon_fan_enable) @@ -326,6 +335,7 @@ enum hwmon_fan_attributes { #define HWMON_F_MIN_ALARM BIT(hwmon_fan_min_alarm) #define HWMON_F_MAX_ALARM BIT(hwmon_fan_max_alarm) #define HWMON_F_FAULT BIT(hwmon_fan_fault) +#define HWMON_F_BEEP BIT(hwmon_fan_beep) enum hwmon_pwm_attributes { hwmon_pwm_input, -- cgit v1.2.3 From 0eeaf1eb40a34fddd1d568a9b32c3d6669238743 Mon Sep 17 00:00:00 2001 From: Maninder Singh Date: Thu, 8 Jun 2023 09:01:19 +0530 Subject: kallsyms: make kallsyms_show_value() as generic function This change makes function kallsyms_show_value() as generic function without dependency on CONFIG_KALLSYMS. Now module address will be displayed with lsmod and /proc/modules. Earlier: ======= / # insmod test.ko / # lsmod test 12288 0 - Live 0x0000000000000000 (O) // No Module Load address / # With change: ========== / # insmod test.ko / # lsmod test 12288 0 - Live 0xffff800000fc0000 (O) // Module address / # cat /proc/modules test 12288 0 - Live 0xffff800000fc0000 (O) Co-developed-by: Onkarnath Signed-off-by: Onkarnath Signed-off-by: Maninder Singh Reviewed-by: Zhen Lei Signed-off-by: Luis Chamberlain --- include/linux/kallsyms.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h index 1037f4957caa..c3f075e8f60c 100644 --- a/include/linux/kallsyms.h +++ b/include/linux/kallsyms.h @@ -65,6 +65,9 @@ static inline void *dereference_symbol_descriptor(void *ptr) return ptr; } +/* How and when do we show kallsyms values? */ +extern bool kallsyms_show_value(const struct cred *cred); + #ifdef CONFIG_KALLSYMS unsigned long kallsyms_sym_address(int idx); int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), @@ -94,9 +97,6 @@ extern int sprint_backtrace_build_id(char *buffer, unsigned long address); int lookup_symbol_name(unsigned long addr, char *symname); -/* How and when do we show kallsyms values? */ -extern bool kallsyms_show_value(const struct cred *cred); - #else /* !CONFIG_KALLSYMS */ static inline unsigned long kallsyms_lookup_name(const char *name) @@ -154,11 +154,6 @@ static inline int lookup_symbol_name(unsigned long addr, char *symname) return -ERANGE; } -static inline bool kallsyms_show_value(const struct cred *cred) -{ - return false; -} - static inline int kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned long), void *data) { -- cgit v1.2.3 From 861dc0b46432a7086bc6de526aae775b4d615e28 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Sun, 28 May 2023 13:43:46 -0700 Subject: sysctl: move umh sysctl registration to its own file Move the umh sysctl registration to its own file, the array is already there. We do this to remove the clutter out of kernel/sysctl.c to avoid merge conflicts. This also lets the sysctls not be built at all now when CONFIG_SYSCTL is not enabled. This has a small penalty of 23 bytes but soon we'll be removing all the empty entries on sysctl arrays so just do this cleanup now: ./scripts/bloat-o-meter vmlinux.base vmlinux.1 add/remove: 2/0 grow/shrink: 0/1 up/down: 49/-26 (23) Function old new delta init_umh_sysctls - 33 +33 __pfx_init_umh_sysctls - 16 +16 sysctl_init_bases 111 85 -26 Total: Before=21256914, After=21256937, chg +0.00% Acked-by: Jarkko Sakkinen Signed-off-by: Luis Chamberlain --- include/linux/umh.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/umh.h b/include/linux/umh.h index 5d1f6129b847..daa6a7048c11 100644 --- a/include/linux/umh.h +++ b/include/linux/umh.h @@ -42,8 +42,6 @@ call_usermodehelper_setup(const char *path, char **argv, char **envp, extern int call_usermodehelper_exec(struct subprocess_info *info, int wait); -extern struct ctl_table usermodehelper_table[]; - enum umh_disable_depth { UMH_ENABLED = 0, UMH_FREEZING, -- cgit v1.2.3 From 28898e260a34e840f86ca80bf0c7657d76ad3f80 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Sun, 28 May 2023 13:54:20 -0700 Subject: sysctl: move security keys sysctl registration to its own file The security keys sysctls are already declared on its own file, just move the sysctl registration to its own file to help avoid merge conflicts on sysctls.c, and help with clearing up sysctl.c further. This creates a small penalty of 23 bytes: ./scripts/bloat-o-meter vmlinux.1 vmlinux.2 add/remove: 2/0 grow/shrink: 0/1 up/down: 49/-26 (23) Function old new delta init_security_keys_sysctls - 33 +33 __pfx_init_security_keys_sysctls - 16 +16 sysctl_init_bases 85 59 -26 Total: Before=21256937, After=21256960, chg +0.00% But soon we'll be saving tons of bytes anyway, as we modify the sysctl registrations to use ARRAY_SIZE and so we get rid of all the empty array elements so let's just clean this up now. Reviewed-by: Paul Moore Acked-by: Jarkko Sakkinen Acked-by: David Howells Signed-off-by: Luis Chamberlain --- include/linux/key.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/key.h b/include/linux/key.h index 8dc7f7c3088b..938d7ecfb495 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -490,9 +490,6 @@ do { \ rcu_assign_pointer((KEY)->payload.rcu_data0, (PAYLOAD)); \ } while (0) -#ifdef CONFIG_SYSCTL -extern struct ctl_table key_sysctls[]; -#endif /* * the userspace interface */ -- cgit v1.2.3 From 33b70fbc4f815f0acb327fa506c988ef25cd943d Mon Sep 17 00:00:00 2001 From: Stephen Boyd Date: Fri, 5 May 2023 13:25:06 +0200 Subject: clk: Introduce clk_hw_determine_rate_no_reparent() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some clock drivers do not want to allow any reparenting on a given clock, but usually do so by not providing any determine_rate implementation. Whenever we call clk_round_rate() or clk_set_rate(), this leads to clk_core_can_round() returning false and thus the rest of the function either forwarding the rate request to its current parent if CLK_SET_RATE_PARENT is set, or just returning the current clock rate. This behaviour happens implicitly, and as we move forward to making a determine_rate implementation required for muxes, we need some way to explicitly opt-in for that behaviour. Fortunately, this is exactly what the clk_core_determine_rate_no_reparent() function is doing, so we can simply make it available to drivers. Cc: Abel Vesa Cc: Alessandro Zummo Cc: Alexandre Belloni Cc: Alexandre Torgue Cc: "Andreas Färber" Cc: AngeloGioacchino Del Regno Cc: Baolin Wang Cc: Charles Keepax Cc: Chen-Yu Tsai Cc: Chen-Yu Tsai Cc: Chunyan Zhang Cc: Claudiu Beznea Cc: Daniel Vetter Cc: David Airlie Cc: David Lechner Cc: Dinh Nguyen Cc: Fabio Estevam Cc: Geert Uytterhoeven Cc: Jaroslav Kysela Cc: Jernej Skrabec Cc: Jonathan Hunter Cc: Kishon Vijay Abraham I Cc: Liam Girdwood Cc: Linus Walleij Cc: Luca Ceresoli Cc: Manivannan Sadhasivam Cc: Mark Brown Cc: Markus Schneider-Pargmann Cc: Max Filippov Cc: Maxime Coquelin Cc: Mikko Perttunen Cc: Miles Chen Cc: Nicolas Ferre Cc: Orson Zhai Cc: Paul Cercueil Cc: Peng Fan Cc: Peter De Schrijver Cc: Prashant Gaikwad Cc: Richard Fitzgerald Cc: Samuel Holland Cc: Sascha Hauer Cc: Sekhar Nori Cc: Shawn Guo Cc: Takashi Iwai Cc: Thierry Reding Cc: Ulf Hansson Cc: Vinod Koul Cc: dri-devel@lists.freedesktop.org Cc: linux-actions@lists.infradead.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-mips@vger.kernel.org Cc: linux-phy@lists.infradead.org Cc: linux-renesas-soc@vger.kernel.org Cc: linux-rtc@vger.kernel.org Cc: linux-stm32@st-md-mailman.stormreply.com Cc: linux-sunxi@lists.linux.dev Cc: linux-tegra@vger.kernel.org Cc: NXP Linux Team Cc: patches@opensource.cirrus.com Cc: Pengutronix Kernel Team Signed-off-by: Stephen Boyd Signed-off-by: Maxime Ripard Link: https://lore.kernel.org/r/20221018-clk-range-checks-fixes-v4-4-971d5077e7d2@cerno.tech | Reported-by: kernel test robot : --- include/linux/clk-provider.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 28ff6f1a6ada..f8f220fb5dab 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -1333,6 +1333,8 @@ int __clk_mux_determine_rate_closest(struct clk_hw *hw, int clk_mux_determine_rate_flags(struct clk_hw *hw, struct clk_rate_request *req, unsigned long flags); +int clk_hw_determine_rate_no_reparent(struct clk_hw *hw, + struct clk_rate_request *req); void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent); void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate, unsigned long *max_rate); -- cgit v1.2.3 From b3b984dc0ba60ce4787f661d8fc7f44e8953b51d Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 7 Jun 2023 12:58:24 +0100 Subject: net: pcs: lynx: remove lynx_get_mdio_device() lynx_get_mdio_device() is no longer necessary, let's remove it so the lynx PCS code is always managing the lifetime of the mdiodev. Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- include/linux/pcs-lynx.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pcs-lynx.h b/include/linux/pcs-lynx.h index 885b59d10581..25f68a096bfe 100644 --- a/include/linux/pcs-lynx.h +++ b/include/linux/pcs-lynx.h @@ -9,8 +9,6 @@ #include #include -struct mdio_device *lynx_get_mdio_device(struct phylink_pcs *pcs); - struct phylink_pcs *lynx_pcs_create(struct mdio_device *mdio); struct phylink_pcs *lynx_pcs_create_mdiodev(struct mii_bus *bus, int addr); -- cgit v1.2.3 From 6e1a12821d34ee37b1196872eccc7dc9b5218a87 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 7 Jun 2023 12:58:29 +0100 Subject: net: pcs: lynx: add lynx_pcs_create_fwnode() Add a helper to create a lynx PCS from a fwnode handle. Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- include/linux/pcs-lynx.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pcs-lynx.h b/include/linux/pcs-lynx.h index 25f68a096bfe..123e813df771 100644 --- a/include/linux/pcs-lynx.h +++ b/include/linux/pcs-lynx.h @@ -11,6 +11,7 @@ struct phylink_pcs *lynx_pcs_create(struct mdio_device *mdio); struct phylink_pcs *lynx_pcs_create_mdiodev(struct mii_bus *bus, int addr); +struct phylink_pcs *lynx_pcs_create_fwnode(struct fwnode_handle *node); void lynx_pcs_destroy(struct phylink_pcs *pcs); -- cgit v1.2.3 From 84e476b876d9164af4b965c97eee90fa88204b63 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Wed, 7 Jun 2023 12:58:44 +0100 Subject: net: pcs: lynx: make lynx_pcs_create() static We no longer need to export lynx_pcs_create() for drivers to use as we now have all the functionality we need in the two new creation helpers. Remove the export and prototype, and make it static. Signed-off-by: Russell King (Oracle) Signed-off-by: Jakub Kicinski --- include/linux/pcs-lynx.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pcs-lynx.h b/include/linux/pcs-lynx.h index 123e813df771..7958cccd16f2 100644 --- a/include/linux/pcs-lynx.h +++ b/include/linux/pcs-lynx.h @@ -9,7 +9,6 @@ #include #include -struct phylink_pcs *lynx_pcs_create(struct mdio_device *mdio); struct phylink_pcs *lynx_pcs_create_mdiodev(struct mii_bus *bus, int addr); struct phylink_pcs *lynx_pcs_create_fwnode(struct fwnode_handle *node); -- cgit v1.2.3 From 4fe38acdac8a71f7cccf347a2e9902bc818ecef7 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 7 Jun 2023 19:19:07 +0100 Subject: net: Block MSG_SENDPAGE_* from being passed to sendmsg() by userspace It is necessary to allow MSG_SENDPAGE_* to be passed into ->sendmsg() to allow sendmsg(MSG_SPLICE_PAGES) to replace ->sendpage(). Unblocking them in the network protocol, however, allows these flags to be passed in by userspace too[1]. Fix this by marking MSG_SENDPAGE_NOPOLICY, MSG_SENDPAGE_NOTLAST and MSG_SENDPAGE_DECRYPTED as internal flags, which causes sendmsg() to object if they are passed to sendmsg() by userspace. Network protocol ->sendmsg() implementations can then allow them through. Note that it should be possible to remove MSG_SENDPAGE_NOTLAST once sendpage is removed as a whole slew of pages will be passed in in one go by splice through sendmsg, with MSG_MORE being set if it has more data waiting in the pipe. Signed-off-by: David Howells cc: Chuck Lever cc: Boris Pismenny cc: John Fastabend cc: Jens Axboe cc: Matthew Wilcox Link: https://lore.kernel.org/r/20230526181338.03a99016@kernel.org/ [1] Signed-off-by: Jakub Kicinski --- include/linux/socket.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index bd1cc3238851..3fd3436bc09f 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -339,7 +339,9 @@ struct ucred { #endif /* Flags to be cleared on entry by sendmsg and sendmmsg syscalls */ -#define MSG_INTERNAL_SENDMSG_FLAGS (MSG_SPLICE_PAGES) +#define MSG_INTERNAL_SENDMSG_FLAGS \ + (MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_NOTLAST | \ + MSG_SENDPAGE_DECRYPTED) /* Setsockoptions(2) level. Thanks to BSD these must match IPPROTO_xxx */ #define SOL_IP 0 -- cgit v1.2.3 From 2dc334f1a63a8839b88483a3e73c0f27c9c1791c Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 7 Jun 2023 19:19:09 +0100 Subject: splice, net: Use sendmsg(MSG_SPLICE_PAGES) rather than ->sendpage() Replace generic_splice_sendpage() + splice_from_pipe + pipe_to_sendpage() with a net-specific handler, splice_to_socket(), that calls sendmsg() with MSG_SPLICE_PAGES set instead of calling ->sendpage(). MSG_MORE is used to indicate if the sendmsg() is expected to be followed with more data. This allows multiple pipe-buffer pages to be passed in a single call in a BVEC iterator, allowing the processing to be pushed down to a loop in the protocol driver. This helps pave the way for passing multipage folios down too. Protocols that haven't been converted to handle MSG_SPLICE_PAGES yet should just ignore it and do a normal sendmsg() for now - although that may be a bit slower as it may copy everything. Signed-off-by: David Howells Reviewed-by: Jakub Kicinski cc: Jens Axboe cc: Matthew Wilcox Signed-off-by: Jakub Kicinski --- include/linux/fs.h | 2 -- include/linux/splice.h | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 133f0640fb24..df92f4b3d122 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2759,8 +2759,6 @@ extern ssize_t generic_file_splice_read(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); extern ssize_t iter_file_splice_write(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); -extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, - struct file *out, loff_t *, size_t len, unsigned int flags); extern long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, loff_t *opos, size_t len, unsigned int flags); diff --git a/include/linux/splice.h b/include/linux/splice.h index a55179fd60fc..991ae318b6eb 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -84,6 +84,8 @@ extern long do_splice(struct file *in, loff_t *off_in, extern long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags); +extern ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags); /* * for dynamic pipe sizing -- cgit v1.2.3 From 2bfc66850952b6921b2033b09729ec59eabbc81d Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 7 Jun 2023 19:19:10 +0100 Subject: splice, net: Add a splice_eof op to file-ops and socket-ops Add an optional method, ->splice_eof(), to allow splice to indicate the premature termination of a splice to struct file_operations and struct proto_ops. This is called if sendfile() or splice() encounters all of the following conditions inside splice_direct_to_actor(): (1) the user did not set SPLICE_F_MORE (splice only), and (2) an EOF condition occurred (->splice_read() returned 0), and (3) we haven't read enough to fulfill the request (ie. len > 0 still), and (4) we have already spliced at least one byte. A further patch will modify the behaviour of SPLICE_F_MORE to always be passed to the actor if either the user set it or we haven't yet read sufficient data to fulfill the request. Suggested-by: Linus Torvalds Link: https://lore.kernel.org/r/CAHk-=wh=V579PDYvkpnTobCLGczbgxpMgGmmhqiTyE34Cpi5Gg@mail.gmail.com/ Signed-off-by: David Howells Reviewed-by: Jakub Kicinski cc: Jens Axboe cc: Christoph Hellwig cc: Al Viro cc: Matthew Wilcox cc: Jan Kara cc: Jeff Layton cc: David Hildenbrand cc: Christian Brauner cc: Chuck Lever cc: Boris Pismenny cc: John Fastabend cc: linux-mm@kvack.org Signed-off-by: Jakub Kicinski --- include/linux/fs.h | 1 + include/linux/net.h | 1 + include/linux/splice.h | 1 + 3 files changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index df92f4b3d122..de2cb1132f07 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1796,6 +1796,7 @@ struct file_operations { int (*flock) (struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); + void (*splice_eof)(struct file *file); int (*setlease)(struct file *, long, struct file_lock **, void **); long (*fallocate)(struct file *file, int mode, loff_t offset, loff_t len); diff --git a/include/linux/net.h b/include/linux/net.h index b73ad8e3c212..8defc8f1d82e 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -210,6 +210,7 @@ struct proto_ops { int offset, size_t size, int flags); ssize_t (*splice_read)(struct socket *sock, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); + void (*splice_eof)(struct socket *sock); int (*set_peek_off)(struct sock *sk, int val); int (*peek_len)(struct socket *sock); diff --git a/include/linux/splice.h b/include/linux/splice.h index 991ae318b6eb..4fab18a6e371 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -38,6 +38,7 @@ struct splice_desc { struct file *file; /* file to read/write */ void *data; /* cookie */ } u; + void (*splice_eof)(struct splice_desc *sd); /* Unexpected EOF handler */ loff_t pos; /* file position */ loff_t *opos; /* sendfile: output position */ size_t num_spliced; /* number of bytes already spliced */ -- cgit v1.2.3 From 57fd7d59b1c7d6f6a1c34863a2bc4ff1f6c92d40 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Wed, 7 Jun 2023 11:34:52 -0700 Subject: net: phy: broadcom: Rename LED registers These registers are common to most PHYs and are not specific to the BCM5482, renamed the constants accordingly, no functional change. Signed-off-by: Florian Fainelli Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/brcmphy.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 251833ab271f..ab21b8a1b2c8 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -206,11 +206,11 @@ #define BCM_NO_ANEG_APD_EN 0x0060 /* bits 5 & 6 */ #define BCM_APD_SINGLELP_EN 0x0100 /* Bit 8 */ -#define BCM5482_SHD_LEDS1 0x0d /* 01101: LED Selector 1 */ +#define BCM54XX_SHD_LEDS1 0x0d /* 01101: LED Selector 1 */ /* LED3 / ~LINKSPD[2] selector */ -#define BCM5482_SHD_LEDS1_LED3(src) ((src & 0xf) << 4) +#define BCM54XX_SHD_LEDS1_LED3(src) ((src & 0xf) << 4) /* LED1 / ~LINKSPD[1] selector */ -#define BCM5482_SHD_LEDS1_LED1(src) ((src & 0xf) << 0) +#define BCM54XX_SHD_LEDS1_LED1(src) ((src & 0xf) << 0) #define BCM54XX_SHD_RGMII_MODE 0x0b /* 01011: RGMII Mode Selector */ #define BCM5482_SHD_SSD 0x14 /* 10100: Secondary SerDes control */ #define BCM5482_SHD_SSD_LEDM 0x0008 /* SSD LED Mode enable */ -- cgit v1.2.3 From bd5736e146e35f9eabe8c1bfc0ab00979ae62930 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Wed, 7 Jun 2023 11:34:53 -0700 Subject: net: phy: broadcom: Add support for setting LED brightness Broadcom PHYs have two LEDs selector registers which allow us to control the LED assignment, including how to turn them on/off. Signed-off-by: Florian Fainelli Reviewed-by: Simon Horman Signed-off-by: David S. Miller --- include/linux/brcmphy.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index ab21b8a1b2c8..5d732f48f787 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -161,6 +161,7 @@ #define BCM_LED_SRC_OPENSHORT 0xb #define BCM_LED_SRC_OFF 0xe /* Tied high */ #define BCM_LED_SRC_ON 0xf /* Tied low */ +#define BCM_LED_SRC_MASK GENMASK(3, 0) /* * Broadcom Multicolor LED configurations (expansion register 4) @@ -208,9 +209,11 @@ #define BCM54XX_SHD_LEDS1 0x0d /* 01101: LED Selector 1 */ /* LED3 / ~LINKSPD[2] selector */ +#define BCM54XX_SHD_LEDS_SHIFT(led) (4 * (led)) #define BCM54XX_SHD_LEDS1_LED3(src) ((src & 0xf) << 4) /* LED1 / ~LINKSPD[1] selector */ #define BCM54XX_SHD_LEDS1_LED1(src) ((src & 0xf) << 0) +#define BCM54XX_SHD_LEDS2 0x0e /* 01110: LED Selector 2 */ #define BCM54XX_SHD_RGMII_MODE 0x0b /* 01011: RGMII Mode Selector */ #define BCM5482_SHD_SSD 0x14 /* 10100: Secondary SerDes control */ #define BCM5482_SHD_SSD_LEDM 0x0008 /* SSD LED Mode enable */ -- cgit v1.2.3 From 21d4631eedb136f101d2633b72cf42c20db79202 Mon Sep 17 00:00:00 2001 From: "Jason-JH.Lin" Date: Mon, 6 Mar 2023 16:06:59 +0800 Subject: soc: mediatek: remove DDP_DOMPONENT_DITHER from enum After mmsys and drm change DITHER enum to DDP_COMPONENT_DITHER0, mmsys header can remove the useless DDP_COMPONENT_DITHER enum. Signed-off-by: Jason-JH.Lin Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Rex-BC Chen Acked-by: Matthias Brugger Link: https://lore.kernel.org/r/20230306080659.15261-3-jason-jh.lin@mediatek.com Signed-off-by: Matthias Brugger --- include/linux/soc/mediatek/mtk-mmsys.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/soc/mediatek/mtk-mmsys.h b/include/linux/soc/mediatek/mtk-mmsys.h index 37544ea6286d..2475ef914746 100644 --- a/include/linux/soc/mediatek/mtk-mmsys.h +++ b/include/linux/soc/mediatek/mtk-mmsys.h @@ -27,8 +27,7 @@ enum mtk_ddp_comp_id { DDP_COMPONENT_CCORR, DDP_COMPONENT_COLOR0, DDP_COMPONENT_COLOR1, - DDP_COMPONENT_DITHER, - DDP_COMPONENT_DITHER0 = DDP_COMPONENT_DITHER, + DDP_COMPONENT_DITHER0, DDP_COMPONENT_DITHER1, DDP_COMPONENT_DP_INTF0, DDP_COMPONENT_DP_INTF1, -- cgit v1.2.3 From 9d65b1b4bcf3918164e17365eec169875eef8ee3 Mon Sep 17 00:00:00 2001 From: Shiwu Zhang Date: Tue, 23 May 2023 12:02:32 +0800 Subject: drm/amdgpu: add the accelerator PCIe class Add the accelerator PCIe class and match the class in amdgpu for 0x1002 devices of that class. From PCI spec: "PCI Code and ID Assignment, r1.9, sec 1, 1.19" Signed-off-by: Shiwu Zhang Acked-by: Lijo Lazar Acked-by: Bjorn Helgaas # pci_ids.h Signed-off-by: Alex Deucher --- include/linux/pci_ids.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 45c3d62e616d..0fbfbda3dc26 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -151,6 +151,9 @@ #define PCI_CLASS_SP_DPIO 0x1100 #define PCI_CLASS_SP_OTHER 0x1180 +#define PCI_BASE_CLASS_ACCELERATOR 0x12 +#define PCI_CLASS_ACCELERATOR_PROCESSING 0x1200 + #define PCI_CLASS_OTHERS 0xff /* Vendors and devices. Sort key: vendor first, device next. */ -- cgit v1.2.3 From 35822fdae3bf509532b0954088070f17de51ff15 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Fri, 21 Apr 2023 17:40:19 +0000 Subject: memcg: remove mem_cgroup_flush_stats_atomic() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous patches removed all callers of mem_cgroup_flush_stats_atomic(). Remove the function and simplify the code. Link: https://lkml.kernel.org/r/20230421174020.2994750-5-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Shakeel Butt Cc: Alexander Viro Cc: Christian Brauner Cc: Jan Kara Cc: Jens Axboe Cc: Johannes Weiner Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Cc: Tejun Heo Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 222d7370134c..00a88cf947e1 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1038,7 +1038,6 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, } void mem_cgroup_flush_stats(void); -void mem_cgroup_flush_stats_atomic(void); void mem_cgroup_flush_stats_ratelimited(void); void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, @@ -1537,10 +1536,6 @@ static inline void mem_cgroup_flush_stats(void) { } -static inline void mem_cgroup_flush_stats_atomic(void) -{ -} - static inline void mem_cgroup_flush_stats_ratelimited(void) { } -- cgit v1.2.3 From 0a2dc6ac33297f8a1a65f81b633a1ea753f19f69 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Fri, 21 Apr 2023 17:40:20 +0000 Subject: cgroup: remove cgroup_rstat_flush_atomic() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previous patches removed the only caller of cgroup_rstat_flush_atomic(). Remove the function and simplify the code. Link: https://lkml.kernel.org/r/20230421174020.2994750-6-yosryahmed@google.com Signed-off-by: Yosry Ahmed Acked-by: Shakeel Butt Acked-by: Tejun Heo Cc: Alexander Viro Cc: Christian Brauner Cc: Jan Kara Cc: Jens Axboe Cc: Johannes Weiner Cc: Michal Hocko Cc: Michal Koutný Cc: Muchun Song Cc: Roman Gushchin Signed-off-by: Andrew Morton --- include/linux/cgroup.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 885f5395fcd0..567c547cf371 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -692,7 +692,6 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) */ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu); void cgroup_rstat_flush(struct cgroup *cgrp); -void cgroup_rstat_flush_atomic(struct cgroup *cgrp); void cgroup_rstat_flush_hold(struct cgroup *cgrp); void cgroup_rstat_flush_release(void); -- cgit v1.2.3 From ffcb5f5262b756a598eefb11e340bbd027cde037 Mon Sep 17 00:00:00 2001 From: Nhat Pham Date: Tue, 2 May 2023 18:36:06 -0700 Subject: workingset: refactor LRU refault to expose refault recency check Patch series "cachestat: a new syscall for page cache state of files", v13. There is currently no good way to query the page cache statistics of large files and directory trees. There is mincore(), but it scales poorly: the kernel writes out a lot of bitmap data that userspace has to aggregate, when the user really does not care about per-page information in that case. The user also needs to mmap and unmap each file as it goes along, which can be quite slow as well. Some use cases where this information could come in handy: * Allowing database to decide whether to perform an index scan or direct table queries based on the in-memory cache state of the index. * Visibility into the writeback algorithm, for performance issues diagnostic. * Workload-aware writeback pacing: estimating IO fulfilled by page cache (and IO to be done) within a range of a file, allowing for more frequent syncing when and where there is IO capacity, and batching when there is not. * Computing memory usage of large files/directory trees, analogous to the du tool for disk usage. More information about these use cases could be found in this thread: https://lore.kernel.org/lkml/20230315170934.GA97793@cmpxchg.org/ This series of patches introduces a new system call, cachestat, that summarizes the page cache statistics (number of cached pages, dirty pages, pages marked for writeback, evicted pages etc.) of a file, in a specified range of bytes. It also include a selftest suite that tests some typical usage. Currently, the syscall is only wired in for x86 architecture. This interface is inspired by past discussion and concerns with fincore, which has a similar design (and as a result, issues) as mincore. Relevant links: https://lkml.indiana.edu/hypermail/linux/kernel/1302.1/04207.html https://lkml.indiana.edu/hypermail/linux/kernel/1302.1/04209.html I have also developed a small tool that computes the memory usage of files and directories, analogous to the du utility. User can choose between mincore or cachestat (with cachestat exporting more information than mincore). To compare the performance of these two options, I benchmarked the tool on the root directory of a Meta's server machine, each for five runs: Using cachestat real -- Median: 33.377s, Average: 33.475s, Standard Deviation: 0.3602 user -- Median: 4.08s, Average: 4.1078s, Standard Deviation: 0.0742 sys -- Median: 28.823s, Average: 28.8866s, Standard Deviation: 0.2689 Using mincore: real -- Median: 102.352s, Average: 102.3442s, Standard Deviation: 0.2059 user -- Median: 10.149s, Average: 10.1482s, Standard Deviation: 0.0162 sys -- Median: 91.186s, Average: 91.2084s, Standard Deviation: 0.2046 I also ran both syscalls on a 2TB sparse file: Using cachestat: real 0m0.009s user 0m0.000s sys 0m0.009s Using mincore: real 0m37.510s user 0m2.934s sys 0m34.558s Very large files like this are the pathological case for mincore. In fact, to compute the stats for a single 2TB file, mincore takes as long as cachestat takes to compute the stats for the entire tree! This could easily happen inadvertently when we run it on subdirectories. Mincore is clearly not suitable for a general-purpose command line tool. Regarding security concerns, cachestat() should not pose any additional issues. The caller already has read permission to the file itself (since they need an fd to that file to call cachestat). This means that the caller can access the underlying data in its entirety, which is a much greater source of information (and as a result, a much greater security risk) than the cache status itself. The latest API change (in v13 of the patch series) is suggested by Jens Axboe. It allows for 64-bit length argument, even on 32-bit architecture (which is previously not possible due to the limit on the number of syscall arguments). Furthermore, it eliminates the need for compatibility handling - every user can use the same ABI. This patch (of 4): In preparation for computing recently evicted pages in cachestat, refactor workingset_refault and lru_gen_refault to expose a helper function that would test if an evicted page is recently evicted. [penguin-kernel@I-love.SAKURA.ne.jp: add missing rcu_read_unlock() in lru_gen_refault()] Link: https://lkml.kernel.org/r/610781bc-cf11-fc89-a46f-87cb8235d439@I-love.SAKURA.ne.jp Link: https://lkml.kernel.org/r/20230503013608.2431726-1-nphamcs@gmail.com Link: https://lkml.kernel.org/r/20230503013608.2431726-2-nphamcs@gmail.com Signed-off-by: Nhat Pham Signed-off-by: Tetsuo Handa Acked-by: Johannes Weiner Cc: Brian Foster Cc: Johannes Weiner Cc: Matthew Wilcox (Oracle) Cc: Michael Kerrisk Cc: Tetsuo Handa Signed-off-by: Andrew Morton --- include/linux/swap.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 3c69cb653cb9..b2128df5edea 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -368,6 +368,7 @@ static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry) } /* linux/mm/workingset.c */ +bool workingset_test_recent(void *shadow, bool file, bool *workingset); void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); void workingset_refault(struct folio *folio, void *shadow); -- cgit v1.2.3 From cf264e1329fb0307e044f7675849f9f38b44c11a Mon Sep 17 00:00:00 2001 From: Nhat Pham Date: Tue, 2 May 2023 18:36:07 -0700 Subject: cachestat: implement cachestat syscall There is currently no good way to query the page cache state of large file sets and directory trees. There is mincore(), but it scales poorly: the kernel writes out a lot of bitmap data that userspace has to aggregate, when the user really doesn not care about per-page information in that case. The user also needs to mmap and unmap each file as it goes along, which can be quite slow as well. Some use cases where this information could come in handy: * Allowing database to decide whether to perform an index scan or direct table queries based on the in-memory cache state of the index. * Visibility into the writeback algorithm, for performance issues diagnostic. * Workload-aware writeback pacing: estimating IO fulfilled by page cache (and IO to be done) within a range of a file, allowing for more frequent syncing when and where there is IO capacity, and batching when there is not. * Computing memory usage of large files/directory trees, analogous to the du tool for disk usage. More information about these use cases could be found in the following thread: https://lore.kernel.org/lkml/20230315170934.GA97793@cmpxchg.org/ This patch implements a new syscall that queries cache state of a file and summarizes the number of cached pages, number of dirty pages, number of pages marked for writeback, number of (recently) evicted pages, etc. in a given range. Currently, the syscall is only wired in for x86 architecture. NAME cachestat - query the page cache statistics of a file. SYNOPSIS #include struct cachestat_range { __u64 off; __u64 len; }; struct cachestat { __u64 nr_cache; __u64 nr_dirty; __u64 nr_writeback; __u64 nr_evicted; __u64 nr_recently_evicted; }; int cachestat(unsigned int fd, struct cachestat_range *cstat_range, struct cachestat *cstat, unsigned int flags); DESCRIPTION cachestat() queries the number of cached pages, number of dirty pages, number of pages marked for writeback, number of evicted pages, number of recently evicted pages, in the bytes range given by `off` and `len`. An evicted page is a page that is previously in the page cache but has been evicted since. A page is recently evicted if its last eviction was recent enough that its reentry to the cache would indicate that it is actively being used by the system, and that there is memory pressure on the system. These values are returned in a cachestat struct, whose address is given by the `cstat` argument. The `off` and `len` arguments must be non-negative integers. If `len` > 0, the queried range is [`off`, `off` + `len`]. If `len` == 0, we will query in the range from `off` to the end of the file. The `flags` argument is unused for now, but is included for future extensibility. User should pass 0 (i.e no flag specified). Currently, hugetlbfs is not supported. Because the status of a page can change after cachestat() checks it but before it returns to the application, the returned values may contain stale information. RETURN VALUE On success, cachestat returns 0. On error, -1 is returned, and errno is set to indicate the error. ERRORS EFAULT cstat or cstat_args points to an invalid address. EINVAL invalid flags. EBADF invalid file descriptor. EOPNOTSUPP file descriptor is of a hugetlbfs file [nphamcs@gmail.com: replace rounddown logic with the existing helper] Link: https://lkml.kernel.org/r/20230504022044.3675469-1-nphamcs@gmail.com Link: https://lkml.kernel.org/r/20230503013608.2431726-3-nphamcs@gmail.com Signed-off-by: Nhat Pham Acked-by: Johannes Weiner Cc: Brian Foster Cc: Matthew Wilcox (Oracle) Cc: Michael Kerrisk Signed-off-by: Andrew Morton --- include/linux/syscalls.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 33a0ee3bcb2e..6648c07c4381 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -72,6 +72,8 @@ struct open_how; struct mount_attr; struct landlock_ruleset_attr; enum landlock_rule_type; +struct cachestat_range; +struct cachestat; #include #include @@ -1058,6 +1060,9 @@ asmlinkage long sys_memfd_secret(unsigned int flags); asmlinkage long sys_set_mempolicy_home_node(unsigned long start, unsigned long len, unsigned long home_node, unsigned long flags); +asmlinkage long sys_cachestat(unsigned int fd, + struct cachestat_range __user *cstat_range, + struct cachestat __user *cstat, unsigned int flags); /* * Architecture-specific system calls -- cgit v1.2.3 From c963901197188189e85b4d768a059fe1bbc2a502 Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Wed, 10 May 2023 14:47:16 +0200 Subject: filemap: remove page_endio() page_endio() is not used anymore. Remove it. Link: https://lkml.kernel.org/r/20230510124716.73655-1-p.raghav@samsung.com Signed-off-by: Pankaj Raghav Reviewed-by: Christoph Hellwig Acked-by: Matthew Wilcox (Oracle) Cc: Luis Chamberlain Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index a56308a9d1a4..c1ae5ebc375f 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1078,8 +1078,6 @@ int filemap_migrate_folio(struct address_space *mapping, struct folio *dst, #else #define filemap_migrate_folio NULL #endif -void page_endio(struct page *page, bool is_write, int err); - void folio_end_private_2(struct folio *folio); void folio_wait_private_2(struct folio *folio); int folio_wait_private_2_killable(struct folio *folio); -- cgit v1.2.3 From bb6e04a173f06e51819a4bb512e127dfbc50dcfa Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 9 May 2023 16:57:21 +0200 Subject: kasan: use internal prototypes matching gcc-13 builtins gcc-13 warns about function definitions for builtin interfaces that have a different prototype, e.g.: In file included from kasan_test.c:31: kasan.h:574:6: error: conflicting types for built-in function '__asan_register_globals'; expected 'void(void *, long int)' [-Werror=builtin-declaration-mismatch] 574 | void __asan_register_globals(struct kasan_global *globals, size_t size); kasan.h:577:6: error: conflicting types for built-in function '__asan_alloca_poison'; expected 'void(void *, long int)' [-Werror=builtin-declaration-mismatch] 577 | void __asan_alloca_poison(unsigned long addr, size_t size); kasan.h:580:6: error: conflicting types for built-in function '__asan_load1'; expected 'void(void *)' [-Werror=builtin-declaration-mismatch] 580 | void __asan_load1(unsigned long addr); kasan.h:581:6: error: conflicting types for built-in function '__asan_store1'; expected 'void(void *)' [-Werror=builtin-declaration-mismatch] 581 | void __asan_store1(unsigned long addr); kasan.h:643:6: error: conflicting types for built-in function '__hwasan_tag_memory'; expected 'void(void *, unsigned char, long int)' [-Werror=builtin-declaration-mismatch] 643 | void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size); The two problems are: - Addresses are passes as 'unsigned long' in the kernel, but gcc-13 expects a 'void *'. - sizes meant to use a signed ssize_t rather than size_t. Change all the prototypes to match these. Using 'void *' consistently for addresses gets rid of a couple of type casts, so push that down to the leaf functions where possible. This now passes all randconfig builds on arm, arm64 and x86, but I have not tested it on the other architectures that support kasan, since they tend to fail randconfig builds in other ways. This might fail if any of the 32-bit architectures expect a 'long' instead of 'int' for the size argument. The __asan_allocas_unpoison() function prototype is somewhat weird, since it uses a pointer for 'stack_top' and an size_t for 'stack_bottom'. This looks like it is meant to be 'addr' and 'size' like the others, but the implementation clearly treats them as 'top' and 'bottom'. Link: https://lkml.kernel.org/r/20230509145735.9263-2-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Alexander Potapenko Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Marco Elver Cc: Vincenzo Frascino Cc: Signed-off-by: Andrew Morton --- include/linux/kasan.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/kasan.h b/include/linux/kasan.h index f7ef70661ce2..819b6bc8ac08 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -343,7 +343,7 @@ static inline void *kasan_reset_tag(const void *addr) * @is_write: whether the bad access is a write or a read * @ip: instruction pointer for the accessibility check or the bad access itself */ -bool kasan_report(unsigned long addr, size_t size, +bool kasan_report(const void *addr, size_t size, bool is_write, unsigned long ip); #else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ -- cgit v1.2.3 From 870388db25324fec267862baddc28aaaf0baca73 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 8 May 2023 19:41:27 +0800 Subject: mm: memory_failure: move memory_failure_attr_group under MEMORY_FAILURE The memory_failure_attr_group is only called if MEMORY_FAILURE enabled, move it under this configuration. Link: https://lkml.kernel.org/r/20230508114128.37081-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: Naoya Horiguchi Signed-off-by: Andrew Morton --- include/linux/mm.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 27ce77080c79..f64bfbd53c65 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3586,6 +3586,10 @@ extern void shake_page(struct page *p); extern atomic_long_t num_poisoned_pages __read_mostly; extern int soft_offline_page(unsigned long pfn, int flags); #ifdef CONFIG_MEMORY_FAILURE +/* + * Sysfs entries for memory failure handling statistics. + */ +extern const struct attribute_group memory_failure_attr_group; extern void memory_failure_queue(unsigned long pfn, int flags); extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, bool *migratable_cleared); @@ -3678,11 +3682,6 @@ enum mf_action_page_type { MF_MSG_UNKNOWN, }; -/* - * Sysfs entries for memory failure handling statistics. - */ -extern const struct attribute_group memory_failure_attr_group; - #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) extern void clear_huge_page(struct page *page, unsigned long addr_hint, -- cgit v1.2.3 From 904d58578fce531be07619a2bc2cdc16c9fd49b6 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 16 May 2023 14:38:11 +0800 Subject: mm: page_alloc: move set_zone_contiguous() into mm_init.c set_zone_contiguous() is only used in mm init/hotplug, and clear_zone_contiguous() only used in hotplug, move them from page_alloc.c to the more appropriate file. Link: https://lkml.kernel.org/r/20230516063821.121844-4-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Iurii Zaikin Cc: Kees Cook Cc: Len Brown Cc: Luis Chamberlain Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Cc: Pavel Machek Cc: Rafael J. Wysocki Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 9fcbf5706595..04bc286eed42 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -326,9 +326,6 @@ static inline int remove_memory(u64 start, u64 size) static inline void __remove_memory(u64 start, u64 size) {} #endif /* CONFIG_MEMORY_HOTREMOVE */ -extern void set_zone_contiguous(struct zone *zone); -extern void clear_zone_contiguous(struct zone *zone); - #ifdef CONFIG_MEMORY_HOTPLUG extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); -- cgit v1.2.3 From 0866e82e40fba45dae07e6e8385929b574201752 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 16 May 2023 14:38:15 +0800 Subject: mm: page_alloc: split out FAIL_PAGE_ALLOC ... to a single file to reduce a bit of page_alloc.c. Link: https://lkml.kernel.org/r/20230516063821.121844-8-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Iurii Zaikin Cc: Kees Cook Cc: Len Brown Cc: Luis Chamberlain Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Cc: Pavel Machek Cc: Rafael J. Wysocki Signed-off-by: Andrew Morton --- include/linux/fault-inject.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fault-inject.h b/include/linux/fault-inject.h index 481abf530b3c..6d5edef09d45 100644 --- a/include/linux/fault-inject.h +++ b/include/linux/fault-inject.h @@ -93,6 +93,15 @@ struct kmem_cache; bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order); +#ifdef CONFIG_FAIL_PAGE_ALLOC +bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order); +#else +static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) +{ + return false; +} +#endif /* CONFIG_FAIL_PAGE_ALLOC */ + int should_failslab(struct kmem_cache *s, gfp_t gfpflags); #ifdef CONFIG_FAILSLAB extern bool __should_failslab(struct kmem_cache *s, gfp_t gfpflags); -- cgit v1.2.3 From 884c175f12ce1fabff18ac113349628149fc6cf2 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 16 May 2023 14:38:16 +0800 Subject: mm: page_alloc: split out DEBUG_PAGEALLOC Move DEBUG_PAGEALLOC related functions into a single file to reduce a bit of page_alloc.c. Link: https://lkml.kernel.org/r/20230516063821.121844-9-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Iurii Zaikin Cc: Kees Cook Cc: Len Brown Cc: Luis Chamberlain Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Cc: Pavel Machek Cc: Rafael J. Wysocki Signed-off-by: Andrew Morton --- include/linux/mm.h | 76 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index f64bfbd53c65..2382eaf6fd81 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3471,9 +3471,58 @@ static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) if (debug_pagealloc_enabled_static()) __kernel_map_pages(page, numpages, 0); } + +extern unsigned int _debug_guardpage_minorder; +DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); + +static inline unsigned int debug_guardpage_minorder(void) +{ + return _debug_guardpage_minorder; +} + +static inline bool debug_guardpage_enabled(void) +{ + return static_branch_unlikely(&_debug_guardpage_enabled); +} + +static inline bool page_is_guard(struct page *page) +{ + if (!debug_guardpage_enabled()) + return false; + + return PageGuard(page); +} + +bool __set_page_guard(struct zone *zone, struct page *page, unsigned int order, + int migratetype); +static inline bool set_page_guard(struct zone *zone, struct page *page, + unsigned int order, int migratetype) +{ + if (!debug_guardpage_enabled()) + return false; + return __set_page_guard(zone, page, order, migratetype); +} + +void __clear_page_guard(struct zone *zone, struct page *page, unsigned int order, + int migratetype); +static inline void clear_page_guard(struct zone *zone, struct page *page, + unsigned int order, int migratetype) +{ + if (!debug_guardpage_enabled()) + return; + __clear_page_guard(zone, page, order, migratetype); +} + #else /* CONFIG_DEBUG_PAGEALLOC */ static inline void debug_pagealloc_map_pages(struct page *page, int numpages) {} static inline void debug_pagealloc_unmap_pages(struct page *page, int numpages) {} +static inline unsigned int debug_guardpage_minorder(void) { return 0; } +static inline bool debug_guardpage_enabled(void) { return false; } +static inline bool page_is_guard(struct page *page) { return false; } +static inline bool set_page_guard(struct zone *zone, struct page *page, + unsigned int order, int migratetype) { return false; } +static inline void clear_page_guard(struct zone *zone, struct page *page, + unsigned int order, int migratetype) {} #endif /* CONFIG_DEBUG_PAGEALLOC */ #ifdef __HAVE_ARCH_GATE_AREA @@ -3711,33 +3760,6 @@ static inline bool vma_is_special_huge(const struct vm_area_struct *vma) #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ -#ifdef CONFIG_DEBUG_PAGEALLOC -extern unsigned int _debug_guardpage_minorder; -DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled); - -static inline unsigned int debug_guardpage_minorder(void) -{ - return _debug_guardpage_minorder; -} - -static inline bool debug_guardpage_enabled(void) -{ - return static_branch_unlikely(&_debug_guardpage_enabled); -} - -static inline bool page_is_guard(struct page *page) -{ - if (!debug_guardpage_enabled()) - return false; - - return PageGuard(page); -} -#else -static inline unsigned int debug_guardpage_minorder(void) { return 0; } -static inline bool debug_guardpage_enabled(void) { return false; } -static inline bool page_is_guard(struct page *page) { return false; } -#endif /* CONFIG_DEBUG_PAGEALLOC */ - #if MAX_NUMNODES > 1 void __init setup_nr_node_ids(void); #else -- cgit v1.2.3 From 31a1b9d7fe768db521b12287ec6426983e9787e3 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 16 May 2023 14:38:17 +0800 Subject: mm: page_alloc: move mark_free_page() into snapshot.c The mark_free_page() is only used in kernel/power/snapshot.c, move it out to reduce a bit of page_alloc.c Link: https://lkml.kernel.org/r/20230516063821.121844-10-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Iurii Zaikin Cc: Kees Cook Cc: Len Brown Cc: Luis Chamberlain Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Cc: Pavel Machek Cc: Rafael J. Wysocki Signed-off-by: Andrew Morton --- include/linux/suspend.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index d0d4598a7b3f..3950a7bf33ae 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -364,9 +364,6 @@ struct pbe { struct pbe *next; }; -/* mm/page_alloc.c */ -extern void mark_free_pages(struct zone *zone); - /** * struct platform_hibernation_ops - hibernation platform support * -- cgit v1.2.3 From 07f44ac3c90c50a201307d3fe4dda120ee8394f5 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 16 May 2023 14:38:18 +0800 Subject: mm: page_alloc: move pm_* function into power pm_restrict_gfp_mask()/pm_restore_gfp_mask() only used in power, let's move them out of page_alloc.c. Adding a general gfp_has_io_fs() function which return true if gfp with both __GFP_IO and __GFP_FS flags, then use it inside of pm_suspended_storage(), also the pm_suspended_storage() is moved into suspend.h. Link: https://lkml.kernel.org/r/20230516063821.121844-11-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Iurii Zaikin Cc: Kees Cook Cc: Len Brown Cc: Luis Chamberlain Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Cc: Pavel Machek Cc: Rafael J. Wysocki Signed-off-by: Andrew Morton --- include/linux/gfp.h | 15 ++++----------- include/linux/suspend.h | 6 ++++++ 2 files changed, 10 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index ed8cb537c6a7..665f06675c83 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -338,19 +338,12 @@ extern gfp_t gfp_allowed_mask; /* Returns true if the gfp_mask allows use of ALLOC_NO_WATERMARK */ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask); -extern void pm_restrict_gfp_mask(void); -extern void pm_restore_gfp_mask(void); - -extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma); - -#ifdef CONFIG_PM_SLEEP -extern bool pm_suspended_storage(void); -#else -static inline bool pm_suspended_storage(void) +static inline bool gfp_has_io_fs(gfp_t gfp) { - return false; + return (gfp & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS); } -#endif /* CONFIG_PM_SLEEP */ + +extern gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma); #ifdef CONFIG_CONTIG_ALLOC /* The below functions must be run on a range from a single zone. */ diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 3950a7bf33ae..76923051c03d 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -502,6 +502,11 @@ extern void pm_report_max_hw_sleep(u64 t); extern bool events_check_enabled; extern suspend_state_t pm_suspend_target_state; +static inline bool pm_suspended_storage(void) +{ + return !gfp_has_io_fs(gfp_allowed_mask); +} + extern bool pm_wakeup_pending(void); extern void pm_system_wakeup(void); extern void pm_system_cancel_wakeup(void); @@ -535,6 +540,7 @@ static inline void ksys_sync_helper(void) {} #define pm_notifier(fn, pri) do { (void)(fn); } while (0) +static inline bool pm_suspended_storage(void) { return false; } static inline bool pm_wakeup_pending(void) { return false; } static inline void pm_system_wakeup(void) {} static inline void pm_wakeup_clear(bool reset) {} -- cgit v1.2.3 From e95d372c4cd46b6ec4eeacc07adcb7260ab4cfa0 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 16 May 2023 14:38:20 +0800 Subject: mm: page_alloc: move sysctls into it own fils This moves all page alloc related sysctls to its own file, as part of the kernel/sysctl.c spring cleaning, also move some functions declarations from mm.h into internal.h. Link: https://lkml.kernel.org/r/20230516063821.121844-13-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Iurii Zaikin Cc: Kees Cook Cc: Len Brown Cc: Luis Chamberlain Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Cc: Pavel Machek Cc: Rafael J. Wysocki Signed-off-by: Andrew Morton --- include/linux/mm.h | 11 ----------- include/linux/mmzone.h | 21 --------------------- 2 files changed, 32 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 2382eaf6fd81..6d7e03d83da7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2994,12 +2994,6 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); #endif extern void set_dma_reserve(unsigned long new_dma_reserve); -extern void memmap_init_range(unsigned long, int, unsigned long, - unsigned long, unsigned long, enum meminit_context, - struct vmem_altmap *, int migratetype); -extern void setup_per_zone_wmarks(void); -extern void calculate_min_free_kbytes(void); -extern int __meminit init_per_zone_wmark_min(void); extern void mem_init(void); extern void __init mmap_init(void); @@ -3020,11 +3014,6 @@ void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); extern void setup_per_cpu_pageset(void); -/* page_alloc.c */ -extern int min_free_kbytes; -extern int watermark_boost_factor; -extern int watermark_scale_factor; - /* nommu.c */ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a4889c9d4055..3a68326c9989 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1512,27 +1512,6 @@ static inline bool has_managed_dma(void) } #endif -/* These two functions are used to setup the per zone pages min values */ -struct ctl_table; - -int min_free_kbytes_sysctl_handler(struct ctl_table *, int, void *, size_t *, - loff_t *); -int watermark_scale_factor_sysctl_handler(struct ctl_table *, int, void *, - size_t *, loff_t *); -extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES]; -int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, void *, - size_t *, loff_t *); -int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *, int, - void *, size_t *, loff_t *); -int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int, - void *, size_t *, loff_t *); -int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int, - void *, size_t *, loff_t *); -int numa_zonelist_order_handler(struct ctl_table *, int, - void *, size_t *, loff_t *); -extern int percpu_pagelist_high_fraction; -extern char numa_zonelist_order[]; -#define NUMA_ZONELIST_ORDER_LEN 16 #ifndef CONFIG_NUMA -- cgit v1.2.3 From f6797adff7f09b4d7f7607c99116409b5ddb54d9 Mon Sep 17 00:00:00 2001 From: Sidhartha Kumar Date: Tue, 16 May 2023 15:52:05 -0700 Subject: mm/hugetlb: remove hugetlb_page_subpool() All users of hugetlb_page_subpool() have been converted to use the folio equivalent. This function can be safely removed. Link: https://lkml.kernel.org/r/20230516225205.1429196-1-sidhartha.kumar@oracle.com Signed-off-by: Sidhartha Kumar Reviewed-by: David Hildenbrand Reviewed-by: Mike Kravetz Cc: Matthew Wilcox (Oracle) Cc: Muchun Song Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 6d041aa9f0fe..f1543a0568ff 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -757,14 +757,6 @@ static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio return folio->_hugetlb_subpool; } -/* - * hugetlb page subpool pointer located in hpage[2].hugetlb_subpool - */ -static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage) -{ - return hugetlb_folio_subpool(page_folio(hpage)); -} - static inline void hugetlb_set_folio_subpool(struct folio *folio, struct hugepage_subpool *subpool) { @@ -1031,11 +1023,6 @@ static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio return NULL; } -static inline struct hugepage_subpool *hugetlb_page_subpool(struct page *hpage) -{ - return NULL; -} - static inline int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list) { -- cgit v1.2.3 From 54d020692b342f7bd02d7f5795fb5c401caecfcc Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 17 May 2023 20:25:33 +0100 Subject: mm/gup: remove unused vmas parameter from get_user_pages() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "remove the vmas parameter from GUP APIs", v6. (pin_/get)_user_pages[_remote]() each provide an optional output parameter for an array of VMA objects associated with each page in the input range. These provide the means for VMAs to be returned, as long as mm->mmap_lock is never released during the GUP operation (i.e. the internal flag FOLL_UNLOCKABLE is not specified). In addition, these VMAs can only be accessed with the mmap_lock held and become invalidated the moment it is released. The vast majority of invocations do not use this functionality and of those that do, all but one case retrieve a single VMA to perform checks upon. It is not egregious in the single VMA cases to simply replace the operation with a vma_lookup(). In these cases we duplicate the (fast) lookup on a slow path already under the mmap_lock, abstracted to a new get_user_page_vma_remote() inline helper function which also performs error checking and reference count maintenance. The special case is io_uring, where io_pin_pages() specifically needs to assert that the VMAs underlying the range do not result in broken long-term GUP file-backed mappings. As GUP now internally asserts that FOLL_LONGTERM mappings are not file-backed in a broken fashion (i.e. requiring dirty tracking) - as implemented in "mm/gup: disallow FOLL_LONGTERM GUP-nonfast writing to file-backed mappings" - this logic is no longer required and so we can simply remove it altogether from io_uring. Eliminating the vmas parameter eliminates an entire class of danging pointer errors that might have occured should the lock have been incorrectly released. In addition, the API is simplified and now clearly expresses what it is intended for - applying the specified GUP flags and (if pinning) returning pinned pages. This change additionally opens the door to further potential improvements in GUP and the possible marrying of disparate code paths. I have run this series against gup_test with no issues. Thanks to Matthew Wilcox for suggesting this refactoring! This patch (of 6): No invocation of get_user_pages() use the vmas parameter, so remove it. The GUP API is confusing and caveated. Recent changes have done much to improve that, however there is more we can do. Exporting vmas is a prime target as the caller has to be extremely careful to preclude their use after the mmap_lock has expired or otherwise be left with dangling pointers. Removing the vmas parameter focuses the GUP functions upon their primary purpose - pinning (and outputting) pages as well as performing the actions implied by the input flags. This is part of a patch series aiming to remove the vmas parameter altogether. Link: https://lkml.kernel.org/r/cover.1684350871.git.lstoakes@gmail.com Link: https://lkml.kernel.org/r/589e0c64794668ffc799651e8d85e703262b1e9d.1684350871.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Suggested-by: Matthew Wilcox (Oracle) Acked-by: Greg Kroah-Hartman Acked-by: David Hildenbrand Reviewed-by: Jason Gunthorpe Acked-by: Christian König (for radeon parts) Acked-by: Jarkko Sakkinen Reviewed-by: Christoph Hellwig Acked-by: Sean Christopherson (KVM) Cc: Catalin Marinas Cc: Dennis Dalessandro Cc: Janosch Frank Cc: Jens Axboe Cc: Sakari Ailus Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 6d7e03d83da7..6336253c18e2 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2369,8 +2369,7 @@ long pin_user_pages_remote(struct mm_struct *mm, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas, int *locked); long get_user_pages(unsigned long start, unsigned long nr_pages, - unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas); + unsigned int gup_flags, struct page **pages); long pin_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas); -- cgit v1.2.3 From 0b295316b3a9b7858eafbebdc31b4827a6edde03 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 17 May 2023 20:25:36 +0100 Subject: mm/gup: remove unused vmas parameter from pin_user_pages_remote() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No invocation of pin_user_pages_remote() uses the vmas parameter, so remove it. This forms part of a larger patch set eliminating the use of the vmas parameters altogether. Link: https://lkml.kernel.org/r/28f000beb81e45bf538a2aaa77c90f5482b67a32.1684350871.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Acked-by: David Hildenbrand Reviewed-by: Jason Gunthorpe Reviewed-by: Christoph Hellwig Cc: Catalin Marinas Cc: Christian König Cc: Dennis Dalessandro Cc: Greg Kroah-Hartman Cc: Janosch Frank Cc: Jarkko Sakkinen Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Sakari Ailus Cc: Sean Christopherson Signed-off-by: Andrew Morton --- include/linux/mm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 6336253c18e2..cf17ffdf4fbf 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2367,7 +2367,7 @@ long get_user_pages_remote(struct mm_struct *mm, long pin_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas, int *locked); + int *locked); long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages); long pin_user_pages(unsigned long start, unsigned long nr_pages, -- cgit v1.2.3 From ca5e863233e8f6acd1792fd85d6bc2729a1b2c10 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 17 May 2023 20:25:39 +0100 Subject: mm/gup: remove vmas parameter from get_user_pages_remote() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only instances of get_user_pages_remote() invocations which used the vmas parameter were for a single page which can instead simply look up the VMA directly. In particular:- - __update_ref_ctr() looked up the VMA but did nothing with it so we simply remove it. - __access_remote_vm() was already using vma_lookup() when the original lookup failed so by doing the lookup directly this also de-duplicates the code. We are able to perform these VMA operations as we already hold the mmap_lock in order to be able to call get_user_pages_remote(). As part of this work we add get_user_page_vma_remote() which abstracts the VMA lookup, error handling and decrementing the page reference count should the VMA lookup fail. This forms part of a broader set of patches intended to eliminate the vmas parameter altogether. [akpm@linux-foundation.org: avoid passing NULL to PTR_ERR] Link: https://lkml.kernel.org/r/d20128c849ecdbf4dd01cc828fcec32127ed939a.1684350871.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Reviewed-by: Catalin Marinas (for arm64) Acked-by: David Hildenbrand Reviewed-by: Janosch Frank (for s390) Reviewed-by: Christoph Hellwig Cc: Christian König Cc: Dennis Dalessandro Cc: Greg Kroah-Hartman Cc: Jarkko Sakkinen Cc: Jason Gunthorpe Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Sakari Ailus Cc: Sean Christopherson Signed-off-by: Andrew Morton --- include/linux/mm.h | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index cf17ffdf4fbf..fcbfb961b49f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2353,6 +2353,9 @@ static inline void unmap_shared_mapping_range(struct address_space *mapping, unmap_mapping_range(mapping, holebegin, holelen, 0); } +static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, + unsigned long addr); + extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags); extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, @@ -2361,13 +2364,38 @@ extern int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, - unsigned long start, unsigned long nr_pages, - unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas, int *locked); + unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages, + int *locked); long pin_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, int *locked); + +static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, + unsigned long addr, + int gup_flags, + struct vm_area_struct **vmap) +{ + struct page *page; + struct vm_area_struct *vma; + int got = get_user_pages_remote(mm, addr, 1, gup_flags, &page, NULL); + + if (got < 0) + return ERR_PTR(got); + if (got == 0) + return NULL; + + vma = vma_lookup(mm, addr); + if (WARN_ON_ONCE(!vma)) { + put_page(page); + return ERR_PTR(-EINVAL); + } + + *vmap = vma; + return page; +} + long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages); long pin_user_pages(unsigned long start, unsigned long nr_pages, -- cgit v1.2.3 From 4c630f307455c06f99bdeca7f7a1ab5318604fe0 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 17 May 2023 20:25:45 +0100 Subject: mm/gup: remove vmas parameter from pin_user_pages() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are now in a position where no caller of pin_user_pages() requires the vmas parameter at all, so eliminate this parameter from the function and all callers. This clears the way to removing the vmas parameter from GUP altogether. Link: https://lkml.kernel.org/r/195a99ae949c9f5cb589d2222b736ced96ec199a.1684350871.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Acked-by: David Hildenbrand Acked-by: Dennis Dalessandro [qib] Reviewed-by: Christoph Hellwig Acked-by: Sakari Ailus [drivers/media] Cc: Catalin Marinas Cc: Christian König Cc: Greg Kroah-Hartman Cc: Janosch Frank Cc: Jarkko Sakkinen Cc: Jason Gunthorpe Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Sean Christopherson Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index fcbfb961b49f..280429ffa91d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2399,8 +2399,7 @@ static inline struct page *get_user_page_vma_remote(struct mm_struct *mm, long get_user_pages(unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages); long pin_user_pages(unsigned long start, unsigned long nr_pages, - unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas); + unsigned int gup_flags, struct page **pages); long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, struct page **pages, unsigned int gup_flags); long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, -- cgit v1.2.3 From b2cac248191b7466c5819e0da617b0705a26e197 Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Wed, 17 May 2023 20:25:48 +0100 Subject: mm/gup: remove vmas array from internal GUP functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now we have eliminated all callers to GUP APIs which use the vmas parameter, eliminate it altogether. This eliminates a class of bugs where vmas might have been kept around longer than the mmap_lock and thus we need not be concerned about locks being dropped during this operation leaving behind dangling pointers. This simplifies the GUP API and makes it considerably clearer as to its purpose - follow flags are applied and if pinning, an array of pages is returned. Link: https://lkml.kernel.org/r/6811b4b2b4b3baf3dd07f422bb18853bb2cd09fb.1684350871.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Acked-by: David Hildenbrand Reviewed-by: Christoph Hellwig Cc: Catalin Marinas Cc: Christian König Cc: Dennis Dalessandro Cc: Greg Kroah-Hartman Cc: Janosch Frank Cc: Jarkko Sakkinen Cc: Jason Gunthorpe Cc: Jens Axboe Cc: Matthew Wilcox (Oracle) Cc: Sakari Ailus Cc: Sean Christopherson Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index f1543a0568ff..21f942025fec 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -133,9 +133,8 @@ int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma, unsigned long address, unsigned int flags); long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, - struct page **, struct vm_area_struct **, - unsigned long *, unsigned long *, long, unsigned int, - int *); + struct page **, unsigned long *, unsigned long *, + long, unsigned int, int *); void unmap_hugepage_range(struct vm_area_struct *, unsigned long, unsigned long, struct page *, zap_flags_t); @@ -306,9 +305,8 @@ static inline struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma, static inline long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, - struct vm_area_struct **vmas, unsigned long *position, - unsigned long *nr_pages, long i, unsigned int flags, - int *nonblocking) + unsigned long *position, unsigned long *nr_pages, + long i, unsigned int flags, int *nonblocking) { BUG(); return 0; -- cgit v1.2.3 From 4e096ae1801e24b338e02715c65c3ffa8883ba5d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 13 May 2023 01:11:01 +0100 Subject: mm: convert migrate_pages() to work on folios Almost all of the callers & implementors of migrate_pages() were already converted to use folios. compaction_alloc() & compaction_free() are trivial to convert a part of this patch and not worth splitting out. Link: https://lkml.kernel.org/r/20230513001101.276972-1-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: "Huang, Ying" Signed-off-by: Andrew Morton --- include/linux/migrate.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 6241a1596a75..6de5756d8533 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -7,8 +7,8 @@ #include #include -typedef struct page *new_page_t(struct page *page, unsigned long private); -typedef void free_page_t(struct page *page, unsigned long private); +typedef struct folio *new_folio_t(struct folio *folio, unsigned long private); +typedef void free_folio_t(struct folio *folio, unsigned long private); struct migration_target_control; @@ -67,10 +67,10 @@ int migrate_folio_extra(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode, int extra_count); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); -int migrate_pages(struct list_head *l, new_page_t new, free_page_t free, +int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, unsigned long private, enum migrate_mode mode, int reason, unsigned int *ret_succeeded); -struct page *alloc_migration_target(struct page *page, unsigned long private); +struct folio *alloc_migration_target(struct folio *src, unsigned long private); bool isolate_movable_page(struct page *page, isolate_mode_t mode); int migrate_huge_page_move_mapping(struct address_space *mapping, @@ -85,11 +85,11 @@ int folio_migrate_mapping(struct address_space *mapping, #else static inline void putback_movable_pages(struct list_head *l) {} -static inline int migrate_pages(struct list_head *l, new_page_t new, - free_page_t free, unsigned long private, enum migrate_mode mode, - int reason, unsigned int *ret_succeeded) +static inline int migrate_pages(struct list_head *l, new_folio_t new, + free_folio_t free, unsigned long private, + enum migrate_mode mode, int reason, unsigned int *ret_succeeded) { return -ENOSYS; } -static inline struct page *alloc_migration_target(struct page *page, +static inline struct folio *alloc_migration_target(struct folio *src, unsigned long private) { return NULL; } static inline bool isolate_movable_page(struct page *page, isolate_mode_t mode) -- cgit v1.2.3 From 89f499f35c11af61ba7075ddc23209d10805a25a Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:14 -0400 Subject: maple_tree: add format option to mt_dump() Allow different formatting strings to be used when dumping the tree. Currently supports hex and decimal. Link: https://lkml.kernel.org/r/20230518145544.1722059-6-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 1fadb5f5978b..140fb271be4a 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -670,10 +670,15 @@ void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max); #ifdef CONFIG_DEBUG_MAPLE_TREE +enum mt_dump_format { + mt_dump_dec, + mt_dump_hex, +}; + extern atomic_t maple_tree_tests_run; extern atomic_t maple_tree_tests_passed; -void mt_dump(const struct maple_tree *mt); +void mt_dump(const struct maple_tree *mt, enum mt_dump_format format); void mt_validate(struct maple_tree *mt); void mt_cache_shrink(void); #define MT_BUG_ON(__tree, __x) do { \ @@ -681,7 +686,7 @@ void mt_cache_shrink(void); if (__x) { \ pr_info("BUG at %s:%d (%u)\n", \ __func__, __LINE__, __x); \ - mt_dump(__tree); \ + mt_dump(__tree, mt_dump_hex); \ pr_info("Pass: %u Run:%u\n", \ atomic_read(&maple_tree_tests_passed), \ atomic_read(&maple_tree_tests_run)); \ -- cgit v1.2.3 From f0a1f866aba1ca62ef6f17d1c441eba65f2d6845 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:15 -0400 Subject: maple_tree: add debug BUG_ON and WARN_ON variants Add debug macros to dump the maple state and/or the tree for both warning and bug_on calls. Link: https://lkml.kernel.org/r/20230518145544.1722059-7-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 100 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 140fb271be4a..204d7941a39e 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -482,13 +482,13 @@ static inline void mas_init(struct ma_state *mas, struct maple_tree *tree, } /* Checks if a mas has not found anything */ -static inline bool mas_is_none(struct ma_state *mas) +static inline bool mas_is_none(const struct ma_state *mas) { return mas->node == MAS_NONE; } /* Checks if a mas has been paused */ -static inline bool mas_is_paused(struct ma_state *mas) +static inline bool mas_is_paused(const struct ma_state *mas) { return mas->node == MAS_PAUSE; } @@ -679,6 +679,8 @@ extern atomic_t maple_tree_tests_run; extern atomic_t maple_tree_tests_passed; void mt_dump(const struct maple_tree *mt, enum mt_dump_format format); +void mas_dump(const struct ma_state *mas); +void mas_wr_dump(const struct ma_wr_state *wr_mas); void mt_validate(struct maple_tree *mt); void mt_cache_shrink(void); #define MT_BUG_ON(__tree, __x) do { \ @@ -695,8 +697,100 @@ void mt_cache_shrink(void); atomic_inc(&maple_tree_tests_passed); \ } \ } while (0) + +#define MAS_BUG_ON(__mas, __x) do { \ + atomic_inc(&maple_tree_tests_run); \ + if (__x) { \ + pr_info("BUG at %s:%d (%u)\n", \ + __func__, __LINE__, __x); \ + mas_dump(__mas); \ + mt_dump((__mas)->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", \ + atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } else { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ +} while (0) + +#define MAS_WR_BUG_ON(__wrmas, __x) do { \ + atomic_inc(&maple_tree_tests_run); \ + if (__x) { \ + pr_info("BUG at %s:%d (%u)\n", \ + __func__, __LINE__, __x); \ + mas_wr_dump(__wrmas); \ + mas_dump((__wrmas)->mas); \ + mt_dump((__wrmas)->mas->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", \ + atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } else { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ +} while (0) + +#define MT_WARN_ON(__tree, __x) ({ \ + int ret = !!(__x); \ + atomic_inc(&maple_tree_tests_run); \ + if (ret) { \ + pr_info("WARN at %s:%d (%u)\n", \ + __func__, __LINE__, __x); \ + mt_dump(__tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", \ + atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } else { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + unlikely(ret); \ +}) + +#define MAS_WARN_ON(__mas, __x) ({ \ + int ret = !!(__x); \ + atomic_inc(&maple_tree_tests_run); \ + if (ret) { \ + pr_info("WARN at %s:%d (%u)\n", \ + __func__, __LINE__, __x); \ + mas_dump(__mas); \ + mt_dump((__mas)->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", \ + atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } else { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + unlikely(ret); \ +}) + +#define MAS_WR_WARN_ON(__wrmas, __x) ({ \ + int ret = !!(__x); \ + atomic_inc(&maple_tree_tests_run); \ + if (ret) { \ + pr_info("WARN at %s:%d (%u)\n", \ + __func__, __LINE__, __x); \ + mas_wr_dump(__wrmas); \ + mas_dump((__wrmas)->mas); \ + mt_dump((__wrmas)->mas->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", \ + atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } else { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + unlikely(ret); \ +}) #else -#define MT_BUG_ON(__tree, __x) BUG_ON(__x) +#define MT_BUG_ON(__tree, __x) BUG_ON(__x) +#define MAS_BUG_ON(__mas, __x) BUG_ON(__x) +#define MAS_WR_BUG_ON(__mas, __x) BUG_ON(__x) +#define MT_WARN_ON(__tree, __x) WARN_ON(__x) +#define MAS_WARN_ON(__mas, __x) WARN_ON(__x) +#define MAS_WR_WARN_ON(__mas, __x) WARN_ON(__x) #endif /* CONFIG_DEBUG_MAPLE_TREE */ #endif /*_LINUX_MAPLE_TREE_H */ -- cgit v1.2.3 From 7f2f9dc16fee59afdec2df8c794e97c36e387257 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:17 -0400 Subject: maple_tree: change RCU checks to WARN_ON() instead of BUG_ON() If RCU is enabled and the tree isn't locked, just warn the user and avoid crashing the kernel. Link: https://lkml.kernel.org/r/20230518145544.1722059-9-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 204d7941a39e..ed92abf4c1fb 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -616,7 +616,7 @@ static inline void mt_clear_in_rcu(struct maple_tree *mt) return; if (mt_external_lock(mt)) { - BUG_ON(!mt_lock_is_held(mt)); + WARN_ON(!mt_lock_is_held(mt)); mt->ma_flags &= ~MT_FLAGS_USE_RCU; } else { mtree_lock(mt); @@ -635,7 +635,7 @@ static inline void mt_set_in_rcu(struct maple_tree *mt) return; if (mt_external_lock(mt)) { - BUG_ON(!mt_lock_is_held(mt)); + WARN_ON(!mt_lock_is_held(mt)); mt->ma_flags |= MT_FLAGS_USE_RCU; } else { mtree_lock(mt); -- cgit v1.2.3 From b50e195ff436625b26dcc9839bc52cc7c5bf1a54 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:26 -0400 Subject: mm: update validate_mm() to use vma iterator Use the vma iterator in the validation code and combine the code to check the maple tree into the main validate_mm() function. Introduce a new function vma_iter_dump_tree() to dump the maple tree in hex layout. Replace all calls to validate_mm_mt() with validate_mm(). [Liam.Howlett@oracle.com: update validate_mm() to use vma iterator CONFIG flag] Link: https://lkml.kernel.org/r/20230606183538.588190-1-Liam.Howlett@oracle.com Link: https://lkml.kernel.org/r/20230518145544.1722059-18-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/mmdebug.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mmdebug.h b/include/linux/mmdebug.h index b8728d11c949..7c3e7b0b0e8f 100644 --- a/include/linux/mmdebug.h +++ b/include/linux/mmdebug.h @@ -8,10 +8,12 @@ struct page; struct vm_area_struct; struct mm_struct; +struct vma_iterator; void dump_page(struct page *page, const char *reason); void dump_vma(const struct vm_area_struct *vma); void dump_mm(const struct mm_struct *mm); +void vma_iter_dump_tree(const struct vma_iterator *vmi); #ifdef CONFIG_DEBUG_VM #define VM_BUG_ON(cond) BUG_ON(cond) @@ -74,6 +76,17 @@ void dump_mm(const struct mm_struct *mm); } \ unlikely(__ret_warn_once); \ }) +#define VM_WARN_ON_ONCE_MM(cond, mm) ({ \ + static bool __section(".data.once") __warned; \ + int __ret_warn_once = !!(cond); \ + \ + if (unlikely(__ret_warn_once && !__warned)) { \ + dump_mm(mm); \ + __warned = true; \ + WARN_ON(1); \ + } \ + unlikely(__ret_warn_once); \ +}) #define VM_WARN_ON(cond) (void)WARN_ON(cond) #define VM_WARN_ON_ONCE(cond) (void)WARN_ON_ONCE(cond) @@ -90,6 +103,7 @@ void dump_mm(const struct mm_struct *mm); #define VM_WARN_ON_ONCE_PAGE(cond, page) BUILD_BUG_ON_INVALID(cond) #define VM_WARN_ON_FOLIO(cond, folio) BUILD_BUG_ON_INVALID(cond) #define VM_WARN_ON_ONCE_FOLIO(cond, folio) BUILD_BUG_ON_INVALID(cond) +#define VM_WARN_ON_ONCE_MM(cond, mm) BUILD_BUG_ON_INVALID(cond) #define VM_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond) #define VM_WARN(cond, format...) BUILD_BUG_ON_INVALID(cond) #endif -- cgit v1.2.3 From 6169b553195a193c52a675e45a9578f595fe194f Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:37 -0400 Subject: maple_tree: add mas_next_range() and mas_find_range() interfaces Some users of the maple tree may want to move to the next range in the tree, even if it stores a NULL. This family of function provides that functionality by advancing one slot at a time and returning the result, while mas_contiguous() will iterate over the range and stop on encountering the first NULL. Link: https://lkml.kernel.org/r/20230518145544.1722059-29-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index ed92abf4c1fb..9d040043858a 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -455,6 +455,7 @@ void *mas_erase(struct ma_state *mas); int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp); void mas_store_prealloc(struct ma_state *mas, void *entry); void *mas_find(struct ma_state *mas, unsigned long max); +void *mas_find_range(struct ma_state *mas, unsigned long max); void *mas_find_rev(struct ma_state *mas, unsigned long min); int mas_preallocate(struct ma_state *mas, gfp_t gfp); bool mas_is_err(struct ma_state *mas); @@ -467,6 +468,7 @@ int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); void *mas_prev(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); +void *mas_next_range(struct ma_state *mas, unsigned long max); int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size); @@ -528,7 +530,6 @@ static inline void mas_reset(struct ma_state *mas) #define mas_for_each(__mas, __entry, __max) \ while (((__entry) = mas_find((__mas), (__max))) != NULL) - /** * mas_set_range() - Set up Maple Tree operation state for a different index. * @mas: Maple Tree operation state. -- cgit v1.2.3 From 6b9e93e0102048e64681c2fa265ae81c221f6c6d Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:40 -0400 Subject: maple_tree: add mas_prev_range() and mas_find_range_rev interface Some users of the maple tree may want to move to the previous range regardless of the value stored there. Add this interface as well as the 'find' variant to support walking to the first value, then iterating over the previous ranges. Link: https://lkml.kernel.org/r/20230518145544.1722059-32-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: Vernon Yang Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 9d040043858a..541675229568 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -457,6 +457,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry); void *mas_find(struct ma_state *mas, unsigned long max); void *mas_find_range(struct ma_state *mas, unsigned long max); void *mas_find_rev(struct ma_state *mas, unsigned long min); +void *mas_find_range_rev(struct ma_state *mas, unsigned long max); int mas_preallocate(struct ma_state *mas, gfp_t gfp); bool mas_is_err(struct ma_state *mas); @@ -467,6 +468,7 @@ void mas_destroy(struct ma_state *mas); int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); void *mas_prev(struct ma_state *mas, unsigned long min); +void *mas_prev_range(struct ma_state *mas, unsigned long max); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); -- cgit v1.2.3 From bb5dbd2272b8d7b3a34d234bb916819afbf802d1 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 18 May 2023 10:55:43 -0400 Subject: mm: add vma_iter_{next,prev}_range() to vma iterator Add functionality to the VMA iterator to advance and retreat one offset within the maple tree, regardless of the value contained. This can lead to less re-walking to find an area of interest, especially when there is nothing in that offset. Link: https://lkml.kernel.org/r/20230518145544.1722059-35-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton --- include/linux/mm.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 280429ffa91d..62bb3272e531 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -866,11 +866,24 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) return mas_find(&vmi->mas, ULONG_MAX); } +static inline +struct vm_area_struct *vma_iter_next_range(struct vma_iterator *vmi) +{ + return mas_next_range(&vmi->mas, ULONG_MAX); +} + + static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) { return mas_prev(&vmi->mas, 0); } +static inline +struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi) +{ + return mas_prev_range(&vmi->mas, 0); +} + static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) { return vmi->mas.index; -- cgit v1.2.3 From ecd8b2928f2efc7b678b361d51920c15b5ef3885 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Fri, 19 May 2023 14:39:55 +0200 Subject: mm: compaction: remove compaction result helpers Patch series "mm: compaction: cleanups & simplifications". These compaction cleanups are split out from the huge page allocator series[1], as requested by reviewer feedback. [1] https://lore.kernel.org/linux-mm/20230418191313.268131-1-hannes@cmpxchg.org/ This patch (of 5): The compaction result helpers encode quirks that are specific to the allocator's retry logic. E.g. COMPACT_SUCCESS and COMPACT_COMPLETE actually represent failures that should be retried upon, and so on. I frequently found myself pulling up the helper implementation in order to understand and work on the retry logic. They're not quite clean abstractions; rather they split the retry logic into two locations. Remove the helpers and inline the checks. Then comment on the result interpretations directly where the decision making happens. Link: https://lkml.kernel.org/r/20230519123959.77335-1-hannes@cmpxchg.org Link: https://lkml.kernel.org/r/20230519123959.77335-2-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Acked-by: Vlastimil Babka Cc: Mel Gorman Cc: Michal Hocko Signed-off-by: Andrew Morton --- include/linux/compaction.h | 92 ---------------------------------------------- 1 file changed, 92 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compaction.h b/include/linux/compaction.h index a6e512cfb670..1f0328a2ba48 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -95,78 +95,6 @@ extern enum compact_result compaction_suitable(struct zone *zone, int order, extern void compaction_defer_reset(struct zone *zone, int order, bool alloc_success); -/* Compaction has made some progress and retrying makes sense */ -static inline bool compaction_made_progress(enum compact_result result) -{ - /* - * Even though this might sound confusing this in fact tells us - * that the compaction successfully isolated and migrated some - * pageblocks. - */ - if (result == COMPACT_SUCCESS) - return true; - - return false; -} - -/* Compaction has failed and it doesn't make much sense to keep retrying. */ -static inline bool compaction_failed(enum compact_result result) -{ - /* All zones were scanned completely and still not result. */ - if (result == COMPACT_COMPLETE) - return true; - - return false; -} - -/* Compaction needs reclaim to be performed first, so it can continue. */ -static inline bool compaction_needs_reclaim(enum compact_result result) -{ - /* - * Compaction backed off due to watermark checks for order-0 - * so the regular reclaim has to try harder and reclaim something. - */ - if (result == COMPACT_SKIPPED) - return true; - - return false; -} - -/* - * Compaction has backed off for some reason after doing some work or none - * at all. It might be throttling or lock contention. Retrying might be still - * worthwhile, but with a higher priority if allowed. - */ -static inline bool compaction_withdrawn(enum compact_result result) -{ - /* - * If compaction is deferred for high-order allocations, it is - * because sync compaction recently failed. If this is the case - * and the caller requested a THP allocation, we do not want - * to heavily disrupt the system, so we fail the allocation - * instead of entering direct reclaim. - */ - if (result == COMPACT_DEFERRED) - return true; - - /* - * If compaction in async mode encounters contention or blocks higher - * priority task we back off early rather than cause stalls. - */ - if (result == COMPACT_CONTENDED) - return true; - - /* - * Page scanners have met but we haven't scanned full zones so this - * is a back off in fact. - */ - if (result == COMPACT_PARTIAL_SKIPPED) - return true; - - return false; -} - - bool compaction_zonelist_suitable(struct alloc_context *ac, int order, int alloc_flags); @@ -185,26 +113,6 @@ static inline enum compact_result compaction_suitable(struct zone *zone, int ord return COMPACT_SKIPPED; } -static inline bool compaction_made_progress(enum compact_result result) -{ - return false; -} - -static inline bool compaction_failed(enum compact_result result) -{ - return false; -} - -static inline bool compaction_needs_reclaim(enum compact_result result) -{ - return false; -} - -static inline bool compaction_withdrawn(enum compact_result result) -{ - return true; -} - static inline void kcompactd_run(int nid) { } -- cgit v1.2.3 From e8606320e9af9774fd879e71c940fc9e5fd9b901 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Fri, 19 May 2023 14:39:57 +0200 Subject: mm: compaction: refactor __compaction_suitable() __compaction_suitable() is supposed to check for available migration targets. However, it also checks whether the operation was requested via /proc/sys/vm/compact_memory, and whether the original allocation request can already succeed. These don't apply to all callsites. Move the checks out to the callers, so that later patches can deal with them one by one. No functional change intended. [hannes@cmpxchg.org: fix comment, per Vlastimil] Link: https://lkml.kernel.org/r/20230602144942.GC161817@cmpxchg.org Link: https://lkml.kernel.org/r/20230519123959.77335-4-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Acked-by: Vlastimil Babka Cc: Mel Gorman Cc: Michal Hocko Signed-off-by: Andrew Morton --- include/linux/compaction.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 1f0328a2ba48..9f7cf3e1bf89 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -90,7 +90,7 @@ extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, struct page **page); extern void reset_isolation_suitable(pg_data_t *pgdat); extern enum compact_result compaction_suitable(struct zone *zone, int order, - unsigned int alloc_flags, int highest_zoneidx); + int highest_zoneidx); extern void compaction_defer_reset(struct zone *zone, int order, bool alloc_success); @@ -108,7 +108,7 @@ static inline void reset_isolation_suitable(pg_data_t *pgdat) } static inline enum compact_result compaction_suitable(struct zone *zone, int order, - int alloc_flags, int highest_zoneidx) + int highest_zoneidx) { return COMPACT_SKIPPED; } -- cgit v1.2.3 From 3cf04937529020e149666f56a41ebdeb226b69ed Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Fri, 2 Jun 2023 11:12:04 -0400 Subject: mm: compaction: have compaction_suitable() return bool Since it only returns COMPACT_CONTINUE or COMPACT_SKIPPED now, a bool return value simplifies the callsites. Link: https://lkml.kernel.org/r/20230602151204.GD161817@cmpxchg.org Signed-off-by: Johannes Weiner Suggested-by: Vlastimil Babka Acked-by: Vlastimil Babka Cc: Baolin Wang Cc: Mel Gorman Cc: Michal Hocko Signed-off-by: Andrew Morton --- include/linux/compaction.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 9f7cf3e1bf89..57b16e69c19a 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -89,7 +89,7 @@ extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, const struct alloc_context *ac, enum compact_priority prio, struct page **page); extern void reset_isolation_suitable(pg_data_t *pgdat); -extern enum compact_result compaction_suitable(struct zone *zone, int order, +extern bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx); extern void compaction_defer_reset(struct zone *zone, int order, @@ -107,10 +107,10 @@ static inline void reset_isolation_suitable(pg_data_t *pgdat) { } -static inline enum compact_result compaction_suitable(struct zone *zone, int order, +static inline bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx) { - return COMPACT_SKIPPED; + return false; } static inline void kcompactd_run(int nid) -- cgit v1.2.3 From 5c7e7a0d79072eb02780a2c0dee730b23cde711d Mon Sep 17 00:00:00 2001 From: "T.J. Alumbaugh" Date: Mon, 22 May 2023 11:20:56 +0000 Subject: mm: multi-gen LRU: cleanup lru_gen_soft_reclaim() lru_gen_soft_reclaim() gets the lruvec from the memcg and node ID to keep a cleaner interface on the caller side. Link: https://lkml.kernel.org/r/20230522112058.2965866-2-talumbau@google.com Signed-off-by: T.J. Alumbaugh Reviewed-by: Yuanchu Xie Cc: David Hildenbrand Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 3a68326c9989..5a7ada0413da 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -534,7 +534,7 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg); void lru_gen_online_memcg(struct mem_cgroup *memcg); void lru_gen_offline_memcg(struct mem_cgroup *memcg); void lru_gen_release_memcg(struct mem_cgroup *memcg); -void lru_gen_soft_reclaim(struct lruvec *lruvec); +void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid); #else /* !CONFIG_MEMCG */ @@ -585,7 +585,7 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg) { } -static inline void lru_gen_soft_reclaim(struct lruvec *lruvec) +static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) { } -- cgit v1.2.3 From 06b27ce36a1a3dc5ea6f8314d0c7d1baa9f8ece7 Mon Sep 17 00:00:00 2001 From: Peng Zhang Date: Wed, 24 May 2023 11:12:47 +0800 Subject: maple_tree: relocate the declaration of mas_empty_area_rev(). Relocate the declaration of mas_empty_area_rev() so that mas_empty_area() and mas_empty_area_rev() are together. Link: https://lkml.kernel.org/r/20230524031247.65949-11-zhangpeng.00@bytedance.com Signed-off-by: Peng Zhang Reviewed-by: Liam R. Howlett Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 541675229568..295548cca8b3 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -474,6 +474,12 @@ void *mas_next_range(struct ma_state *mas, unsigned long max); int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size); +/* + * This finds an empty area from the highest address to the lowest. + * AKA "Topdown" version, + */ +int mas_empty_area_rev(struct ma_state *mas, unsigned long min, + unsigned long max, unsigned long size); static inline void mas_init(struct ma_state *mas, struct maple_tree *tree, unsigned long addr) @@ -497,12 +503,6 @@ static inline bool mas_is_paused(const struct ma_state *mas) return mas->node == MAS_PAUSE; } -/* - * This finds an empty area from the highest address to the lowest. - * AKA "Topdown" version, - */ -int mas_empty_area_rev(struct ma_state *mas, unsigned long min, - unsigned long max, unsigned long size); /** * mas_reset() - Reset a Maple Tree operation state. * @mas: Maple Tree operation state. -- cgit v1.2.3 From 3ecdeb0f876e91c4a7129ba2ba5baa530aa6c4f9 Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Mon, 29 May 2023 14:13:53 +0800 Subject: swap: remove __swp_swapcount() __swp_swapcount() just encloses the calling to swap_swapcount() with get/put_swap_device(). It is called in __read_swap_cache_async() only, which encloses the calling with get/put_swap_device() already. So, __read_swap_cache_async() can call swap_swapcount() directly. Link: https://lkml.kernel.org/r/20230529061355.125791-4-ying.huang@intel.com Signed-off-by: "Huang, Ying" Reviewed-by: David Hildenbrand Reviewed-by: Chris Li (Google) Cc: Hugh Dickins Cc: Johannes Weiner Cc: Matthew Wilcox Cc: Michal Hocko Cc: Minchan Kim Cc: Tim Chen Cc: Yang Shi Cc: Yu Zhao Cc: Yosry Ahmed Signed-off-by: Andrew Morton --- include/linux/swap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index b2128df5edea..2ddbfd85f6c7 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -513,7 +513,7 @@ int find_first_swap(dev_t *device); extern unsigned int count_swap_pages(int, int); extern sector_t swapdev_block(int, pgoff_t); extern int __swap_count(swp_entry_t entry); -extern int __swp_swapcount(swp_entry_t entry); +extern int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); extern struct swap_info_struct *page_swap_info(struct page *); extern struct swap_info_struct *swp_swap_info(swp_entry_t entry); @@ -591,7 +591,7 @@ static inline int __swap_count(swp_entry_t entry) return 0; } -static inline int __swp_swapcount(swp_entry_t entry) +static inline int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) { return 0; } -- cgit v1.2.3 From 0d625446d0a451a683a357799912b9e688629707 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 16:58:53 +0200 Subject: backing_dev: remove current->backing_dev_info Patch series "cleanup the filemap / direct I/O interaction", v4. This series cleans up some of the generic write helper calling conventions and the page cache writeback / invalidation for direct I/O. This is a spinoff from the no-bufferhead kernel project, for which we'll want to an use iomap based buffered write path in the block layer. This patch (of 12): The last user of current->backing_dev_info disappeared in commit b9b1335e6403 ("remove bdi_congested() and wb_congested() and related functions"). Remove the field and all assignments to it. Link: https://lkml.kernel.org/r/20230601145904.1385409-1-hch@lst.de Link: https://lkml.kernel.org/r/20230601145904.1385409-2-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Christian Brauner Reviewed-by: Damien Le Moal Reviewed-by: Hannes Reinecke Reviewed-by: Johannes Thumshirn Reviewed-by: Darrick J. Wong Acked-by: Theodore Ts'o Cc: Al Viro Cc: Andreas Gruenbacher Cc: Anna Schumaker Cc: Chao Yu Cc: Ilya Dryomov Cc: Jaegeuk Kim Cc: Jens Axboe Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Miklos Szeredi Cc: Trond Myklebust Cc: Xiubo Li Signed-off-by: Andrew Morton --- include/linux/sched.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index eed5d65b8d1f..54780571fe9a 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -41,7 +41,6 @@ /* task_struct member predeclarations (sorted alphabetically): */ struct audit_context; -struct backing_dev_info; struct bio_list; struct blk_plug; struct bpf_local_storage; @@ -1186,8 +1185,6 @@ struct task_struct { /* VM state: */ struct reclaim_state *reclaim_state; - struct backing_dev_info *backing_dev_info; - struct io_context *io_context; #ifdef CONFIG_COMPACTION -- cgit v1.2.3 From 3c435a0fe35c220bec442dffff04a64aacf952b0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 16:58:56 +0200 Subject: filemap: add a kiocb_write_and_wait helper Factor out a helper that does filemap_write_and_wait_range for the range covered by a read kiocb, or returns -EAGAIN if the kiocb is marked as nowait and there would be pages to write. Link: https://lkml.kernel.org/r/20230601145904.1385409-5-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Hannes Reinecke Acked-by: Darrick J. Wong Cc: Al Viro Cc: Andreas Gruenbacher Cc: Anna Schumaker Cc: Chao Yu Cc: Christian Brauner Cc: Ilya Dryomov Cc: Jaegeuk Kim Cc: Jens Axboe Cc: Johannes Thumshirn Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Miklos Szeredi Cc: Theodore Ts'o Cc: Trond Myklebust Cc: Xiubo Li Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index c1ae5ebc375f..b6a12ca108b7 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -30,6 +30,7 @@ static inline void invalidate_remote_inode(struct inode *inode) int invalidate_inode_pages2(struct address_space *mapping); int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end); + int write_inode_now(struct inode *, int sync); int filemap_fdatawrite(struct address_space *); int filemap_flush(struct address_space *); @@ -54,6 +55,7 @@ int filemap_check_errors(struct address_space *mapping); void __filemap_set_wb_err(struct address_space *mapping, int err); int filemap_fdatawrite_wbc(struct address_space *mapping, struct writeback_control *wbc); +int kiocb_write_and_wait(struct kiocb *iocb, size_t count); static inline int filemap_write_and_wait(struct address_space *mapping) { -- cgit v1.2.3 From e003f74afbd2feadbb9ffbf9135e2d2fb5d320a5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 16:58:57 +0200 Subject: filemap: add a kiocb_invalidate_pages helper Factor out a helper that calls filemap_write_and_wait_range and invalidate_inode_pages2_range for the range covered by a write kiocb or returns -EAGAIN if the kiocb is marked as nowait and there would be pages to write or invalidate. Link: https://lkml.kernel.org/r/20230601145904.1385409-6-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Hannes Reinecke Acked-by: Darrick J. Wong Cc: Al Viro Cc: Andreas Gruenbacher Cc: Anna Schumaker Cc: Chao Yu Cc: Christian Brauner Cc: Ilya Dryomov Cc: Jaegeuk Kim Cc: Jens Axboe Cc: Johannes Thumshirn Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Miklos Szeredi Cc: Theodore Ts'o Cc: Trond Myklebust Cc: Xiubo Li Signed-off-by: Andrew Morton --- include/linux/pagemap.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index b6a12ca108b7..7b66a67dba51 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -30,6 +30,7 @@ static inline void invalidate_remote_inode(struct inode *inode) int invalidate_inode_pages2(struct address_space *mapping); int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end); +int kiocb_invalidate_pages(struct kiocb *iocb, size_t count); int write_inode_now(struct inode *, int sync); int filemap_fdatawrite(struct address_space *); -- cgit v1.2.3 From c402a9a9430b670926decbb284b756ee6f47c1ec Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 16:58:58 +0200 Subject: filemap: add a kiocb_invalidate_post_direct_write helper Add a helper to invalidate page cache after a dio write. Link: https://lkml.kernel.org/r/20230601145904.1385409-7-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Hannes Reinecke Acked-by: Darrick J. Wong Cc: Al Viro Cc: Andreas Gruenbacher Cc: Anna Schumaker Cc: Chao Yu Cc: Christian Brauner Cc: Ilya Dryomov Cc: Jaegeuk Kim Cc: Jens Axboe Cc: Johannes Thumshirn Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Miklos Szeredi Cc: Theodore Ts'o Cc: Trond Myklebust Cc: Xiubo Li Signed-off-by: Andrew Morton --- include/linux/fs.h | 5 ----- include/linux/pagemap.h | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 86b50271b4f7..4f196f827d9d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2843,11 +2843,6 @@ static inline void inode_dio_end(struct inode *inode) wake_up_bit(&inode->i_state, __I_DIO_WAKEUP); } -/* - * Warn about a page cache invalidation failure diring a direct I/O write. - */ -void dio_warn_stale_pagecache(struct file *filp); - extern void inode_set_flags(struct inode *inode, unsigned int flags, unsigned int mask); diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 7b66a67dba51..716953ee1ebd 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -31,6 +31,7 @@ int invalidate_inode_pages2(struct address_space *mapping); int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end); int kiocb_invalidate_pages(struct kiocb *iocb, size_t count); +void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count); int write_inode_now(struct inode *, int sync); int filemap_fdatawrite(struct address_space *); -- cgit v1.2.3 From 44fff0fa08ec5a6d9d5fb05443a36d854d0ece4d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 1 Jun 2023 16:59:01 +0200 Subject: fs: factor out a direct_write_fallback helper Add a helper dealing with handling the syncing of a buffered write fallback for direct I/O. Link: https://lkml.kernel.org/r/20230601145904.1385409-10-hch@lst.de Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Reviewed-by: Miklos Szeredi Reviewed-by: Darrick J. Wong Cc: Al Viro Cc: Andreas Gruenbacher Cc: Anna Schumaker Cc: Chao Yu Cc: Christian Brauner Cc: Hannes Reinecke Cc: Ilya Dryomov Cc: Jaegeuk Kim Cc: Jens Axboe Cc: Johannes Thumshirn Cc: Matthew Wilcox Cc: Miklos Szeredi Cc: Theodore Ts'o Cc: Trond Myklebust Cc: Xiubo Li Signed-off-by: Andrew Morton --- include/linux/fs.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 4f196f827d9d..c363f8687c7e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2744,6 +2744,8 @@ extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *); ssize_t generic_perform_write(struct kiocb *, struct iov_iter *); +ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter, + ssize_t direct_written, ssize_t buffered_written); ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags); -- cgit v1.2.3 From 54cbbbf3faf610fb4eba6f8d39d933bcbfc6f4de Mon Sep 17 00:00:00 2001 From: Lorenzo Stoakes Date: Thu, 4 May 2023 22:27:51 +0100 Subject: mm/mmap: separate writenotify and dirty tracking logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm/gup: disallow GUP writing to file-backed mappings by default", v9. Writing to file-backed mappings which require folio dirty tracking using GUP is a fundamentally broken operation, as kernel write access to GUP mappings do not adhere to the semantics expected by a file system. A GUP caller uses the direct mapping to access the folio, which does not cause write notify to trigger, nor does it enforce that the caller marks the folio dirty. The problem arises when, after an initial write to the folio, writeback results in the folio being cleaned and then the caller, via the GUP interface, writes to the folio again. As a result of the use of this secondary, direct, mapping to the folio no write notify will occur, and if the caller does mark the folio dirty, this will be done so unexpectedly. For example, consider the following scenario:- 1. A folio is written to via GUP which write-faults the memory, notifying the file system and dirtying the folio. 2. Later, writeback is triggered, resulting in the folio being cleaned and the PTE being marked read-only. 3. The GUP caller writes to the folio, as it is mapped read/write via the direct mapping. 4. The GUP caller, now done with the page, unpins it and sets it dirty (though it does not have to). This change updates both the PUP FOLL_LONGTERM slow and fast APIs. As pin_user_pages_fast_only() does not exist, we can rely on a slightly imperfect whitelisting in the PUP-fast case and fall back to the slow case should this fail. This patch (of 3): vma_wants_writenotify() is specifically intended for setting PTE page table flags, accounting for existing page table flag state and whether the underlying filesystem performs dirty tracking for a file-backed mapping. Everything is predicated firstly on whether the mapping is shared writable, as this is the only instance where dirty tracking is pertinent - MAP_PRIVATE mappings will always be CoW'd and unshared, and read-only file-backed shared mappings cannot be written to, even with FOLL_FORCE. All other checks are in line with existing logic, though now separated into checks eplicitily for dirty tracking and those for determining how to set page table flags. We make this change so we can perform checks in the GUP logic to determine which mappings might be problematic when written to. Link: https://lkml.kernel.org/r/cover.1683235180.git.lstoakes@gmail.com Link: https://lkml.kernel.org/r/0f218370bd49b4e6bbfbb499f7c7b92c26ba1ceb.1683235180.git.lstoakes@gmail.com Signed-off-by: Lorenzo Stoakes Reviewed-by: John Hubbard Reviewed-by: Mika Penttilä Reviewed-by: Jan Kara Reviewed-by: Jason Gunthorpe Acked-by: David Hildenbrand Cc: Kirill A . Shutemov Cc: Peter Zijlstra Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 62bb3272e531..66032f0d515c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2461,6 +2461,7 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma, #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE) +bool vma_needs_dirty_tracking(struct vm_area_struct *vma); int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot); static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) { -- cgit v1.2.3 From 3db55767da7427cebc49e22b25b5f50ff455add6 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Fri, 12 May 2023 13:33:38 +0300 Subject: add intptr_t Add signed intptr_t given that a) it is standard type and b) uintptr_t is in tree. Link: https://lkml.kernel.org/r/ed66b9e4-1fb7-45be-9bb9-d4bc291c691f@p183 Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton --- include/linux/types.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/types.h b/include/linux/types.h index 688fb943556a..7e5a1fb7cfa1 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -35,6 +35,7 @@ typedef __kernel_uid16_t uid16_t; typedef __kernel_gid16_t gid16_t; typedef unsigned long uintptr_t; +typedef long intptr_t; #ifdef CONFIG_HAVE_UID16 /* This is defined by include/asm-{arch}/posix_types.h */ -- cgit v1.2.3 From 6ca0f81c0b96a5e29de48cb02062b5130d27dbe3 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:49 +0200 Subject: mm: percpu: unhide pcpu_embed_first_chunk prototype Patch series "mm/init/kernel: missing-prototypes warnings". These are patches addressing -Wmissing-prototypes warnings in common kernel code and memory management code files that usually get merged through the -mm tree. This patch (of 12): This function is called whenever CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK or CONFIG_HAVE_SETUP_PER_CPU_AREA, but only declared when the former is set: mm/percpu.c:3055:12: error: no previous prototype for 'pcpu_embed_first_chunk' [-Werror=missing-prototypes] There is no real point in hiding declarations, so just remove the #ifdef here. Link: https://lkml.kernel.org/r/20230517131102.934196-1-arnd@kernel.org Link: https://lkml.kernel.org/r/20230517131102.934196-2-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/percpu.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 1338ea2aa720..42125cf9c506 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -103,12 +103,10 @@ extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai); extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, void *base_addr); -#ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); -#endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK void __init pcpu_populate_pte(unsigned long addr); -- cgit v1.2.3 From 8f14a96386b2676a1ccdd9d2f1732fbd7248fa98 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:50 +0200 Subject: mm: page_poison: always declare __kernel_map_pages() function The __kernel_map_pages() function is mainly used for CONFIG_DEBUG_PAGEALLOC, but has a number of architecture specific definitions that may also be used in other configurations, as well as a global fallback definition for architectures that do not support DEBUG_PAGEALLOC. When the option is disabled, any definitions without the prototype cause a warning: mm/page_poison.c:102:6: error: no previous prototype for '__kernel_map_pages' [-Werror=missing-prototypes] The function is a trivial nop here, so just declare it anyway to avoid the warning. Link: https://lkml.kernel.org/r/20230517131102.934196-3-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 27ce77080c79..e95d7c575ea6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3453,13 +3453,12 @@ static inline bool debug_pagealloc_enabled_static(void) return static_branch_unlikely(&_debug_pagealloc_enabled); } -#ifdef CONFIG_DEBUG_PAGEALLOC /* * To support DEBUG_PAGEALLOC architecture must ensure that * __kernel_map_pages() never fails */ extern void __kernel_map_pages(struct page *page, int numpages, int enable); - +#ifdef CONFIG_DEBUG_PAGEALLOC static inline void debug_pagealloc_map_pages(struct page *page, int numpages) { if (debug_pagealloc_enabled_static()) -- cgit v1.2.3 From d9cdb43189ef9aa4f8a12b00e86875544942fa6a Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:56 +0200 Subject: panic: make function declarations visible A few panic() related functions have a global definition but not declaration, which causes a warning with W=1: kernel/panic.c:710:6: error: no previous prototype for '__warn_printk' [-Werror=missing-prototypes] kernel/panic.c:756:24: error: no previous prototype for '__stack_chk_fail' [-Werror=missing-prototypes] kernel/exit.c:1917:32: error: no previous prototype for 'abort' [-Werror=missing-prototypes] __warn_printk() is called both as a global function when CONFIG_BUG is enabled, and as a local function in other configs. The other two here are called indirectly from generated or assembler code. Add prototypes for all of these. Link: https://lkml.kernel.org/r/20230517131102.934196-9-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/panic.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/panic.h b/include/linux/panic.h index 979b776e3bcb..6717b15e798c 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -32,6 +32,9 @@ extern int sysctl_panic_on_stackoverflow; extern bool crash_kexec_post_notifiers; +extern void __stack_chk_fail(void); +void abort(void); + /* * panic_cpu is used for synchronizing panic() and crash_kexec() execution. It * holds a CPU number which is executing panic() currently. A value of -- cgit v1.2.3 From ad1a48301f659a02df5bff0a121d4a5c0411d36b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:10:59 +0200 Subject: init: consolidate prototypes in linux/init.h The init/main.c file contains some extern declarations for functions defined in architecture code, and it defines some other functions that are called from architecture code with a custom prototype. Both of those result in warnings with 'make W=1': init/calibrate.c:261:37: error: no previous prototype for 'calibrate_delay_is_known' [-Werror=missing-prototypes] init/main.c:790:20: error: no previous prototype for 'mem_encrypt_init' [-Werror=missing-prototypes] init/main.c:792:20: error: no previous prototype for 'poking_init' [-Werror=missing-prototypes] arch/arm64/kernel/irq.c:122:13: error: no previous prototype for 'init_IRQ' [-Werror=missing-prototypes] arch/arm64/kernel/time.c:55:13: error: no previous prototype for 'time_init' [-Werror=missing-prototypes] arch/x86/kernel/process.c:935:13: error: no previous prototype for 'arch_post_acpi_subsys_init' [-Werror=missing-prototypes] init/calibrate.c:261:37: error: no previous prototype for 'calibrate_delay_is_known' [-Werror=missing-prototypes] kernel/fork.c:991:20: error: no previous prototype for 'arch_task_cache_init' [-Werror=missing-prototypes] Add prototypes for all of these in include/linux/init.h or another appropriate header, and remove the duplicate declarations from architecture specific code. [sfr@canb.auug.org.au: declare time_init_early()] Link: https://lkml.kernel.org/r/20230519124311.5167221c@canb.auug.org.au Link: https://lkml.kernel.org/r/20230517131102.934196-12-arnd@kernel.org Signed-off-by: Arnd Bergmann Signed-off-by: Stephen Rothwell Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/acpi.h | 3 ++- include/linux/delay.h | 1 + include/linux/init.h | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7b71dd74baeb..f4c2a87d02c1 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -712,7 +712,6 @@ int acpi_match_platform_list(const struct acpi_platform_list *plat); extern void acpi_early_init(void); extern void acpi_subsystem_init(void); -extern void arch_post_acpi_subsys_init(void); extern int acpi_nvs_register(__u64 start, __u64 size); @@ -1084,6 +1083,8 @@ static inline bool acpi_sleep_state_supported(u8 sleep_state) #endif /* !CONFIG_ACPI */ +extern void arch_post_acpi_subsys_init(void); + #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC int acpi_ioapic_add(acpi_handle root); #else diff --git a/include/linux/delay.h b/include/linux/delay.h index 039e7e0c7378..ff9cda975e30 100644 --- a/include/linux/delay.h +++ b/include/linux/delay.h @@ -56,6 +56,7 @@ static inline void ndelay(unsigned long x) extern unsigned long lpj_fine; void calibrate_delay(void); +unsigned long calibrate_delay_is_known(void); void __attribute__((weak)) calibration_delay_done(void); void msleep(unsigned int msecs); unsigned long msleep_interruptible(unsigned int msecs); diff --git a/include/linux/init.h b/include/linux/init.h index c5fe6d26f5b1..1200fa99e848 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -152,6 +152,24 @@ extern unsigned int reset_devices; void setup_arch(char **); void prepare_namespace(void); void __init init_rootfs(void); + +void init_IRQ(void); +void time_init(void); +void mem_encrypt_init(void); +void poking_init(void); +void pgtable_cache_init(void); + +extern initcall_entry_t __initcall_start[]; +extern initcall_entry_t __initcall0_start[]; +extern initcall_entry_t __initcall1_start[]; +extern initcall_entry_t __initcall2_start[]; +extern initcall_entry_t __initcall3_start[]; +extern initcall_entry_t __initcall4_start[]; +extern initcall_entry_t __initcall5_start[]; +extern initcall_entry_t __initcall6_start[]; +extern initcall_entry_t __initcall7_start[]; +extern initcall_entry_t __initcall_end[]; + extern struct file_system_type rootfs_fs_type; #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX) @@ -309,6 +327,8 @@ struct obs_kernel_param { int early; }; +extern const struct obs_kernel_param __setup_start[], __setup_end[]; + /* * Only for really core code. See moduleparam.h for the normal way. * -- cgit v1.2.3 From 73648e6fa79a832a6a50e87d4a9e9da416f82aaa Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:11:00 +0200 Subject: init: move cifs_root_data() prototype into linux/mount.h cifs_root_data() is defined in cifs and called from early init code, but lacks a global prototype: fs/cifs/cifsroot.c:83:12: error: no previous prototype for 'cifs_root_data' Move the declaration from do_mounts.c into an appropriate header. Link: https://lkml.kernel.org/r/20230517131102.934196-13-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/mount.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mount.h b/include/linux/mount.h index 1ea326c368f7..f381eb44b24c 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -124,4 +124,6 @@ extern int iterate_mounts(int (*)(struct vfsmount *, void *), void *, struct vfsmount *); extern void kern_unmount_array(struct vfsmount *mnt[], unsigned int num); +extern int cifs_root_data(char **dev, char **opts); + #endif /* _LINUX_MOUNT_H */ -- cgit v1.2.3 From af0a76e1269516d940214be48255669b0b5ff40b Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:11:01 +0200 Subject: thread_info: move function declarations to linux/thread_info.h There are a few __weak functions in kernel/fork.c, which architectures can override. If there is no prototype, the compiler warns about them: kernel/fork.c:164:13: error: no previous prototype for 'arch_release_task_struct' [-Werror=missing-prototypes] kernel/fork.c:991:20: error: no previous prototype for 'arch_task_cache_init' [-Werror=missing-prototypes] kernel/fork.c:1086:12: error: no previous prototype for 'arch_dup_task_struct' [-Werror=missing-prototypes] There are already prototypes in a number of architecture specific headers that have addressed those warnings before, but it's much better to have these in a single place so the warning no longer shows up anywhere. Link: https://lkml.kernel.org/r/20230517131102.934196-14-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/thread_info.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index c02646884fa8..9ea0b28068f4 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -256,6 +256,11 @@ check_copy_size(const void *addr, size_t bytes, bool is_source) static inline void arch_setup_new_exec(void) { } #endif +void arch_task_cache_init(void); /* for CONFIG_SH */ +void arch_release_task_struct(struct task_struct *tsk); +int arch_dup_task_struct(struct task_struct *dst, + struct task_struct *src); + #endif /* __KERNEL__ */ #endif /* _LINUX_THREAD_INFO_H */ -- cgit v1.2.3 From 3403bb4ea5983b4e84f404f91322a9a0cc75d700 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 15:11:02 +0200 Subject: time_namespace: always provide arch_get_vdso_data() prototype for vdso The arch_get_vdso_data() function is defined separately on each architecture, but only called when CONFIG_TIME_NS is set. If the definition is a global function, this causes a W=1 warning without TIME_NS: arch/x86/entry/vdso/vma.c:35:19: error: no previous prototype for 'arch_get_vdso_data' [-Werror=missing-prototypes] Move the prototype out of the #ifdef block to reliably turn off that warning. Link: https://lkml.kernel.org/r/20230517131102.934196-15-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Boqun Feng Cc: Catalin Marinas Cc: Christoph Lameter Cc: Dennis Zhou Cc: Eric Paris Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Michael Ellerman Cc: Michal Simek Cc: Palmer Dabbelt Cc: Paul Moore Cc: Pavel Machek Cc: Peter Zijlstra Cc: Rafael J. Wysocki Cc: Russell King Cc: Tejun Heo Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Waiman Long Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/time_namespace.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index bb9d3f5542f8..03d9c5ac01d1 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -44,7 +44,6 @@ struct time_namespace *copy_time_ns(unsigned long flags, struct time_namespace *old_ns); void free_time_ns(struct time_namespace *ns); void timens_on_fork(struct nsproxy *nsproxy, struct task_struct *tsk); -struct vdso_data *arch_get_vdso_data(void *vvar_page); struct page *find_timens_vvar_page(struct vm_area_struct *vma); static inline void put_time_ns(struct time_namespace *ns) @@ -163,4 +162,6 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim) } #endif +struct vdso_data *arch_get_vdso_data(void *vvar_page); + #endif /* _LINUX_TIMENS_H */ -- cgit v1.2.3 From e0ddec73fd4822d2ffe914d5ce3e2718f985276a Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 17 May 2023 14:49:25 +0200 Subject: kcov: add prototypes for helper functions A number of internal functions in kcov are only called from generated code and don't technically need a declaration, but 'make W=1' warns about global symbols without a prototype: kernel/kcov.c:199:14: error: no previous prototype for '__sanitizer_cov_trace_pc' [-Werror=missing-prototypes] kernel/kcov.c:264:14: error: no previous prototype for '__sanitizer_cov_trace_cmp1' [-Werror=missing-prototypes] kernel/kcov.c:270:14: error: no previous prototype for '__sanitizer_cov_trace_cmp2' [-Werror=missing-prototypes] kernel/kcov.c:276:14: error: no previous prototype for '__sanitizer_cov_trace_cmp4' [-Werror=missing-prototypes] kernel/kcov.c:282:14: error: no previous prototype for '__sanitizer_cov_trace_cmp8' [-Werror=missing-prototypes] kernel/kcov.c:288:14: error: no previous prototype for '__sanitizer_cov_trace_const_cmp1' [-Werror=missing-prototypes] kernel/kcov.c:295:14: error: no previous prototype for '__sanitizer_cov_trace_const_cmp2' [-Werror=missing-prototypes] kernel/kcov.c:302:14: error: no previous prototype for '__sanitizer_cov_trace_const_cmp4' [-Werror=missing-prototypes] kernel/kcov.c:309:14: error: no previous prototype for '__sanitizer_cov_trace_const_cmp8' [-Werror=missing-prototypes] kernel/kcov.c:316:14: error: no previous prototype for '__sanitizer_cov_trace_switch' [-Werror=missing-prototypes] Adding prototypes for these in a header solves that problem, but now there is a mismatch between the built-in type and the prototype on 64-bit architectures because they expect some functions to take a 64-bit 'unsigned long' argument rather than an 'unsigned long long' u64 type: include/linux/kcov.h:84:6: error: conflicting types for built-in function '__sanitizer_cov_trace_switch'; expected 'void(long long unsigned int, void *)' [-Werror=builtin-declaration-mismatch] 84 | void __sanitizer_cov_trace_switch(u64 val, u64 *cases); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~ Avoid this as well with a custom type definition. Link: https://lkml.kernel.org/r/20230517124944.929997-1-arnd@kernel.org Signed-off-by: Arnd Bergmann Cc: Andrey Konovalov Cc: Dmitry Vyukov Cc: Rong Tao Signed-off-by: Andrew Morton --- include/linux/kcov.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/kcov.h b/include/linux/kcov.h index ee04256f28af..b851ba415e03 100644 --- a/include/linux/kcov.h +++ b/include/linux/kcov.h @@ -72,6 +72,23 @@ static inline void kcov_remote_stop_softirq(void) kcov_remote_stop(); } +#ifdef CONFIG_64BIT +typedef unsigned long kcov_u64; +#else +typedef unsigned long long kcov_u64; +#endif + +void __sanitizer_cov_trace_pc(void); +void __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2); +void __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2); +void __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2); +void __sanitizer_cov_trace_cmp8(kcov_u64 arg1, kcov_u64 arg2); +void __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2); +void __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2); +void __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2); +void __sanitizer_cov_trace_const_cmp8(kcov_u64 arg1, kcov_u64 arg2); +void __sanitizer_cov_trace_switch(kcov_u64 val, void *cases); + #else static inline void kcov_task_init(struct task_struct *t) {} -- cgit v1.2.3 From 5e008df11c55228a86a1bae692cc2002503572c9 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:25 -0700 Subject: watchdog/perf: define dummy watchdog_update_hrtimer_threshold() on correct config Patch series "watchdog/hardlockup: Add the buddy hardlockup detector", v5. This patch series adds the "buddy" hardlockup detector. In brief, the buddy hardlockup detector can detect hardlockups without arch-level support by having CPUs checkup on a "buddy" CPU periodically. Given the new design of this patch series, testing all combinations is fairly difficult. I've attempted to make sure that all combinations of CONFIG_ options are good, but it wouldn't surprise me if I missed something. I apologize in advance and I'll do my best to fix any problems that are found. This patch (of 18): The real watchdog_update_hrtimer_threshold() is defined in kernel/watchdog_hld.c. That file is included if CONFIG_HARDLOCKUP_DETECTOR_PERF and the function is defined in that file if CONFIG_HARDLOCKUP_CHECK_TIMESTAMP. The dummy version of the function in "nmi.h" didn't get that quite right. While this doesn't appear to be a huge deal, it's nice to make it consistent. It doesn't break builds because CHECK_TIMESTAMP is only defined by x86 so others don't get a double definition, and x86 uses perf lockup detector, so it gets the out of line version. Link: https://lkml.kernel.org/r/20230519101840.v5.18.Ia44852044cdcb074f387e80df6b45e892965d4a1@changeid Link: https://lkml.kernel.org/r/20230519101840.v5.1.I8cbb2f4fa740528fcfade4f5439b6cdcdd059251@changeid Fixes: 7edaeb6841df ("kernel/watchdog: Prevent false positives with turbo modes") Signed-off-by: Douglas Anderson Reviewed-by: Nicholas Piggin Reviewed-by: Petr Mladek Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Cc: Colin Cross Signed-off-by: Andrew Morton --- include/linux/nmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 048c0b9aa623..771d77b62bc1 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -197,7 +197,7 @@ u64 hw_nmi_get_sample_period(int watchdog_thresh); #endif #if defined(CONFIG_HARDLOCKUP_CHECK_TIMESTAMP) && \ - defined(CONFIG_HARDLOCKUP_DETECTOR) + defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) void watchdog_update_hrtimer_threshold(u64 period); #else static inline void watchdog_update_hrtimer_threshold(u64 period) { } -- cgit v1.2.3 From 730211182ed083898fa5feb4b28459ffac4c9615 Mon Sep 17 00:00:00 2001 From: Lecopzer Chen Date: Fri, 19 May 2023 10:18:28 -0700 Subject: watchdog/hardlockup: change watchdog_nmi_enable() to void Nobody cares about the return value of watchdog_nmi_enable(), changing its prototype to void. Link: https://lkml.kernel.org/r/20230519101840.v5.4.Ic3a19b592eb1ac4c6f6eade44ffd943e8637b6e5@changeid Signed-off-by: Pingfan Liu Signed-off-by: Lecopzer Chen Signed-off-by: Douglas Anderson Reviewed-by: Petr Mladek Acked-by: Nicholas Piggin Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 771d77b62bc1..454fe99c4874 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -119,7 +119,7 @@ static inline int hardlockup_detector_perf_init(void) { return 0; } void watchdog_nmi_stop(void); void watchdog_nmi_start(void); int watchdog_nmi_probe(void); -int watchdog_nmi_enable(unsigned int cpu); +void watchdog_nmi_enable(unsigned int cpu); void watchdog_nmi_disable(unsigned int cpu); void lockup_detector_reconfigure(void); -- cgit v1.2.3 From 8b5c59a92b5b37e5c65ec185810d67e3f30f5a2e Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:30 -0700 Subject: watchdog/hardlockup: add comments to touch_nmi_watchdog() In preparation for the buddy hardlockup detector, add comments to touch_nmi_watchdog() to make it obvious that it touches the configured hardlockup detector regardless of whether it's backed by an NMI. Also note that arch_touch_nmi_watchdog() may not be architecture-specific. Ideally, we'd like to rename these functions but that is a fairly disruptive change touching a lot of drivers. After discussion [1] the plan is to defer this until a good time. [1] https://lore.kernel.org/r/ZFy0TX1tfhlH8gxj@alley [akpm@linux-foundation.org: comment changes, per Petr] Link: https://lkml.kernel.org/r/ZGyONWPXpE1DcxA5@alley Link: https://lkml.kernel.org/r/20230519101840.v5.6.I4e47cbfa1bb2ebbcdb5ca16817aa2887f15dc82c@changeid Signed-off-by: Douglas Anderson Reviewed-by: Petr Mladek Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 454fe99c4874..dd75e361a11f 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -125,15 +125,30 @@ void watchdog_nmi_disable(unsigned int cpu); void lockup_detector_reconfigure(void); /** - * touch_nmi_watchdog - restart NMI watchdog timeout. + * touch_nmi_watchdog - manually reset the hardlockup watchdog timeout. * - * If the architecture supports the NMI watchdog, touch_nmi_watchdog() - * may be used to reset the timeout - for code which intentionally - * disables interrupts for a long time. This call is stateless. + * If we support detecting hardlockups, touch_nmi_watchdog() may be + * used to pet the watchdog (reset the timeout) - for code which + * intentionally disables interrupts for a long time. This call is stateless. + * + * Though this function has "nmi" in the name, the hardlockup watchdog might + * not be backed by NMIs. This function will likely be renamed to + * touch_hardlockup_watchdog() in the future. */ static inline void touch_nmi_watchdog(void) { + /* + * Pass on to the hardlockup detector selected via CONFIG_. Note that + * the hardlockup detector may not be arch-specific nor using NMIs + * and the arch_touch_nmi_watchdog() function will likely be renamed + * in the future. + */ arch_touch_nmi_watchdog(); + + /* + * Touching the hardlock detector implicitly resets the + * softlockup detector too + */ touch_softlockup_watchdog(); } -- cgit v1.2.3 From 81972551df9d168a8183b786ff4de06008469c2e Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:32 -0700 Subject: watchdog/hardlockup: move perf hardlockup checking/panic to common watchdog.c The perf hardlockup detector works by looking at interrupt counts and seeing if they change from run to run. The interrupt counts are managed by the common watchdog code via its watchdog_timer_fn(). Currently the API between the perf detector and the common code is a function: is_hardlockup(). When the hard lockup detector sees that function return true then it handles printing out debug info and inducing a panic if necessary. Let's change the API a little bit in preparation for the buddy hardlockup detector. The buddy hardlockup detector wants to print nearly the same debug info and have nearly the same panic behavior. That means we want to move all that code to the common file. For now, the code in the common file will only be there if the perf hardlockup detector is enabled, but eventually it will be selected by a common config. Right now, this _just_ moves the code from the perf detector file to the common file and changes the names. It doesn't make the changes that the buddy hardlockup detector will need and doesn't do any style cleanups. A future patch will do cleanup to make it more obvious what changed. With the above, we no longer have any callers of is_hardlockup() outside of the "watchdog.c" file, so we can remove it from the header, make it static, and move it to the same "#ifdef" block as our new watchdog_hardlockup_check(). While doing this, it can be noted that even if no hardlockup detectors were configured the existing code used to still have the code for counting/checking "hrtimer_interrupts" even if the perf hardlockup detector wasn't configured. We didn't need to do that, so move all the "hrtimer_interrupts" counting to only be there if the perf hardlockup detector is configured as well. This change is expected to be a no-op. Link: https://lkml.kernel.org/r/20230519101840.v5.8.Id4133d3183e798122dc3b6205e7852601f289071@changeid Signed-off-by: Douglas Anderson Reviewed-by: Petr Mladek Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index dd75e361a11f..67921436974d 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -15,7 +15,6 @@ void lockup_detector_init(void); void lockup_detector_soft_poweroff(void); void lockup_detector_cleanup(void); -bool is_hardlockup(void); extern int watchdog_user_enabled; extern int nmi_watchdog_user_enabled; @@ -88,6 +87,10 @@ extern unsigned int hardlockup_panic; static inline void hardlockup_detector_disable(void) {} #endif +#if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) +void watchdog_hardlockup_check(struct pt_regs *regs); +#endif + #if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) # define NMI_WATCHDOG_SYSCTL_PERM 0644 #else -- cgit v1.2.3 From 77c12fc95980d100fdc49e88a5727c242d0dfedc Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:34 -0700 Subject: watchdog/hardlockup: add a "cpu" param to watchdog_hardlockup_check() In preparation for the buddy hardlockup detector where the CPU checking for lockup might not be the currently running CPU, add a "cpu" parameter to watchdog_hardlockup_check(). As part of this change, make hrtimer_interrupts an atomic_t since now the CPU incrementing the value and the CPU reading the value might be different. Technially this could also be done with just READ_ONCE and WRITE_ONCE, but atomic_t feels a little cleaner in this case. While hrtimer_interrupts is made atomic_t, we change hrtimer_interrupts_saved from "unsigned long" to "int". The "int" is needed to match the data type backing atomic_t for hrtimer_interrupts. Even if this changes us from 64-bits to 32-bits (which I don't think is true for most compilers), it doesn't really matter. All we ever do is increment it every few seconds and compare it to an old value so 32-bits is fine (even 16-bits would be). The "signed" vs "unsigned" also doesn't matter for simple equality comparisons. hrtimer_interrupts_saved is _not_ switched to atomic_t nor even accessed with READ_ONCE / WRITE_ONCE. The hrtimer_interrupts_saved is always consistently accessed with the same CPU. NOTE: with the upcoming "buddy" detector there is one special case. When a CPU goes offline/online then we can change which CPU is the one to consistently access a given instance of hrtimer_interrupts_saved. We still can't end up with a partially updated hrtimer_interrupts_saved, however, because we end up petting all affected CPUs to make sure the new and old CPU can't end up somehow read/write hrtimer_interrupts_saved at the same time. Link: https://lkml.kernel.org/r/20230519101840.v5.10.I3a7d4dd8c23ac30ee0b607d77feb6646b64825c0@changeid Signed-off-by: Douglas Anderson Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 67921436974d..521bf2ee6eff 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -88,7 +88,7 @@ static inline void hardlockup_detector_disable(void) {} #endif #if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) -void watchdog_hardlockup_check(struct pt_regs *regs); +void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); #endif #if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) -- cgit v1.2.3 From ed92e1ef52224c7c9c15fba559448396b059c2ee Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:35 -0700 Subject: watchdog/hardlockup: move perf hardlockup watchdog petting to watchdog.c In preparation for the buddy hardlockup detector, which wants the same petting logic as the current perf hardlockup detector, move the code to watchdog.c. While doing this, rename the global variable to match others nearby. As part of this change we have to change the code to account for the fact that the CPU we're running on might be different than the one we're checking. Currently the code in watchdog.c is guarded by CONFIG_HARDLOCKUP_DETECTOR_PERF, which makes this change seem silly. However, a future patch will change this. Link: https://lkml.kernel.org/r/20230519101840.v5.11.I00dfd6386ee00da25bf26d140559a41339b53e57@changeid Signed-off-by: Douglas Anderson Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 521bf2ee6eff..56fdc3de6894 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -88,7 +88,10 @@ static inline void hardlockup_detector_disable(void) {} #endif #if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) +void arch_touch_nmi_watchdog(void); void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); +#elif !defined(CONFIG_HAVE_NMI_WATCHDOG) +static inline void arch_touch_nmi_watchdog(void) { } #endif #if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) @@ -98,7 +101,6 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); #endif #if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) -extern void arch_touch_nmi_watchdog(void); extern void hardlockup_detector_perf_stop(void); extern void hardlockup_detector_perf_restart(void); extern void hardlockup_detector_perf_disable(void); @@ -113,7 +115,6 @@ static inline void hardlockup_detector_perf_enable(void) { } static inline void hardlockup_detector_perf_cleanup(void) { } # if !defined(CONFIG_HAVE_NMI_WATCHDOG) static inline int hardlockup_detector_perf_init(void) { return -ENODEV; } -static inline void arch_touch_nmi_watchdog(void) {} # else static inline int hardlockup_detector_perf_init(void) { return 0; } # endif -- cgit v1.2.3 From df95d3085caa5b99a60eb033d7ad6c2ff2b43dbf Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:36 -0700 Subject: watchdog/hardlockup: rename some "NMI watchdog" constants/function Do a search and replace of: - NMI_WATCHDOG_ENABLED => WATCHDOG_HARDLOCKUP_ENABLED - SOFT_WATCHDOG_ENABLED => WATCHDOG_SOFTOCKUP_ENABLED - watchdog_nmi_ => watchdog_hardlockup_ - nmi_watchdog_available => watchdog_hardlockup_available - nmi_watchdog_user_enabled => watchdog_hardlockup_user_enabled - soft_watchdog_user_enabled => watchdog_softlockup_user_enabled - NMI_WATCHDOG_DEFAULT => WATCHDOG_HARDLOCKUP_DEFAULT Then update a few comments near where names were changed. This is specifically to make it less confusing when we want to introduce the buddy hardlockup detector, which isn't using NMIs. As part of this, we sanitized a few names for consistency. [trix@redhat.com: make variables static] Link: https://lkml.kernel.org/r/20230525162822.1.I0fb41d138d158c9230573eaa37dc56afa2fb14ee@changeid Link: https://lkml.kernel.org/r/20230519101840.v5.12.I91f7277bab4bf8c0cb238732ed92e7ce7bbd71a6@changeid Signed-off-by: Douglas Anderson Signed-off-by: Tom Rix Reviewed-by: Tom Rix Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 56fdc3de6894..17b2ae7103d9 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -17,8 +17,6 @@ void lockup_detector_soft_poweroff(void); void lockup_detector_cleanup(void); extern int watchdog_user_enabled; -extern int nmi_watchdog_user_enabled; -extern int soft_watchdog_user_enabled; extern int watchdog_thresh; extern unsigned long watchdog_enabled; @@ -68,17 +66,17 @@ static inline void reset_hung_task_detector(void) { } * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit - * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector. * - * 'watchdog_user_enabled', 'nmi_watchdog_user_enabled' and - * 'soft_watchdog_user_enabled' are variables that are only used as an + * 'watchdog_user_enabled', 'watchdog_hardlockup_user_enabled' and + * 'watchdog_softlockup_user_enabled' are variables that are only used as an * 'interface' between the parameters in /proc/sys/kernel and the internal * state bits in 'watchdog_enabled'. The 'watchdog_thresh' variable is * handled differently because its value is not boolean, and the lockup * detectors are 'suspended' while 'watchdog_thresh' is equal zero. */ -#define NMI_WATCHDOG_ENABLED_BIT 0 -#define SOFT_WATCHDOG_ENABLED_BIT 1 -#define NMI_WATCHDOG_ENABLED (1 << NMI_WATCHDOG_ENABLED_BIT) -#define SOFT_WATCHDOG_ENABLED (1 << SOFT_WATCHDOG_ENABLED_BIT) +#define WATCHDOG_HARDLOCKUP_ENABLED_BIT 0 +#define WATCHDOG_SOFTOCKUP_ENABLED_BIT 1 +#define WATCHDOG_HARDLOCKUP_ENABLED (1 << WATCHDOG_HARDLOCKUP_ENABLED_BIT) +#define WATCHDOG_SOFTOCKUP_ENABLED (1 << WATCHDOG_SOFTOCKUP_ENABLED_BIT) #if defined(CONFIG_HARDLOCKUP_DETECTOR) extern void hardlockup_detector_disable(void); @@ -120,11 +118,11 @@ static inline int hardlockup_detector_perf_init(void) { return 0; } # endif #endif -void watchdog_nmi_stop(void); -void watchdog_nmi_start(void); -int watchdog_nmi_probe(void); -void watchdog_nmi_enable(unsigned int cpu); -void watchdog_nmi_disable(unsigned int cpu); +void watchdog_hardlockup_stop(void); +void watchdog_hardlockup_start(void); +int watchdog_hardlockup_probe(void); +void watchdog_hardlockup_enable(unsigned int cpu); +void watchdog_hardlockup_disable(unsigned int cpu); void lockup_detector_reconfigure(void); -- cgit v1.2.3 From d9b3629ade8ebffb0075e311409796a56bac8282 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:37 -0700 Subject: watchdog/hardlockup: have the perf hardlockup use __weak functions more cleanly The fact that there watchdog_hardlockup_enable(), watchdog_hardlockup_disable(), and watchdog_hardlockup_probe() are declared __weak means that the configured hardlockup detector can define non-weak versions of those functions if it needs to. Instead of doing this, the perf hardlockup detector hooked itself into the default __weak implementation, which was a bit awkward. Clean this up. From comments, it looks as if the original design was done because the __weak function were expected to implemented by the architecture and not by the configured hardlockup detector. This got awkward when we tried to add the buddy lockup detector which was not arch-specific but wanted to hook into those same functions. This is not expected to have any functional impact. Link: https://lkml.kernel.org/r/20230519101840.v5.13.I847d9ec852449350997ba00401d2462a9cb4302b@changeid Signed-off-by: Douglas Anderson Reviewed-by: Petr Mladek Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 17b2ae7103d9..e23b4dc01b4a 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -101,21 +101,11 @@ static inline void arch_touch_nmi_watchdog(void) { } #if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) extern void hardlockup_detector_perf_stop(void); extern void hardlockup_detector_perf_restart(void); -extern void hardlockup_detector_perf_disable(void); -extern void hardlockup_detector_perf_enable(void); extern void hardlockup_detector_perf_cleanup(void); -extern int hardlockup_detector_perf_init(void); #else static inline void hardlockup_detector_perf_stop(void) { } static inline void hardlockup_detector_perf_restart(void) { } -static inline void hardlockup_detector_perf_disable(void) { } -static inline void hardlockup_detector_perf_enable(void) { } static inline void hardlockup_detector_perf_cleanup(void) { } -# if !defined(CONFIG_HAVE_NMI_WATCHDOG) -static inline int hardlockup_detector_perf_init(void) { return -ENODEV; } -# else -static inline int hardlockup_detector_perf_init(void) { return 0; } -# endif #endif void watchdog_hardlockup_stop(void); -- cgit v1.2.3 From 1f423c905a6b43b493df1b259e6e6267e5624e62 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:38 -0700 Subject: watchdog/hardlockup: detect hard lockups using secondary (buddy) CPUs Implement a hardlockup detector that doesn't doesn't need any extra arch-specific support code to detect lockups. Instead of using something arch-specific we will use the buddy system, where each CPU watches out for another one. Specifically, each CPU will use its softlockup hrtimer to check that the next CPU is processing hrtimer interrupts by verifying that a counter is increasing. NOTE: unlike the other hard lockup detectors, the buddy one can't easily show what's happening on the CPU that locked up just by doing a simple backtrace. It relies on some other mechanism in the system to get information about the locked up CPUs. This could be support for NMI backtraces like [1], it could be a mechanism for printing the PC of locked CPUs at panic time like [2] / [3], or it could be something else. Even though that means we still rely on arch-specific code, this arch-specific code seems to often be implemented even on architectures that don't have a hardlockup detector. This style of hardlockup detector originated in some downstream Android trees and has been rebased on / carried in ChromeOS trees for quite a long time for use on arm and arm64 boards. Historically on these boards we've leveraged mechanism [2] / [3] to get information about hung CPUs, but we could move to [1]. Although the original motivation for the buddy system was for use on systems without an arch-specific hardlockup detector, it can still be useful to use even on systems that _do_ have an arch-specific hardlockup detector. On x86, for instance, there is a 24-part patch series [4] in progress switching the arch-specific hard lockup detector from a scarce perf counter to a less-scarce hardware resource. Potentially the buddy system could be a simpler alternative to free up the perf counter but still get hard lockup detection. Overall, pros (+) and cons (-) of the buddy system compared to an arch-specific hardlockup detector (which might be implemented using perf): + The buddy system is usable on systems that don't have an arch-specific hardlockup detector, like arm32 and arm64 (though it's being worked on for arm64 [5]). + The buddy system may free up scarce hardware resources. + If a CPU totally goes out to lunch (can't process NMIs) the buddy system could still detect the problem (though it would be unlikely to be able to get a stack trace). + The buddy system uses the same timer function to pet the hardlockup detector on the running CPU as it uses to detect hardlockups on other CPUs. Compared to other hardlockup detectors, this means it generates fewer interrupts and thus is likely better able to let CPUs stay idle longer. - If all CPUs are hard locked up at the same time the buddy system can't detect it. - If we don't have SMP we can't use the buddy system. - The buddy system needs an arch-specific mechanism (possibly NMI backtrace) to get info about the locked up CPU. [1] https://lore.kernel.org/r/20230419225604.21204-1-dianders@chromium.org [2] https://issuetracker.google.com/172213129 [3] https://docs.kernel.org/trace/coresight/coresight-cpu-debug.html [4] https://lore.kernel.org/lkml/20230301234753.28582-1-ricardo.neri-calderon@linux.intel.com/ [5] https://lore.kernel.org/linux-arm-kernel/20220903093415.15850-1-lecopzer.chen@mediatek.com/ Link: https://lkml.kernel.org/r/20230519101840.v5.14.I6bf789d21d0c3d75d382e7e51a804a7a51315f2c@changeid Signed-off-by: Colin Cross Signed-off-by: Matthias Kaehlcke Signed-off-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Signed-off-by: Douglas Anderson Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Daniel Thompson Cc: "David S. Miller" Cc: Ian Rogers Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index e23b4dc01b4a..1cdadc6a6cfd 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -85,8 +85,9 @@ extern unsigned int hardlockup_panic; static inline void hardlockup_detector_disable(void) {} #endif -#if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) +#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER) void arch_touch_nmi_watchdog(void); +void watchdog_hardlockup_touch_cpu(unsigned int cpu); void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); #elif !defined(CONFIG_HAVE_NMI_WATCHDOG) static inline void arch_touch_nmi_watchdog(void) { } @@ -116,6 +117,12 @@ void watchdog_hardlockup_disable(unsigned int cpu); void lockup_detector_reconfigure(void); +#ifdef CONFIG_HARDLOCKUP_DETECTOR_BUDDY +void watchdog_buddy_check_hardlockup(unsigned long hrtimer_interrupts); +#else +static inline void watchdog_buddy_check_hardlockup(unsigned long hrtimer_interrupts) {} +#endif + /** * touch_nmi_watchdog - manually reset the hardlockup watchdog timeout. * -- cgit v1.2.3 From b17aa959330e8058452297049a0056ba4b9c72e8 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:39 -0700 Subject: watchdog/perf: add a weak function for an arch to detect if perf can use NMIs On arm64, NMI support needs to be detected at runtime. Add a weak function to the perf hardlockup detector so that an architecture can implement it to detect whether NMIs are available. Link: https://lkml.kernel.org/r/20230519101840.v5.15.Ic55cb6f90ef5967d8aaa2b503a4e67c753f64d3a@changeid Signed-off-by: Douglas Anderson Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Lecopzer Chen Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Pingfan Liu Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 1cdadc6a6cfd..28e65fd1de13 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -208,6 +208,7 @@ static inline bool trigger_single_cpu_backtrace(int cpu) #ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF u64 hw_nmi_get_sample_period(int watchdog_thresh); +bool arch_perf_nmi_is_available(void); #endif #if defined(CONFIG_HARDLOCKUP_CHECK_TIMESTAMP) && \ -- cgit v1.2.3 From 930d8f8dbab97cb05dba30e67a2dfa0c6dbf4bc7 Mon Sep 17 00:00:00 2001 From: Lecopzer Chen Date: Fri, 19 May 2023 10:18:40 -0700 Subject: watchdog/perf: adapt the watchdog_perf interface for async model When lockup_detector_init()->watchdog_hardlockup_probe(), PMU may be not ready yet. E.g. on arm64, PMU is not ready until device_initcall(armv8_pmu_driver_init). And it is deeply integrated with the driver model and cpuhp. Hence it is hard to push this initialization before smp_init(). But it is easy to take an opposite approach and try to initialize the watchdog once again later. The delayed probe is called using workqueues. It need to allocate memory and must be proceed in a normal context. The delayed probe is able to use if watchdog_hardlockup_probe() returns non-zero which means the return code returned when PMU is not ready yet. Provide an API - lockup_detector_retry_init() for anyone who needs to delayed init lockup detector if they had ever failed at lockup_detector_init(). The original assumption is: nobody should use delayed probe after lockup_detector_check() which has __init attribute. That is, anyone uses this API must call between lockup_detector_init() and lockup_detector_check(), and the caller must have __init attribute Link: https://lkml.kernel.org/r/20230519101840.v5.16.If4ad5dd5d09fb1309cebf8bcead4b6a5a7758ca7@changeid Reviewed-by: Petr Mladek Co-developed-by: Pingfan Liu Signed-off-by: Pingfan Liu Signed-off-by: Lecopzer Chen Signed-off-by: Douglas Anderson Suggested-by: Petr Mladek Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Sumit Garg Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/nmi.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 28e65fd1de13..83577ae736cc 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -13,6 +13,7 @@ #ifdef CONFIG_LOCKUP_DETECTOR void lockup_detector_init(void); +void lockup_detector_retry_init(void); void lockup_detector_soft_poweroff(void); void lockup_detector_cleanup(void); @@ -32,6 +33,7 @@ extern int sysctl_hardlockup_all_cpu_backtrace; #else /* CONFIG_LOCKUP_DETECTOR */ static inline void lockup_detector_init(void) { } +static inline void lockup_detector_retry_init(void) { } static inline void lockup_detector_soft_poweroff(void) { } static inline void lockup_detector_cleanup(void) { } #endif /* !CONFIG_LOCKUP_DETECTOR */ -- cgit v1.2.3 From d7a0fe9ef6d6484fca4ba55c19091932337d4272 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 19 May 2023 10:18:42 -0700 Subject: arm64: enable perf events based hard lockup detector With the recent feature added to enable perf events to use pseudo NMIs as interrupts on platforms which support GICv3 or later, its now been possible to enable hard lockup detector (or NMI watchdog) on arm64 platforms. So enable corresponding support. One thing to note here is that normally lockup detector is initialized just after the early initcalls but PMU on arm64 comes up much later as device_initcall(). To cope with that, override arch_perf_nmi_is_available() to let the watchdog framework know PMU not ready, and inform the framework to re-initialize lockup detection once PMU has been initialized. [dianders@chromium.org: only HAVE_HARDLOCKUP_DETECTOR_PERF if the PMU config is enabled] Link: https://lkml.kernel.org/r/20230523073952.1.I60217a63acc35621e13f10be16c0cd7c363caf8c@changeid Link: https://lkml.kernel.org/r/20230519101840.v5.18.Ia44852044cdcb074f387e80df6b45e892965d4a1@changeid Co-developed-by: Sumit Garg Signed-off-by: Sumit Garg Co-developed-by: Pingfan Liu Signed-off-by: Pingfan Liu Signed-off-by: Lecopzer Chen Signed-off-by: Douglas Anderson Cc: Andi Kleen Cc: Catalin Marinas Cc: Chen-Yu Tsai Cc: Christophe Leroy Cc: Colin Cross Cc: Daniel Thompson Cc: "David S. Miller" Cc: Guenter Roeck Cc: Ian Rogers Cc: Marc Zyngier Cc: Mark Rutland Cc: Masayoshi Mizuma Cc: Matthias Kaehlcke Cc: Michael Ellerman Cc: Nicholas Piggin Cc: Petr Mladek Cc: Randy Dunlap Cc: "Ravi V. Shankar" Cc: Ricardo Neri Cc: Stephane Eranian Cc: Stephen Boyd Cc: Tzung-Bi Shih Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/perf/arm_pmu.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 525b5d64e394..5b00f5cb4cf9 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -171,6 +171,8 @@ void kvm_host_pmu_init(struct arm_pmu *pmu); #define kvm_host_pmu_init(x) do { } while(0) #endif +bool arm_pmu_irq_is_nmi(void); + /* Internal functions only for core arm_pmu code */ struct arm_pmu *armpmu_alloc(void); void armpmu_free(struct arm_pmu *pmu); -- cgit v1.2.3 From 048a9883267f9b8f8e05dca9e9e8e6f991eea61e Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Sat, 20 May 2023 21:25:19 +0300 Subject: include/linux/math.h: fix mult_frac() multiple argument evaluation bug mult_frac() evaluates _all_ arguments multiple times in the body. Clarify comment while I'm at it. Link: https://lkml.kernel.org/r/f9f9fdbb-ec8e-4f5e-a998-2a58627a1a43@p183 Signed-off-by: Alexey Dobriyan Signed-off-by: Andrew Morton --- include/linux/math.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/math.h b/include/linux/math.h index 439b8f0b9ebd..2d388650c556 100644 --- a/include/linux/math.h +++ b/include/linux/math.h @@ -118,17 +118,17 @@ __STRUCT_FRACT(s32) __STRUCT_FRACT(u32) #undef __STRUCT_FRACT -/* - * Multiplies an integer by a fraction, while avoiding unnecessary - * overflow or loss of precision. - */ -#define mult_frac(x, numer, denom)( \ -{ \ - typeof(x) quot = (x) / (denom); \ - typeof(x) rem = (x) % (denom); \ - (quot * (numer)) + ((rem * (numer)) / (denom)); \ -} \ -) +/* Calculate "x * n / d" without unnecessary overflow or loss of precision. */ +#define mult_frac(x, n, d) \ +({ \ + typeof(x) x_ = (x); \ + typeof(n) n_ = (n); \ + typeof(d) d_ = (d); \ + \ + typeof(x_) q = x_ / d_; \ + typeof(x_) r = x_ % d_; \ + q * n_ + r * n_ / d_; \ +}) #define sector_div(a, b) do_div(a, b) -- cgit v1.2.3 From a94181ec064b3ad1b6f573f5953e2011f8c90292 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 7 Jun 2023 16:28:45 +0200 Subject: syscalls: add sys_ni_posix_timers prototype The sys_ni_posix_timers() definition causes a warning when the declaration is missing, so this needs to be added along with the normal syscalls, outside of the #ifdef. kernel/time/posix-stubs.c:26:17: error: no previous prototype for 'sys_ni_posix_timers' [-Werror=missing-prototypes] Link: https://lkml.kernel.org/r/20230607142925.3126422-1-arnd@kernel.org Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Signed-off-by: Andrew Morton --- include/linux/syscalls.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 33a0ee3bcb2e..24871f8ec8bb 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1280,6 +1280,7 @@ asmlinkage long sys_ni_syscall(void); #endif /* CONFIG_ARCH_HAS_SYSCALL_WRAPPER */ +asmlinkage long sys_ni_posix_timers(void); /* * Kernel code should not call syscalls (i.e., sys_xyzyyz()) directly. -- cgit v1.2.3 From 93b36d0f2892357906f1058778c9188ff857baa1 Mon Sep 17 00:00:00 2001 From: Daniel Jurgens Date: Tue, 7 Mar 2023 00:27:21 +0200 Subject: net/mlx5: mlx5_ifc updates for embedded CPU SRIOV Add ec_vf_vport_base to HCA Capabilities 2. This indicates the base vport of embedded CPU virtual functions that are connected to the eswitch. Add ec_vf_function to query/set_hca_caps. If set this indicates accessing a virtual function on the embedded CPU by function ID. This should only be used with other_function set to 1. Signed-off-by: Daniel Jurgens Reviewed-by: Bodong Wang Reviewed-by: William Tu Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index af3a92ad2e6b..1f4f62cb9f34 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1992,7 +1992,10 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 ts_cqe_metadata_size2wqe_counter[0x5]; u8 reserved_at_250[0x10]; - u8 reserved_at_260[0x5a0]; + u8 reserved_at_260[0x120]; + u8 reserved_at_380[0x10]; + u8 ec_vf_vport_base[0x10]; + u8 reserved_at_3a0[0x460]; }; enum mlx5_ifc_flow_destination_type { @@ -4805,7 +4808,8 @@ struct mlx5_ifc_set_hca_cap_in_bits { u8 op_mod[0x10]; u8 other_function[0x1]; - u8 reserved_at_41[0xf]; + u8 ec_vf_function[0x1]; + u8 reserved_at_42[0xe]; u8 function_id[0x10]; u8 reserved_at_60[0x20]; @@ -5956,7 +5960,8 @@ struct mlx5_ifc_query_hca_cap_in_bits { u8 op_mod[0x10]; u8 other_function[0x1]; - u8 reserved_at_41[0xf]; + u8 ec_vf_function[0x1]; + u8 reserved_at_42[0xe]; u8 function_id[0x10]; u8 reserved_at_60[0x20]; -- cgit v1.2.3 From dc13180824b78e1e4e7ae1ce22160ae8e5fb858e Mon Sep 17 00:00:00 2001 From: Daniel Jurgens Date: Tue, 7 Mar 2023 19:36:14 +0200 Subject: net/mlx5: Enable devlink port for embedded cpu VF vports Enable creation of a devlink port for EC VF vports. Signed-off-by: Daniel Jurgens Reviewed-by: William Tu Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 9a744c48eec2..252b6a6965b8 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -474,6 +474,7 @@ struct mlx5_core_sriov { struct mlx5_vf_context *vfs_ctx; int num_vfs; u16 max_vfs; + u16 max_ec_vfs; }; struct mlx5_fc_pool { @@ -1244,6 +1245,11 @@ static inline u16 mlx5_core_max_vfs(const struct mlx5_core_dev *dev) return dev->priv.sriov.max_vfs; } +static inline u16 mlx5_core_max_ec_vfs(const struct mlx5_core_dev *dev) +{ + return dev->priv.sriov.max_ec_vfs; +} + static inline int mlx5_get_gid_table_len(u16 param) { if (param > 4) { -- cgit v1.2.3 From 9ac0b128248e19d06475f4592fe87f6ce18bc554 Mon Sep 17 00:00:00 2001 From: Daniel Jurgens Date: Tue, 7 Mar 2023 19:51:22 +0200 Subject: net/mlx5: Update vport caps query/set for EC VFs These functions are for query/set by vport, there was an underlying assumption that vport was equal to function ID. That's not the case for EC VF functions. Set the ec_vf_function bit accordingly. Signed-off-by: Daniel Jurgens Reviewed-by: William Tu Signed-off-by: Saeed Mahameed --- include/linux/mlx5/vport.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index 7f31432f44c2..fbb9bf447889 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -132,6 +132,6 @@ int mlx5_nic_vport_affiliate_multiport(struct mlx5_core_dev *master_mdev, int mlx5_nic_vport_unaffiliate_multiport(struct mlx5_core_dev *port_mdev); u64 mlx5_query_nic_system_image_guid(struct mlx5_core_dev *mdev); -int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out, +int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 vport, void *out, u16 opmod); #endif /* __MLX5_VPORT_H__ */ -- cgit v1.2.3 From 395ccd6eb49a12b021ac5deaa56e6b0b8f93241b Mon Sep 17 00:00:00 2001 From: Daniel Jurgens Date: Tue, 7 Mar 2023 00:53:21 +0200 Subject: net/mlx5: Add new page type for EC VF pages When the embedded cpu supports SRIOV it can be enabled and disabled independently from the host SRIOV. Track the pages separately so we can properly wait for returned VF pages. Signed-off-by: Daniel Jurgens Reviewed-by: William Tu Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 252b6a6965b8..18a608a1f567 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -581,6 +581,7 @@ enum mlx5_func_type { MLX5_VF, MLX5_SF, MLX5_HOST_PF, + MLX5_EC_VF, MLX5_FUNC_TYPE_NUM, }; -- cgit v1.2.3 From d457a0e329b0bfd3a1450e0b1a18cd2b47a25a08 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 8 Jun 2023 19:17:37 +0000 Subject: net: move gso declarations and functions to their own files Move declarations into include/net/gso.h and code into net/core/gso.c Signed-off-by: Eric Dumazet Cc: Stanislav Fomichev Reviewed-by: Simon Horman Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20230608191738.3947077-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 26 +---------------- include/linux/skbuff.h | 71 ----------------------------------------------- 2 files changed, 1 insertion(+), 96 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index c2f0c6002a84..2d6cb2bf2f05 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4827,13 +4827,6 @@ int skb_crc32c_csum_help(struct sk_buff *skb); int skb_csum_hwoffload_help(struct sk_buff *skb, const netdev_features_t features); -struct sk_buff *__skb_gso_segment(struct sk_buff *skb, - netdev_features_t features, bool tx_path); -struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb, - netdev_features_t features, __be16 type); -struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, - netdev_features_t features); - struct netdev_bonding_info { ifslave slave; ifbond master; @@ -4856,11 +4849,6 @@ static inline void ethtool_notify(struct net_device *dev, unsigned int cmd, } #endif -static inline -struct sk_buff *skb_gso_segment(struct sk_buff *skb, netdev_features_t features) -{ - return __skb_gso_segment(skb, features, true); -} __be16 skb_network_protocol(struct sk_buff *skb, int *depth); static inline bool can_checksum_protocol(netdev_features_t features, @@ -4987,6 +4975,7 @@ netdev_features_t passthru_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features); netdev_features_t netif_skb_features(struct sk_buff *skb); +void skb_warn_bad_offload(const struct sk_buff *skb); static inline bool net_gso_ok(netdev_features_t features, int gso_type) { @@ -5035,19 +5024,6 @@ void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs); void netif_inherit_tso_max(struct net_device *to, const struct net_device *from); -static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol, - int pulled_hlen, u16 mac_offset, - int mac_len) -{ - skb->protocol = protocol; - skb->encapsulation = 1; - skb_push(skb, pulled_hlen); - skb_reset_transport_header(skb); - skb->mac_header = mac_offset; - skb->network_header = skb->mac_header + mac_len; - skb->mac_len = mac_len; -} - static inline bool netif_is_macsec(const struct net_device *dev) { return dev->priv_flags & IFF_MACSEC; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index e2f48ddb2f7c..91ed66952580 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3974,8 +3974,6 @@ int skb_zerocopy(struct sk_buff *to, struct sk_buff *from, void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len); int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen); void skb_scrub_packet(struct sk_buff *skb, bool xnet); -bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu); -bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len); struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features); struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features, unsigned int offset); @@ -4841,75 +4839,6 @@ static inline struct sec_path *skb_sec_path(const struct sk_buff *skb) #endif } -/* Keeps track of mac header offset relative to skb->head. - * It is useful for TSO of Tunneling protocol. e.g. GRE. - * For non-tunnel skb it points to skb_mac_header() and for - * tunnel skb it points to outer mac header. - * Keeps track of level of encapsulation of network headers. - */ -struct skb_gso_cb { - union { - int mac_offset; - int data_offset; - }; - int encap_level; - __wsum csum; - __u16 csum_start; -}; -#define SKB_GSO_CB_OFFSET 32 -#define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET)) - -static inline int skb_tnl_header_len(const struct sk_buff *inner_skb) -{ - return (skb_mac_header(inner_skb) - inner_skb->head) - - SKB_GSO_CB(inner_skb)->mac_offset; -} - -static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra) -{ - int new_headroom, headroom; - int ret; - - headroom = skb_headroom(skb); - ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC); - if (ret) - return ret; - - new_headroom = skb_headroom(skb); - SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom); - return 0; -} - -static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res) -{ - /* Do not update partial checksums if remote checksum is enabled. */ - if (skb->remcsum_offload) - return; - - SKB_GSO_CB(skb)->csum = res; - SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head; -} - -/* Compute the checksum for a gso segment. First compute the checksum value - * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and - * then add in skb->csum (checksum from csum_start to end of packet). - * skb->csum and csum_start are then updated to reflect the checksum of the - * resultant packet starting from the transport header-- the resultant checksum - * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo - * header. - */ -static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res) -{ - unsigned char *csum_start = skb_transport_header(skb); - int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start; - __wsum partial = SKB_GSO_CB(skb)->csum; - - SKB_GSO_CB(skb)->csum = res; - SKB_GSO_CB(skb)->csum_start = csum_start - skb->head; - - return csum_fold(csum_partial(csum_start, plen, partial)); -} - static inline bool skb_is_gso(const struct sk_buff *skb) { return skb_shinfo(skb)->gso_size; -- cgit v1.2.3 From 2ecfd946169e7f56534db2a5f6935858be3005ba Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 5 Jun 2023 13:14:05 +0300 Subject: RDMA/mlx5: Reduce QP table exposure driver.h is common header to whole mlx5 code base, but struct mlx5_qp_table is used in mlx5_ib driver only. So move that struct to be under sole responsibility of mlx5_ib. Link: https://lore.kernel.org/r/bec0dc1158e795813b135d1143147977f26bf668.1685953497.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- include/linux/mlx5/driver.h | 9 --------- 1 file changed, 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index a4c4f737f9c1..e3b616388b18 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -443,15 +443,6 @@ struct mlx5_core_health { struct delayed_work update_fw_log_ts_work; }; -struct mlx5_qp_table { - struct notifier_block nb; - - /* protect radix tree - */ - spinlock_t lock; - struct radix_tree_root tree; -}; - enum { MLX5_PF_NOTIFY_DISABLE_VF, MLX5_PF_NOTIFY_ENABLE_VF, -- cgit v1.2.3 From afff24899846ffca0c25c75b24893c90aef82603 Mon Sep 17 00:00:00 2001 From: Patrisious Haddad Date: Mon, 5 Jun 2023 13:14:06 +0300 Subject: RDMA/mlx5: Handle DCT QP logic separately from low level QP interface Previously when destroying a DCT, if the firmware function for the destruction failed, the common resource would have been destroyed either way, since it was destroyed before the firmware object. Which leads to kernel warning "refcount_t: underflow" which indicates possible use-after-free. Which is triggered when we try to destroy the common resource for the second time and execute refcount_dec_and_test(&common->refcount). So, let's fix the destruction order by factoring out the DCT QP logic to be in separate XArray database. refcount_t: underflow; use-after-free. WARNING: CPU: 8 PID: 1002 at lib/refcount.c:28 refcount_warn_saturate+0xd8/0xe0 Modules linked in: xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter rpcrdma rdma_ucm ib_iser libiscsi scsi_transport_iscsi ib_umad rdma_cm ib_ipoib iw_cm ib_cm mlx5_ib ib_uverbs ib_core overlay mlx5_core fuse CPU: 8 PID: 1002 Comm: python3 Not tainted 5.16.0-rc5+ #1 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014 RIP: 0010:refcount_warn_saturate+0xd8/0xe0 Code: ff 48 c7 c7 18 f5 23 82 c6 05 60 70 ff 00 01 e8 d0 0a 45 00 0f 0b c3 48 c7 c7 c0 f4 23 82 c6 05 4c 70 ff 00 01 e8 ba 0a 45 00 <0f> 0b c3 0f 1f 44 00 00 8b 07 3d 00 00 00 c0 74 12 83 f8 01 74 13 RSP: 0018:ffff8881221d3aa8 EFLAGS: 00010286 RAX: 0000000000000000 RBX: ffff8881313e8d40 RCX: ffff88852cc1b5c8 RDX: 00000000ffffffd8 RSI: 0000000000000027 RDI: ffff88852cc1b5c0 RBP: ffff888100f70000 R08: ffff88853ffd1ba8 R09: 0000000000000003 R10: 00000000fffff000 R11: 3fffffffffffffff R12: 0000000000000246 R13: ffff888100f71fa0 R14: ffff8881221d3c68 R15: 0000000000000020 FS: 00007efebbb13740(0000) GS:ffff88852cc00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00005611aac29f80 CR3: 00000001313de004 CR4: 0000000000370ea0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: destroy_resource_common+0x6e/0x95 [mlx5_ib] mlx5_core_destroy_rq_tracked+0x38/0xbe [mlx5_ib] mlx5_ib_destroy_wq+0x22/0x80 [mlx5_ib] ib_destroy_wq_user+0x1f/0x40 [ib_core] uverbs_free_wq+0x19/0x40 [ib_uverbs] destroy_hw_idr_uobject+0x18/0x50 [ib_uverbs] uverbs_destroy_uobject+0x2f/0x190 [ib_uverbs] uobj_destroy+0x3c/0x80 [ib_uverbs] ib_uverbs_cmd_verbs+0x3e4/0xb80 [ib_uverbs] ? uverbs_free_wq+0x40/0x40 [ib_uverbs] ? ip_list_rcv+0xf7/0x120 ? netif_receive_skb_list_internal+0x1b6/0x2d0 ? task_tick_fair+0xbf/0x450 ? __handle_mm_fault+0x11fc/0x1450 ib_uverbs_ioctl+0xa4/0x110 [ib_uverbs] __x64_sys_ioctl+0x3e4/0x8e0 ? handle_mm_fault+0xb9/0x210 do_syscall_64+0x3d/0x90 entry_SYSCALL_64_after_hwframe+0x44/0xae RIP: 0033:0x7efebc0be17b Code: 0f 1e fa 48 8b 05 1d ad 0c 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d ed ac 0c 00 f7 d8 64 89 01 48 RSP: 002b:00007ffe71813e78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 00007ffe71813fb8 RCX: 00007efebc0be17b RDX: 00007ffe71813fa0 RSI: 00000000c0181b01 RDI: 0000000000000005 RBP: 00007ffe71813f80 R08: 00005611aae96020 R09: 000000000000004f R10: 00007efebbf9ffa0 R11: 0000000000000246 R12: 00007ffe71813f80 R13: 00007ffe71813f4c R14: 00005611aae2eca0 R15: 00007efeae6c89d0 Signed-off-by: Patrisious Haddad Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/4470888466c8a898edc9833286967529cc5f3c0d.1685953497.git.leon@kernel.org Signed-off-by: Leon Romanovsky --- include/linux/mlx5/driver.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index e3b616388b18..e67c603d507b 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -382,7 +382,6 @@ enum mlx5_res_type { MLX5_RES_SRQ = 3, MLX5_RES_XSRQ = 4, MLX5_RES_XRQ = 5, - MLX5_RES_DCT = MLX5_EVENT_QUEUE_TYPE_DCT, }; struct mlx5_core_rsc_common { -- cgit v1.2.3 From 703d7521555504b3a316b105b4806d641b7ebc76 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 18 May 2023 13:46:03 -0400 Subject: NFSD: Hoist rq_vec preparation into nfsd_read() [step two] Now that the preparation of an rq_vec has been removed from the generic read path, nfsd_splice_read() no longer needs to reset rq_next_page. nfsd4_encode_read() calls nfsd_splice_read() directly. As far as I can ascertain, resetting rq_next_page for NFSv4 splice reads is unnecessary because rq_next_page is already set correctly. Moreover, resetting it might even be incorrect if previous operations in the COMPOUND have already consumed at least a page of the send buffer. I would expect that the result would be encoding the READ payload over previously-encoded results. Signed-off-by: Chuck Lever --- include/linux/sunrpc/xdr.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 72014c9216fc..f89ec4b5ea16 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -242,8 +242,7 @@ extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, extern void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf, struct page **pages, struct rpc_rqst *rqst); extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes); -extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, - size_t nbytes); +extern int xdr_reserve_space_vec(struct xdr_stream *xdr, size_t nbytes); extern void __xdr_commit_encode(struct xdr_stream *xdr); extern void xdr_truncate_encode(struct xdr_stream *xdr, size_t len); extern void xdr_truncate_decode(struct xdr_stream *xdr, size_t len); -- cgit v1.2.3 From b6334e2cd46f0dab1131c0272426182c79b51114 Mon Sep 17 00:00:00 2001 From: Mao Zhu Date: Sun, 11 Jun 2023 08:33:14 -0400 Subject: fs: Fix comment typo Delete duplicated word in comment. Signed-off-by: Mao Zhu Message-Id: <20230611123314.5282-1-dengshaomin@cdjrlc.com> Signed-off-by: Christian Brauner --- include/linux/fs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 9c2671b285a4..40bef9bf8749 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2675,7 +2675,7 @@ extern void evict_inodes(struct super_block *sb); void dump_mapping(const struct address_space *); /* - * Userspace may rely on the the inode number being non-zero. For example, glibc + * Userspace may rely on the inode number being non-zero. For example, glibc * simply ignores files with zero i_ino in unlink() and other places. * * As an additional complication, if userspace was compiled with -- cgit v1.2.3 From 5e2ff6704a275be009be8979af17c52361b79b89 Mon Sep 17 00:00:00 2001 From: Alexander Mikhalitsyn Date: Thu, 8 Jun 2023 22:26:25 +0200 Subject: scm: add SO_PASSPIDFD and SCM_PIDFD Implement SCM_PIDFD, a new type of CMSG type analogical to SCM_CREDENTIALS, but it contains pidfd instead of plain pid, which allows programmers not to care about PID reuse problem. We mask SO_PASSPIDFD feature if CONFIG_UNIX is not builtin because it depends on a pidfd_prepare() API which is not exported to the kernel modules. Idea comes from UAPI kernel group: https://uapi-group.org/kernel-features/ Big thanks to Christian Brauner and Lennart Poettering for productive discussions about this. Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Leon Romanovsky Cc: David Ahern Cc: Arnd Bergmann Cc: Kees Cook Cc: Christian Brauner Cc: Kuniyuki Iwashima Cc: Lennart Poettering Cc: Luca Boccassi Cc: linux-kernel@vger.kernel.org Cc: netdev@vger.kernel.org Cc: linux-arch@vger.kernel.org Tested-by: Luca Boccassi Reviewed-by: Kuniyuki Iwashima Reviewed-by: Christian Brauner Signed-off-by: Alexander Mikhalitsyn Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/net.h | 1 + include/linux/socket.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/net.h b/include/linux/net.h index 8defc8f1d82e..23324e9a2b3d 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -43,6 +43,7 @@ struct net; #define SOCK_PASSSEC 4 #define SOCK_SUPPORT_ZC 5 #define SOCK_CUSTOM_SOCKOPT 6 +#define SOCK_PASSPIDFD 7 #ifndef ARCH_HAS_SOCKET_TYPES /** diff --git a/include/linux/socket.h b/include/linux/socket.h index 3fd3436bc09f..58204700018a 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -177,6 +177,7 @@ static inline size_t msg_data_left(struct msghdr *msg) #define SCM_RIGHTS 0x01 /* rw: access rights (array of int) */ #define SCM_CREDENTIALS 0x02 /* rw: struct ucred */ #define SCM_SECURITY 0x03 /* rw: security label */ +#define SCM_PIDFD 0x04 /* ro: pidfd (int) */ struct ucred { __u32 pid; -- cgit v1.2.3 From 5ab8c41cef30d8b6160a80b69d2eb39d570491ac Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 9 Jun 2023 14:53:30 -0700 Subject: netlink: support extack in dump ->start() Commit 4a19edb60d02 ("netlink: Pass extack to dump handlers") added extack support to netlink dumps. It was focused on rtnl and since rtnl does not use ->start(), ->done() callbacks it ignored those. Genetlink on the other hand uses ->start() extensively, for parsing and input validation. Pass the extact in via struct netlink_dump_control and link it to cb for the time of ->start(). Both struct netlink_dump_control and extack itself live on the stack so we can't keep the same extack for the duration of the dump. This means that the extack visible in ->start() and each ->dump() callbacks will be different. Corner cases like reporting a warning message in DONE across dump calls are still not supported. We could put the extack (for dumps) in the socket struct, but layering makes it slightly awkward (extack pointer is decided before the DO / DUMP split). The genetlink dump error extacks are now surfaced: $ cli.py --spec netlink/specs/ethtool.yaml --dump channels-get lib.ynl.NlError: Netlink error: Invalid argument nl_len = 64 (48) nl_flags = 0x300 nl_type = 2 error: -22 extack: {'msg': 'request header missing'} Previously extack was missing: $ cli.py --spec netlink/specs/ethtool.yaml --dump channels-get lib.ynl.NlError: Netlink error: Invalid argument nl_len = 36 (20) nl_flags = 0x100 nl_type = 2 error: -22 Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/linux/netlink.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/netlink.h b/include/linux/netlink.h index 19c0791ed9d5..9eec3f4f5351 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -311,6 +311,7 @@ struct netlink_dump_control { int (*start)(struct netlink_callback *); int (*dump)(struct sk_buff *skb, struct netlink_callback *); int (*done)(struct netlink_callback *); + struct netlink_ext_ack *extack; void *data; struct module *module; u32 min_dump_alloc; -- cgit v1.2.3 From 764b83100b9aff52f950e408539c22a37cdedae8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:30 +0200 Subject: cdrom: remove the unused bdev argument to cdrom_open Signed-off-by: Christoph Hellwig Reviewed-by: Phillip Potter Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-3-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/cdrom.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index 67caa909e3e6..cc5717cb0fa8 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -101,8 +101,7 @@ int cdrom_read_tocentry(struct cdrom_device_info *cdi, struct cdrom_tocentry *entry); /* the general block_device operations structure: */ -extern int cdrom_open(struct cdrom_device_info *cdi, struct block_device *bdev, - fmode_t mode); +int cdrom_open(struct cdrom_device_info *cdi, fmode_t mode); extern void cdrom_release(struct cdrom_device_info *cdi, fmode_t mode); extern int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg); -- cgit v1.2.3 From 473399b50de1fdc12606254351273c71d1786251 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:31 +0200 Subject: cdrom: remove the unused mode argument to cdrom_ioctl Signed-off-by: Christoph Hellwig Reviewed-by: Phillip Potter Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-4-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/cdrom.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index cc5717cb0fa8..4aea8c82d169 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -103,8 +103,8 @@ int cdrom_read_tocentry(struct cdrom_device_info *cdi, /* the general block_device operations structure: */ int cdrom_open(struct cdrom_device_info *cdi, fmode_t mode); extern void cdrom_release(struct cdrom_device_info *cdi, fmode_t mode); -extern int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, - fmode_t mode, unsigned int cmd, unsigned long arg); +int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, + unsigned int cmd, unsigned long arg); extern unsigned int cdrom_check_events(struct cdrom_device_info *cdi, unsigned int clearing); -- cgit v1.2.3 From a4cec8bc14c02e15006a71f02b0e1bbc72b9f796 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:32 +0200 Subject: cdrom: remove the unused cdrom_close_write release code cdrom_close_write is empty, and the for_data flag it is keyed off is never set. Remove all this clutter. Signed-off-by: Christoph Hellwig Reviewed-by: Phillip Potter Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-5-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/cdrom.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index 4aea8c82d169..0a5db0b0c958 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -61,7 +61,6 @@ struct cdrom_device_info { __u8 last_sense; __u8 media_written; /* dirty flag, DVD+RW bookkeeping */ unsigned short mmc3_profile; /* current MMC3 profile */ - int for_data; int (*exit)(struct cdrom_device_info *); int mrw_mode_page; __s64 last_media_change_ms; -- cgit v1.2.3 From 8cdf433e2b8e4fc6c7b4393deb93fb258175d537 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:33 +0200 Subject: cdrom: track if a cdrom_device_info was opened for data Set a flag when a cdrom_device_info is opened for writing, instead of trying to figure out this at release time. This will allow to eventually remove the mode argument to the ->release block_device_operation as nothing but the CDROM drivers uses that argument. Signed-off-by: Christoph Hellwig Reviewed-by: Phillip Potter Acked-by: Christian Brauner Reviewed-by: Hannes Reinecke Link: https://lore.kernel.org/r/20230608110258.189493-6-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/cdrom.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index 0a5db0b0c958..adcc9f2beb26 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -63,6 +63,7 @@ struct cdrom_device_info { unsigned short mmc3_profile; /* current MMC3 profile */ int (*exit)(struct cdrom_device_info *); int mrw_mode_page; + bool opened_for_data; __s64 last_media_change_ms; }; -- cgit v1.2.3 From 7ae24fcee9929f9002b84d8121144b2b3590b58c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:34 +0200 Subject: cdrom: remove the unused mode argument to cdrom_release Signed-off-by: Christoph Hellwig Reviewed-by: Phillip Potter Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-7-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/cdrom.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index adcc9f2beb26..3c253b29f4aa 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -102,7 +102,7 @@ int cdrom_read_tocentry(struct cdrom_device_info *cdi, /* the general block_device operations structure: */ int cdrom_open(struct cdrom_device_info *cdi, fmode_t mode); -extern void cdrom_release(struct cdrom_device_info *cdi, fmode_t mode); +void cdrom_release(struct cdrom_device_info *cdi); int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, unsigned int cmd, unsigned long arg); extern unsigned int cdrom_check_events(struct cdrom_device_info *cdi, -- cgit v1.2.3 From 444aa2c58cb3b6cfe3b7cc7db6c294d73393a894 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:35 +0200 Subject: block: pass a gendisk on bdev_check_media_change bdev_check_media_change should only ever be called for the whole device. Pass a gendisk to make that explicit and rename the function to disk_check_media_change. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-8-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f4c339d9dd03..a1688eba7e5e 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -817,7 +817,7 @@ int __register_blkdev(unsigned int major, const char *name, __register_blkdev(major, name, NULL) void unregister_blkdev(unsigned int major, const char *name); -bool bdev_check_media_change(struct block_device *bdev); +bool disk_check_media_change(struct gendisk *disk); int __invalidate_device(struct block_device *bdev, bool kill_dirty); void set_capacity(struct gendisk *disk, sector_t size); -- cgit v1.2.3 From d32e2bf83791727a84ad5d3e3d713e82f9adbe30 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:36 +0200 Subject: block: pass a gendisk to ->open ->open is only called on the whole device. Make that explicit by passing a gendisk instead of the block_device. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Acked-by: Jack Wang [rnbd] Link: https://lore.kernel.org/r/20230608110258.189493-9-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index a1688eba7e5e..1366eea88186 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1386,7 +1386,7 @@ struct block_device_operations { void (*submit_bio)(struct bio *bio); int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); - int (*open) (struct block_device *, fmode_t); + int (*open)(struct gendisk *disk, fmode_t mode); void (*release) (struct gendisk *, fmode_t); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); -- cgit v1.2.3 From ae220766d87cd6799dbf918fea10613ae14c0654 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:37 +0200 Subject: block: remove the unused mode argument to ->release The mode argument to the ->release block_device_operation is never used, so remove it. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Acked-by: Jack Wang [rnbd] Link: https://lore.kernel.org/r/20230608110258.189493-10-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 1366eea88186..25bdd0cc74dc 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1387,7 +1387,7 @@ struct block_device_operations { int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); int (*open)(struct gendisk *disk, fmode_t mode); - void (*release) (struct gendisk *, fmode_t); + void (*release)(struct gendisk *disk); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); unsigned int (*check_events) (struct gendisk *disk, -- cgit v1.2.3 From 2736e8eeb0ccdc71d1f4256c9c9a28f58cc43307 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:43 +0200 Subject: block: use the holder as indication for exclusive opens The current interface for exclusive opens is rather confusing as it requires both the FMODE_EXCL flag and a holder. Remove the need to pass FMODE_EXCL and just key off the exclusive open off a non-NULL holder. For blkdev_put this requires adding the holder argument, which provides better debug checking that only the holder actually releases the hold, but at the same time allows removing the now superfluous mode argument. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Acked-by: David Sterba [btrfs] Acked-by: Jack Wang [rnbd] Link: https://lore.kernel.org/r/20230608110258.189493-16-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 25bdd0cc74dc..d5b99796f12c 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1480,7 +1480,7 @@ struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, int bd_prepare_to_claim(struct block_device *bdev, void *holder, const struct blk_holder_ops *hops); void bd_abort_claiming(struct block_device *bdev, void *holder); -void blkdev_put(struct block_device *bdev, fmode_t mode); +void blkdev_put(struct block_device *bdev, void *holder); /* just for blk-cgroup, don't use elsewhere */ struct block_device *blkdev_get_no_open(dev_t dev); -- cgit v1.2.3 From 3f0b3e785e8b54a40c530fa77b7ab37bec925c57 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:44 +0200 Subject: block: add a sb_open_mode helper Add a helper to return the open flags for blkdev_get_by* for passed in super block flags instead of open coding the logic in many places. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-17-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d5b99796f12c..978036039020 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1473,6 +1473,13 @@ struct blk_holder_ops { void (*mark_dead)(struct block_device *bdev); }; +/* + * Return the correct open flags for blkdev_get_by_* for super block flags + * as stored in sb->s_flags. + */ +#define sb_open_mode(flags) \ + (FMODE_READ | (((flags) & SB_RDONLY) ? 0 : FMODE_WRITE)) + struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder, const struct blk_holder_ops *hops); struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, -- cgit v1.2.3 From 81b1fb7d17c0110df839e13468ada9e99bb6e5f4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:45 +0200 Subject: fs: remove sb->s_mode There is no real need to store the open mode in the super_block now. It is only used by f2fs, which can easily recalculate it. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-18-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/fs.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 7b2053649820..ad1d2c9afb3f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1215,7 +1215,6 @@ struct super_block { uuid_t s_uuid; /* UUID */ unsigned int s_max_links; - fmode_t s_mode; /* * The next field is for VFS *only*. No filesystems have any business -- cgit v1.2.3 From 1991299e49fa58c3ba7e91599932f84bf537d592 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:48 +0200 Subject: scsi: replace the fmode_t argument to ->sg_io_fn with a simple bool Instead of passing a fmode_t and only checking it for FMODE_WRITE, pass a bool open_for_write to prepare for callers that won't have the fmode_t. Signed-off-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-21-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/bsg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bsg.h b/include/linux/bsg.h index 1ac81c809da9..ee2df73edf83 100644 --- a/include/linux/bsg.h +++ b/include/linux/bsg.h @@ -9,7 +9,7 @@ struct device; struct request_queue; typedef int (bsg_sg_io_fn)(struct request_queue *, struct sg_io_v4 *hdr, - fmode_t mode, unsigned int timeout); + bool open_for_write, unsigned int timeout); struct bsg_device *bsg_register_queue(struct request_queue *q, struct device *parent, const char *name, -- cgit v1.2.3 From 658afed19ceed54a52b9e9e69c0791c8868ff55d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:50 +0200 Subject: mtd: block: use a simple bool to track open for write Instead of propagating the fmode_t, just use a bool to track if a mtd block device was opened for writing. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Acked-by: Richard Weinberger Link: https://lore.kernel.org/r/20230608110258.189493-23-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/mtd/blktrans.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mtd/blktrans.h b/include/linux/mtd/blktrans.h index 15cc9b95e32b..6e471436bba5 100644 --- a/include/linux/mtd/blktrans.h +++ b/include/linux/mtd/blktrans.h @@ -34,7 +34,7 @@ struct mtd_blktrans_dev { struct blk_mq_tag_set *tag_set; spinlock_t queue_lock; void *priv; - fmode_t file_mode; + bool writable; }; struct mtd_blktrans_ops { -- cgit v1.2.3 From cfb425761c79b6056ae5bb73f8d400f03b513959 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:53 +0200 Subject: block: move a few internal definitions out of blkdev.h All these helpers are only used in core block code, so move them out of the public header. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Acked-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-26-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 27 --------------------------- 1 file changed, 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 978036039020..6b65623e447c 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -318,7 +318,6 @@ typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx, void disk_set_zoned(struct gendisk *disk, enum blk_zoned_model model); #ifdef CONFIG_BLK_DEV_ZONED - #define BLK_ALL_ZONES ((unsigned int)-1) int blkdev_report_zones(struct block_device *bdev, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data); @@ -328,33 +327,11 @@ extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op, gfp_t gfp_mask); int blk_revalidate_disk_zones(struct gendisk *disk, void (*update_driver_data)(struct gendisk *disk)); - -extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, - unsigned int cmd, unsigned long arg); -extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, - unsigned int cmd, unsigned long arg); - #else /* CONFIG_BLK_DEV_ZONED */ - static inline unsigned int bdev_nr_zones(struct block_device *bdev) { return 0; } - -static inline int blkdev_report_zones_ioctl(struct block_device *bdev, - fmode_t mode, unsigned int cmd, - unsigned long arg) -{ - return -ENOTTY; -} - -static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev, - fmode_t mode, unsigned int cmd, - unsigned long arg) -{ - return -ENOTTY; -} - #endif /* CONFIG_BLK_DEV_ZONED */ /* @@ -1493,11 +1470,7 @@ void blkdev_put(struct block_device *bdev, void *holder); struct block_device *blkdev_get_no_open(dev_t dev); void blkdev_put_no_open(struct block_device *bdev); -struct block_device *bdev_alloc(struct gendisk *disk, u8 partno); -void bdev_add(struct block_device *bdev, dev_t dev); struct block_device *I_BDEV(struct inode *inode); -int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart, - loff_t lend); #ifdef CONFIG_BLOCK void invalidate_bdev(struct block_device *bdev); -- cgit v1.2.3 From 05bdb9965305bbfdae79b31d22df03d1e2cfcb22 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:55 +0200 Subject: block: replace fmode_t with a block-specific type for block open flags The only overlap between the block open flags mapped into the fmode_t and other uses of fmode_t are FMODE_READ and FMODE_WRITE. Define a new blk_mode_t instead for use in blkdev_get_by_{dev,path}, ->open and ->ioctl and stop abusing fmode_t. Signed-off-by: Christoph Hellwig Acked-by: Jack Wang [rnbd] Reviewed-by: Hannes Reinecke Reviewed-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-28-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 30 +++++++++++++++++++++++------- include/linux/cdrom.h | 3 ++- include/linux/device-mapper.h | 8 ++++---- 3 files changed, 29 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6b65623e447c..824e31dd752a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -112,6 +112,19 @@ struct blk_integrity { unsigned char tag_size; }; +typedef unsigned int __bitwise blk_mode_t; + +/* open for reading */ +#define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) +/* open for writing */ +#define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1)) +/* open exclusively (vs other exclusive openers */ +#define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2)) +/* opened with O_NDELAY */ +#define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3)) +/* open for "writes" only for ioctls (specialy hack for floppy.c) */ +#define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4)) + struct gendisk { /* * major/first_minor/minors should not be set by any new driver, the @@ -187,6 +200,7 @@ struct gendisk { struct badblocks *bb; struct lockdep_map lockdep_map; u64 diskseq; + blk_mode_t open_mode; /* * Independent sector access ranges. This is always NULL for @@ -1363,10 +1377,12 @@ struct block_device_operations { void (*submit_bio)(struct bio *bio); int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob, unsigned int flags); - int (*open)(struct gendisk *disk, fmode_t mode); + int (*open)(struct gendisk *disk, blk_mode_t mode); void (*release)(struct gendisk *disk); - int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); - int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); + int (*ioctl)(struct block_device *bdev, blk_mode_t mode, + unsigned cmd, unsigned long arg); + int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode, + unsigned cmd, unsigned long arg); unsigned int (*check_events) (struct gendisk *disk, unsigned int clearing); void (*unlock_native_capacity) (struct gendisk *); @@ -1393,7 +1409,7 @@ struct block_device_operations { }; #ifdef CONFIG_COMPAT -extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t, +extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t, unsigned int, unsigned long); #else #define blkdev_compat_ptr_ioctl NULL @@ -1455,11 +1471,11 @@ struct blk_holder_ops { * as stored in sb->s_flags. */ #define sb_open_mode(flags) \ - (FMODE_READ | (((flags) & SB_RDONLY) ? 0 : FMODE_WRITE)) + (BLK_OPEN_READ | (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE)) -struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder, +struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder, const struct blk_holder_ops *hops); -struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, +struct block_device *blkdev_get_by_path(const char *path, blk_mode_t mode, void *holder, const struct blk_holder_ops *hops); int bd_prepare_to_claim(struct block_device *bdev, void *holder, const struct blk_holder_ops *hops); diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index 3c253b29f4aa..98c6fd0b39b6 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -13,6 +13,7 @@ #include /* not really needed, later.. */ #include +#include #include #include @@ -101,7 +102,7 @@ int cdrom_read_tocentry(struct cdrom_device_info *cdi, struct cdrom_tocentry *entry); /* the general block_device operations structure: */ -int cdrom_open(struct cdrom_device_info *cdi, fmode_t mode); +int cdrom_open(struct cdrom_device_info *cdi, blk_mode_t mode); void cdrom_release(struct cdrom_device_info *cdi); int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, unsigned int cmd, unsigned long arg); diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index c27b84002d83..69d0435c7ebb 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -166,7 +166,7 @@ void dm_error(const char *message); struct dm_dev { struct block_device *bdev; struct dax_device *dax_dev; - fmode_t mode; + blk_mode_t mode; char name[16]; }; @@ -174,7 +174,7 @@ struct dm_dev { * Constructors should call these functions to ensure destination devices * are opened/closed correctly. */ -int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode, +int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode, struct dm_dev **result); void dm_put_device(struct dm_target *ti, struct dm_dev *d); @@ -543,7 +543,7 @@ int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo); /* * First create an empty table. */ -int dm_table_create(struct dm_table **result, fmode_t mode, +int dm_table_create(struct dm_table **result, blk_mode_t mode, unsigned int num_targets, struct mapped_device *md); /* @@ -586,7 +586,7 @@ void dm_sync_table(struct mapped_device *md); * Queries */ sector_t dm_table_get_size(struct dm_table *t); -fmode_t dm_table_get_mode(struct dm_table *t); +blk_mode_t dm_table_get_mode(struct dm_table *t); struct mapped_device *dm_table_get_md(struct dm_table *t); const char *dm_table_device_name(struct dm_table *t); -- cgit v1.2.3 From 0733ad8002916b9dbbbcfe6e92ad44d2657de1c1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 8 Jun 2023 13:02:58 +0200 Subject: fs: remove the now unused FMODE_* flags FMODE_NDELAY, FMODE_EXCL and FMODE_WRITE_IOCTL were only used for block internal purposed and are now entirely unused, so remove them. Signed-off-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Reviewed-by: Christian Brauner Link: https://lore.kernel.org/r/20230608110258.189493-31-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/fs.h | 7 ------- 1 file changed, 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index ad1d2c9afb3f..8045c7ef4000 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -119,13 +119,6 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset, #define FMODE_PWRITE ((__force fmode_t)0x10) /* File is opened for execution with sys_execve / sys_uselib */ #define FMODE_EXEC ((__force fmode_t)0x20) -/* File is opened with O_NDELAY (only set for block devices) */ -#define FMODE_NDELAY ((__force fmode_t)0x40) -/* File is opened with O_EXCL (only set for block devices) */ -#define FMODE_EXCL ((__force fmode_t)0x80) -/* File is opened using open(.., 3, ..) and is writeable only for ioctls - (specialy hack for floppy.c) */ -#define FMODE_WRITE_IOCTL ((__force fmode_t)0x100) /* 32bit hashes as llseek() offset (for directories) */ #define FMODE_32BITHASH ((__force fmode_t)0x200) /* 64bit hashes as llseek() offset (for directories) */ -- cgit v1.2.3 From 4f1731df60f9033669f024d06ae26a6301260b55 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Sat, 10 Jun 2023 10:30:43 +0800 Subject: blk-mq: fix potential io hang by wrong 'wake_batch' In __blk_mq_tag_busy/idle(), updating 'active_queues' and calculating 'wake_batch' is not atomic: t1: t2: _blk_mq_tag_busy blk_mq_tag_busy inc active_queues // assume 1->2 inc active_queues // 2 -> 3 blk_mq_update_wake_batch // calculate based on 3 blk_mq_update_wake_batch /* calculate based on 2, while active_queues is actually 3. */ Fix this problem by protecting them wih 'tags->lock', this is not a hot path, so performance should not be concerned. And now that all writers are inside the lock, switch 'actives_queues' from atomic to unsigned int. Fixes: 180dccb0dba4 ("blk-mq: fix tag_get wait task can't be awakened") Signed-off-by: Yu Kuai Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230610023043.2559121-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 59b52ec155b1..f401067ac03a 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -739,8 +739,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, struct blk_mq_tags { unsigned int nr_tags; unsigned int nr_reserved_tags; - - atomic_t active_queues; + unsigned int active_queues; struct sbitmap_queue bitmap_tags; struct sbitmap_queue breserved_tags; -- cgit v1.2.3 From c5d68d25bd6b5798bf0eb96661e1b26748e970d7 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 5 Jun 2023 09:11:30 -0400 Subject: svcrdma: Clean up allocation of svc_rdma_recv_ctxt The physical device's favored NUMA node ID is available when allocating a recv_ctxt. Use that value instead of relying on the assumption that the memory allocation happens to be running on a node close to the device. This clean up eliminates the hack of destroying recv_ctxts that were not created by the receive CQ thread -- recv_ctxts are now always allocated on a "good" node. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index fbc4bd423b35..a0f3ea357977 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -135,7 +135,6 @@ struct svc_rdma_recv_ctxt { struct ib_sge rc_recv_sge; void *rc_recv_buf; struct xdr_stream rc_stream; - bool rc_temp; u32 rc_byte_len; unsigned int rc_page_count; u32 rc_inv_rkey; -- cgit v1.2.3 From a944209c11aff9a5c9b7987fc958cc2344dca51f Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 12 Jun 2023 10:10:07 -0400 Subject: SUNRPC: Revert 579900670ac7 ("svcrdma: Remove unused sc_pages field") Pre-requisite for releasing pages in the send completion handler. Reverted by hand: patch -R would not apply cleanly. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index a0f3ea357977..8e654da55170 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -158,8 +158,9 @@ struct svc_rdma_send_ctxt { struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; void *sc_xprt_buf; + int sc_page_count; int sc_cur_sge_no; - + struct page *sc_pages[RPCSVC_MAXPAGES]; struct ib_sge sc_sges[]; }; -- cgit v1.2.3 From c4b50cdf9d9d7962d58ece5efba865f56ec40398 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Mon, 12 Jun 2023 10:10:14 -0400 Subject: svcrdma: Revert 2a1e4f21d841 ("svcrdma: Normalize Send page handling") Get rid of the completion wait in svc_rdma_sendto(), and release pages in the send completion handler again. A subsequent patch will handle releasing those pages more efficiently. Reverted by hand: patch -R would not apply cleanly. Reviewed-by: Jeff Layton Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 8e654da55170..a5ee0af2a310 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -154,7 +154,6 @@ struct svc_rdma_send_ctxt { struct ib_send_wr sc_send_wr; struct ib_cqe sc_cqe; - struct completion sc_done; struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; void *sc_xprt_buf; -- cgit v1.2.3 From d16b3af46679a1eb21652c37711a60d3d4e6b8c0 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Sat, 10 Jun 2023 11:57:37 +0800 Subject: cgroup: remove unused task_cgroup_path() task_cgroup_path() is not used anymore. So remove it. Signed-off-by: Miaohe Lin Signed-off-by: Tejun Heo --- include/linux/cgroup.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 885f5395fcd0..1261a47932a6 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -118,7 +118,6 @@ int cgroup_rm_cftypes(struct cftype *cfts); void cgroup_file_notify(struct cgroup_file *cfile); void cgroup_file_show(struct cgroup_file *cfile, bool show); -int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen); int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry); int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *tsk); -- cgit v1.2.3 From 92bbe55182affa9f3b00a266d5f41fbc8a2114d6 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 1 May 2023 14:40:29 +0200 Subject: nvmet: reorder fields in 'struct nvmefc_fcp_req' Group some variables based on their sizes to reduce holes. On x86_64, this shrinks the size of 'struct nvmefc_fcp_req' from 112 to 104 bytes. This structure is embedded in some other structures (nvme_fc_fcp_op which itself is embedded in nvme_fcp_op_w_sgl), so it helps reducing the size of these structures too. Signed-off-by: Christophe JAILLET Reviewed-by: Sagi Grimberg Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- include/linux/nvme-fc-driver.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nvme-fc-driver.h b/include/linux/nvme-fc-driver.h index fa092b9be2fd..4109f1bd6128 100644 --- a/include/linux/nvme-fc-driver.h +++ b/include/linux/nvme-fc-driver.h @@ -185,7 +185,6 @@ enum nvmefc_fcp_datadir { * @first_sgl: memory for 1st scatter/gather list segment for payload data * @sg_cnt: number of elements in the scatter/gather list * @io_dir: direction of the FCP request (see NVMEFC_FCP_xxx) - * @sqid: The nvme SQID the command is being issued on * @done: The callback routine the LLDD is to invoke upon completion of * the FCP operation. req argument is the pointer to the original * FCP IO operation. @@ -194,12 +193,13 @@ enum nvmefc_fcp_datadir { * while processing the operation. The length of the buffer * corresponds to the fcprqst_priv_sz value specified in the * nvme_fc_port_template supplied by the LLDD. + * @sqid: The nvme SQID the command is being issued on * * Values set by the LLDD indicating completion status of the FCP operation. * Must be set prior to calling the done() callback. + * @rcv_rsplen: length, in bytes, of the FCP RSP IU received. * @transferred_length: amount of payload data, in bytes, that were * transferred. Should equal payload_length on success. - * @rcv_rsplen: length, in bytes, of the FCP RSP IU received. * @status: Completion status of the FCP operation. must be 0 upon success, * negative errno value upon failure (ex: -EIO). Note: this is * NOT a reflection of the NVME CQE completion status. Only the @@ -219,14 +219,14 @@ struct nvmefc_fcp_req { int sg_cnt; enum nvmefc_fcp_datadir io_dir; - __le16 sqid; - void (*done)(struct nvmefc_fcp_req *req); void *private; - u32 transferred_length; + __le16 sqid; + u16 rcv_rsplen; + u32 transferred_length; u32 status; } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ -- cgit v1.2.3 From cdb8c100d8a4b4e31c829724e40b4fdf32977cce Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 2 Jun 2023 02:30:22 -0500 Subject: include/linux/suspend.h: Only show pm_pr_dbg messages at suspend/resume All uses in the kernel are currently already oriented around suspend/resume. As some other parts of the kernel may also use these messages in functions that could also be used outside of suspend/resume, only enable in suspend/resume path. Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- include/linux/suspend.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 1a0426e6761c..74f406c53ac0 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -555,6 +555,7 @@ static inline void unlock_system_sleep(unsigned int flags) {} #ifdef CONFIG_PM_SLEEP_DEBUG extern bool pm_print_times_enabled; extern bool pm_debug_messages_on; +extern bool pm_debug_messages_should_print(void); static inline int pm_dyn_debug_messages_on(void) { #ifdef CONFIG_DYNAMIC_DEBUG @@ -568,14 +569,14 @@ static inline int pm_dyn_debug_messages_on(void) #endif #define __pm_pr_dbg(fmt, ...) \ do { \ - if (pm_debug_messages_on) \ + if (pm_debug_messages_should_print()) \ printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ else if (pm_dyn_debug_messages_on()) \ pr_debug(fmt, ##__VA_ARGS__); \ } while (0) #define __pm_deferred_pr_dbg(fmt, ...) \ do { \ - if (pm_debug_messages_on) \ + if (pm_debug_messages_should_print()) \ printk_deferred(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ } while (0) #else @@ -593,7 +594,8 @@ static inline int pm_dyn_debug_messages_on(void) /** * pm_pr_dbg - print pm sleep debug messages * - * If pm_debug_messages_on is enabled, print message. + * If pm_debug_messages_on is enabled and the system is entering/leaving + * suspend, print message. * If pm_debug_messages_on is disabled and CONFIG_DYNAMIC_DEBUG is enabled, * print message only from instances explicitly enabled on dynamic debug's * control. -- cgit v1.2.3 From a3bbdc52c38fa95488ca713e54bcb40699c26acf Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 9 Jun 2023 11:02:16 +0100 Subject: Remove file->f_op->sendpage Remove file->f_op->sendpage as splicing to a socket now calls sendmsg rather than sendpage. Signed-off-by: David Howells cc: Jens Axboe cc: Matthew Wilcox Signed-off-by: Jakub Kicinski --- include/linux/fs.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index de2cb1132f07..67998c64556d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1790,7 +1790,6 @@ struct file_operations { int (*fsync) (struct file *, loff_t, loff_t, int datasync); int (*fasync) (int, struct file *, int); int (*lock) (struct file *, int, struct file_lock *); - ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); int (*flock) (struct file *, int, struct file_lock *); -- cgit v1.2.3 From 5df5dd03a8f71ca9640f208d8f523856e1069ee7 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 9 Jun 2023 11:02:18 +0100 Subject: sunrpc: Use sendmsg(MSG_SPLICE_PAGES) rather then sendpage When transmitting data, call down into TCP using sendmsg with MSG_SPLICE_PAGES to indicate that content should be spliced rather than performing sendpage calls to transmit header, data pages and trailer. Signed-off-by: David Howells Acked-by: Chuck Lever cc: Trond Myklebust cc: Anna Schumaker cc: Jeff Layton cc: Jens Axboe cc: Matthew Wilcox Signed-off-by: Jakub Kicinski --- include/linux/sunrpc/svc.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 762d7231e574..f66ec8fdb331 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -161,16 +161,15 @@ static inline bool svc_put_not_last(struct svc_serv *serv) extern u32 svc_max_payload(const struct svc_rqst *rqstp); /* - * RPC Requsts and replies are stored in one or more pages. + * RPC Requests and replies are stored in one or more pages. * We maintain an array of pages for each server thread. * Requests are copied into these pages as they arrive. Remaining * pages are available to write the reply into. * - * Pages are sent using ->sendpage so each server thread needs to - * allocate more to replace those used in sending. To help keep track - * of these pages we have a receive list where all pages initialy live, - * and a send list where pages are moved to when there are to be part - * of a reply. + * Pages are sent using ->sendmsg with MSG_SPLICE_PAGES so each server thread + * needs to allocate more to replace those used in sending. To help keep track + * of these pages we have a receive list where all pages initialy live, and a + * send list where pages are moved to when there are to be part of a reply. * * We use xdr_buf for holding responses as it fits well with NFS * read responses (that have a header, and some data pages, and possibly -- cgit v1.2.3 From 7d0b80647f73a170dd20d18fbf01de0f770ed7c8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 5 Jun 2023 15:54:11 +0300 Subject: gpiolib: remove unused gpio_cansleep() There is not a single user in the entire kernel of this deprecated API, kill it for good. Signed-off-by: Andy Shevchenko Reviewed-by: Yanteng Si Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- include/linux/gpio.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gpio.h b/include/linux/gpio.h index 88efac969754..7ecc25c543ce 100644 --- a/include/linux/gpio.h +++ b/include/linux/gpio.h @@ -108,11 +108,6 @@ static inline void gpio_set_value(unsigned gpio, int value) return gpiod_set_raw_value(gpio_to_desc(gpio), value); } -static inline int gpio_cansleep(unsigned gpio) -{ - return gpiod_cansleep(gpio_to_desc(gpio)); -} - static inline int gpio_to_irq(unsigned gpio) { return gpiod_to_irq(gpio_to_desc(gpio)); @@ -195,13 +190,6 @@ static inline void gpio_set_value(unsigned gpio, int value) WARN_ON(1); } -static inline int gpio_cansleep(unsigned gpio) -{ - /* GPIO can never have been requested or set as {in,out}put */ - WARN_ON(1); - return 0; -} - static inline int gpio_get_value_cansleep(unsigned gpio) { /* GPIO can never have been requested or set as {in,out}put */ -- cgit v1.2.3 From 158826c73d48097f843bacc1bcafa6dbc114f4e5 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Tue, 6 Jun 2023 19:16:25 +0530 Subject: soc: qcom: socinfo: Add support for new fields in revision 18 Add support for below fields coming in socinfo structure under v18: * num_kvps: number of key value pairs (KVP) * kvps_offset: the offset of the KVP table from the base address of socinfo structure in SMEM KVP table has boolean values for certain feature flags, used to determine hardware configuration. Signed-off-by: Naman Jain Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20230606134626.18790-2-quic_namajain@quicinc.com --- include/linux/soc/qcom/socinfo.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/socinfo.h b/include/linux/soc/qcom/socinfo.h index d1cbc49a2a2d..3cc266d8a8b4 100644 --- a/include/linux/soc/qcom/socinfo.h +++ b/include/linux/soc/qcom/socinfo.h @@ -65,6 +65,9 @@ struct socinfo { __le32 nnum_partname_mapping; /* Version 17 */ __le32 oem_variant; + /* Version 18 */ + __le32 num_kvps; + __le32 kvps_offset; }; #endif -- cgit v1.2.3 From d9c2a255cfe026c8bf85a39631734f022ecefaff Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Tue, 6 Jun 2023 19:16:26 +0530 Subject: soc: qcom: socinfo: Add support for new fields in revision 19 Add support for below fields coming in socinfo structure under v19: * num_func_clusters: number of clusters with at least one functional core * boot_cluster: cluster selected as boot cluster * boot_core: core selected as boot core While at it, rename some variables to align them with their functionalities. Signed-off-by: Naman Jain Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20230606134626.18790-3-quic_namajain@quicinc.com --- include/linux/soc/qcom/socinfo.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/socinfo.h b/include/linux/soc/qcom/socinfo.h index 3cc266d8a8b4..e78777bb0f4a 100644 --- a/include/linux/soc/qcom/socinfo.h +++ b/include/linux/soc/qcom/socinfo.h @@ -54,8 +54,8 @@ struct socinfo { /* Version 14 */ __le32 num_clusters; __le32 ncluster_array_offset; - __le32 num_defective_parts; - __le32 ndefective_parts_array_offset; + __le32 num_subset_parts; + __le32 nsubset_parts_array_offset; /* Version 15 */ __le32 nmodem_supported; /* Version 16 */ @@ -68,6 +68,10 @@ struct socinfo { /* Version 18 */ __le32 num_kvps; __le32 kvps_offset; + /* Version 19 */ + __le32 num_func_clusters; + __le32 boot_cluster; + __le32 boot_core; }; #endif -- cgit v1.2.3 From 904e6ddf4133c52fdb9654c2cd2ad90f320d48b9 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Jun 2023 18:38:21 +0300 Subject: bpf: Use scalar ids in mark_chain_precision() Change mark_chain_precision() to track precision in situations like below: r2 = unknown value ... --- state #0 --- ... r1 = r2 // r1 and r2 now share the same ID ... --- state #1 {r1.id = A, r2.id = A} --- ... if (r2 > 10) goto exit; // find_equal_scalars() assigns range to r1 ... --- state #2 {r1.id = A, r2.id = A} --- r3 = r10 r3 += r1 // need to mark both r1 and r2 At the beginning of the processing of each state, ensure that if a register with a scalar ID is marked as precise, all registers sharing this ID are also marked as precise. This property would be used by a follow-up change in regsafe(). Signed-off-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20230613153824.3324830-2-eddyz87@gmail.com --- include/linux/bpf_verifier.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5b11a3b0fec0..22fb13c738a9 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -557,6 +557,11 @@ struct backtrack_state { u64 stack_masks[MAX_CALL_FRAMES]; }; +struct bpf_idset { + u32 count; + u32 ids[BPF_ID_MAP_SIZE]; +}; + /* single container for all structs * one verifier_env per bpf_check() call */ @@ -588,7 +593,10 @@ struct bpf_verifier_env { const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; - struct bpf_id_pair idmap_scratch[BPF_ID_MAP_SIZE]; + union { + struct bpf_id_pair idmap_scratch[BPF_ID_MAP_SIZE]; + struct bpf_idset idset_scratch; + }; struct { int *insn_state; int *insn_stack; -- cgit v1.2.3 From 1ffc85d9298e0ca0137ba65c93a786143fe167b8 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Tue, 13 Jun 2023 18:38:23 +0300 Subject: bpf: Verify scalar ids mapping in regsafe() using check_ids() Make sure that the following unsafe example is rejected by verifier: 1: r9 = ... some pointer with range X ... 2: r6 = ... unbound scalar ID=a ... 3: r7 = ... unbound scalar ID=b ... 4: if (r6 > r7) goto +1 5: r6 = r7 6: if (r6 > X) goto ... --- checkpoint --- 7: r9 += r7 8: *(u64 *)r9 = Y This example is unsafe because not all execution paths verify r7 range. Because of the jump at (4) the verifier would arrive at (6) in two states: I. r6{.id=b}, r7{.id=b} via path 1-6; II. r6{.id=a}, r7{.id=b} via path 1-4, 6. Currently regsafe() does not call check_ids() for scalar registers, thus from POV of regsafe() states (I) and (II) are identical. If the path 1-6 is taken by verifier first, and checkpoint is created at (6) the path [1-4, 6] would be considered safe. Changes in this commit: - check_ids() is modified to disallow mapping multiple old_id to the same cur_id. - check_scalar_ids() is added, unlike check_ids() it treats ID zero as a unique scalar ID. - check_scalar_ids() needs to generate temporary unique IDs, field 'tmp_id_gen' is added to bpf_verifier_env::idmap_scratch to facilitate this. - regsafe() is updated to: - use check_scalar_ids() for precise scalar registers. - compare scalar registers using memcmp only for explore_alu_limits branch. This simplifies control flow for scalar case, and has no measurable performance impact. - check_alu_op() is updated to avoid generating bpf_reg_state::id for constant scalar values when processing BPF_MOV. ID is needed to propagate range information for identical values, but there is nothing to propagate for constants. Fixes: 75748837b7e5 ("bpf: Propagate scalar ranges through register assignments.") Signed-off-by: Eduard Zingerman Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20230613153824.3324830-4-eddyz87@gmail.com --- include/linux/bpf_verifier.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 22fb13c738a9..f70f9ac884d2 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -313,11 +313,6 @@ struct bpf_idx_pair { u32 idx; }; -struct bpf_id_pair { - u32 old; - u32 cur; -}; - #define MAX_CALL_FRAMES 8 /* Maximum number of register states that can exist at once */ #define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) * MAX_CALL_FRAMES) @@ -557,6 +552,16 @@ struct backtrack_state { u64 stack_masks[MAX_CALL_FRAMES]; }; +struct bpf_id_pair { + u32 old; + u32 cur; +}; + +struct bpf_idmap { + u32 tmp_id_gen; + struct bpf_id_pair map[BPF_ID_MAP_SIZE]; +}; + struct bpf_idset { u32 count; u32 ids[BPF_ID_MAP_SIZE]; @@ -594,7 +599,7 @@ struct bpf_verifier_env { struct bpf_verifier_log log; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; union { - struct bpf_id_pair idmap_scratch[BPF_ID_MAP_SIZE]; + struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; }; struct { -- cgit v1.2.3 From 18c0ffb404db2093b6afdc8ae15f18ba3975e1ed Mon Sep 17 00:00:00 2001 From: Gregory Greenman Date: Tue, 13 Jun 2023 15:57:19 +0300 Subject: wifi: iwlwifi: mvm: add support for Extra EHT LTF Add support for Extra EHT LTF defined in 9.4.2.313 EHT Capabilities element. Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230613155501.de019d7cc174.I806f0f6042b89274192701a60b4f7900822db666@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 516cd32d6196..5dfed1a6625c 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2859,6 +2859,7 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) /* Maximum number of supported EHT LTF is split */ #define IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK 0xc0 +#define IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF 0x40 #define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK 0x07 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK 0x78 -- cgit v1.2.3 From 65bae54e08c109ddbbf121bb00058cf3b3fb7b8e Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Fri, 9 Jun 2023 16:30:00 +0800 Subject: regulator: mt6358: Merge VCN33_* regulators The VCN33_BT and VCN33_WIFI regulators are actually the same regulator, having the same voltage setting and output pin. There are simply two enable bits that are ORed together to enable the regulator. Having two regulators representing the same output pin is misleading from a design matching standpoint, and also error-prone in driver implementations. If consumers try to set different voltages on either regulator, the one set later would override the one set before. There are ways around this, such as chaining them together and having the downstream one act as a switch. But given there's only one output pin, such a workaround doesn't match reality. Remove the VCN33_WIFI regulator. During the probe phase, have the driver sync the enable status of VCN33_WIFI to VCN33_BT. Also drop the suffix so that the regulator name matches the pin name in the datasheet. Signed-off-by: Chen-Yu Tsai Link: https://lore.kernel.org/r/20230609083009.2822259-4-wenst@chromium.org Signed-off-by: Mark Brown --- include/linux/regulator/mt6358-regulator.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/regulator/mt6358-regulator.h b/include/linux/regulator/mt6358-regulator.h index bdcf83cd719e..a4307cd9edd6 100644 --- a/include/linux/regulator/mt6358-regulator.h +++ b/include/linux/regulator/mt6358-regulator.h @@ -41,8 +41,7 @@ enum { MT6358_ID_VIO28, MT6358_ID_VA12, MT6358_ID_VRF18, - MT6358_ID_VCN33_BT, - MT6358_ID_VCN33_WIFI, + MT6358_ID_VCN33, MT6358_ID_VCAMA2, MT6358_ID_VMC, MT6358_ID_VLDO28, @@ -85,8 +84,7 @@ enum { MT6366_ID_VIO28, MT6366_ID_VA12, MT6366_ID_VRF18, - MT6366_ID_VCN33_BT, - MT6366_ID_VCN33_WIFI, + MT6366_ID_VCN33, MT6366_ID_VMC, MT6366_ID_VAUD28, MT6366_ID_VSIM2, -- cgit v1.2.3 From 04ba665248ed91576d326041108e5fc2ec2254eb Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Fri, 9 Jun 2023 16:30:01 +0800 Subject: regulator: mt6358: Drop *_SSHUB regulators The *_SSHUB regulators are actually alternate configuration interfaces for their non *_SSHUB counterparts. They are not separate regulator outputs. These registers are intended for the companion processor to use to configure the power rails while the main processor is sleeping. They are not intended for the main operating system to use. Since they are not real outputs they shouldn't be modeled separately. Remove them. Luckily no device tree actually uses them. Signed-off-by: Chen-Yu Tsai Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Matthias Brugger Link: https://lore.kernel.org/r/20230609083009.2822259-5-wenst@chromium.org Signed-off-by: Mark Brown --- include/linux/regulator/mt6358-regulator.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/regulator/mt6358-regulator.h b/include/linux/regulator/mt6358-regulator.h index a4307cd9edd6..c71a6a9fce7a 100644 --- a/include/linux/regulator/mt6358-regulator.h +++ b/include/linux/regulator/mt6358-regulator.h @@ -47,8 +47,6 @@ enum { MT6358_ID_VLDO28, MT6358_ID_VAUD28, MT6358_ID_VSIM2, - MT6358_ID_VCORE_SSHUB, - MT6358_ID_VSRAM_OTHERS_SSHUB, MT6358_ID_RG_MAX, }; @@ -88,8 +86,6 @@ enum { MT6366_ID_VMC, MT6366_ID_VAUD28, MT6366_ID_VSIM2, - MT6366_ID_VCORE_SSHUB, - MT6366_ID_VSRAM_OTHERS_SSHUB, MT6366_ID_RG_MAX, }; -- cgit v1.2.3 From 74836ecbc5c7565d24a770917644e96af3e98d25 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 12 Jun 2023 12:00:47 -0700 Subject: fsverity: rework fsverity_get_digest() again Address several issues with the calling convention and documentation of fsverity_get_digest(): - Make it provide the hash algorithm as either a FS_VERITY_HASH_ALG_* value or HASH_ALGO_* value, at the caller's choice, rather than only a HASH_ALGO_* value as it did before. This allows callers to work with the fsverity native algorithm numbers if they want to. HASH_ALGO_* is what IMA uses, but other users (e.g. overlayfs) should use FS_VERITY_HASH_ALG_* to match fsverity-utils and the fsverity UAPI. - Make it return the digest size so that it doesn't need to be looked up separately. Use the return value for this, since 0 works nicely for the "file doesn't have fsverity enabled" case. This also makes it clear that no other errors are possible. - Rename the 'digest' parameter to 'raw_digest' and clearly document that it is only useful in combination with the algorithm ID. This hopefully clears up a point of confusion. - Export it to modules, since overlayfs will need it for checking the fsverity digests of lowerdata files (https://lore.kernel.org/r/dd294a44e8f401e6b5140029d8355f88748cd8fd.1686565330.git.alexl@redhat.com). Acked-by: Mimi Zohar # for the IMA piece Link: https://lore.kernel.org/r/20230612190047.59755-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/linux/fsverity.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index e76605d5b36e..1eb7eae580be 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -143,8 +143,8 @@ int fsverity_ioctl_enable(struct file *filp, const void __user *arg); int fsverity_ioctl_measure(struct file *filp, void __user *arg); int fsverity_get_digest(struct inode *inode, - u8 digest[FS_VERITY_MAX_DIGEST_SIZE], - enum hash_algo *alg); + u8 raw_digest[FS_VERITY_MAX_DIGEST_SIZE], + u8 *alg, enum hash_algo *halg); /* open.c */ @@ -197,10 +197,14 @@ static inline int fsverity_ioctl_measure(struct file *filp, void __user *arg) } static inline int fsverity_get_digest(struct inode *inode, - u8 digest[FS_VERITY_MAX_DIGEST_SIZE], - enum hash_algo *alg) + u8 raw_digest[FS_VERITY_MAX_DIGEST_SIZE], + u8 *alg, enum hash_algo *halg) { - return -EOPNOTSUPP; + /* + * fsverity is not enabled in the kernel configuration, so always report + * that the file doesn't have fsverity enabled (digest size 0). + */ + return 0; } /* open.c */ -- cgit v1.2.3 From cbe7cff4a76bc749dd70264ca5cf924e2adf9296 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Sat, 10 Jun 2023 10:20:01 +0800 Subject: blktrace: use inline function for blk_trace_remove() while blktrace is disabled If config is disabled, call blk_trace_remove() directly will trigger build warning, hence use inline function instead, prepare to fix blktrace debugfs entries leakage. Signed-off-by: Yu Kuai Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20230610022003.2557284-2-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- include/linux/blktrace_api.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h index cfbda114348c..122c62e561fc 100644 --- a/include/linux/blktrace_api.h +++ b/include/linux/blktrace_api.h @@ -85,10 +85,14 @@ extern int blk_trace_remove(struct request_queue *q); # define blk_add_driver_data(rq, data, len) do {} while (0) # define blk_trace_setup(q, name, dev, bdev, arg) (-ENOTTY) # define blk_trace_startstop(q, start) (-ENOTTY) -# define blk_trace_remove(q) (-ENOTTY) # define blk_add_trace_msg(q, fmt, ...) do { } while (0) # define blk_add_cgroup_trace_msg(q, cg, fmt, ...) do { } while (0) # define blk_trace_note_message_enabled(q) (false) + +static inline int blk_trace_remove(struct request_queue *q) +{ + return -ENOTTY; +} #endif /* CONFIG_BLK_DEV_IO_TRACE */ #ifdef CONFIG_COMPAT -- cgit v1.2.3 From 70f7457ad6d655e65f1b93cbba2a519e4b11c946 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 12 Jun 2023 14:49:43 -0700 Subject: net: create device lookup API with reference tracking New users of dev_get_by_index() and dev_get_by_name() keep getting added and it would be nice to steer them towards the APIs with reference tracking. Add variants of those calls which allocate the reference tracker and use them in a couple of places. Signed-off-by: Jakub Kicinski Reviewed-by: Eric Dumazet Reviewed-by: David Ahern Signed-off-by: David S. Miller --- include/linux/netdevice.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 2d6cb2bf2f05..acf706d49c2b 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3124,6 +3124,10 @@ struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev, struct sock *sk); struct net_device *dev_get_by_index(struct net *net, int ifindex); struct net_device *__dev_get_by_index(struct net *net, int ifindex); +struct net_device *netdev_get_by_index(struct net *net, int ifindex, + netdevice_tracker *tracker, gfp_t gfp); +struct net_device *netdev_get_by_name(struct net *net, const char *name, + netdevice_tracker *tracker, gfp_t gfp); struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); struct net_device *dev_get_by_napi_id(unsigned int napi_id); int dev_restart(struct net_device *dev); -- cgit v1.2.3 From cf683e8870bd4be0fd6b98639286700a35088660 Mon Sep 17 00:00:00 2001 From: Thomas Zimmermann Date: Wed, 14 Jun 2023 15:12:12 +0200 Subject: fbdev: Use /* */ comment in initializer macro MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use /* */ in initializer macro to avoid out-commenting the comma at the end of the line. Reported-by: Christian König Closes: https://lore.kernel.org/dri-devel/20230530150253.22758-1-tzimmermann@suse.de/T/#m356cda2679c17d7a01f30ce2b5282cd9046ea6d4 Fixes: f1061fa641b8 ("fbdev: Add initializer macros for struct fb_ops") Cc: Sam Ravnborg Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Signed-off-by: Thomas Zimmermann Reviewed-by: Christian König Link: https://patchwork.freedesktop.org/patch/msgid/20230614131253.10208-1-tzimmermann@suse.de --- include/linux/fb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fb.h b/include/linux/fb.h index ce6823e157e6..ce7d588edc3e 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -552,7 +552,7 @@ extern ssize_t fb_io_write(struct fb_info *info, const char __user *buf, .fb_imageblit = cfb_imageblit #define __FB_DEFAULT_IO_OPS_MMAP \ - .fb_mmap = NULL // default implementation + .fb_mmap = NULL /* default implementation */ #define FB_DEFAULT_IO_OPS \ __FB_DEFAULT_IO_OPS_RDWR, \ @@ -585,7 +585,7 @@ extern ssize_t fb_sys_write(struct fb_info *info, const char __user *buf, .fb_imageblit = sys_imageblit #define __FB_DEFAULT_SYS_OPS_MMAP \ - .fb_mmap = NULL // default implementation + .fb_mmap = NULL /* default implementation */ #define FB_DEFAULT_SYS_OPS \ __FB_DEFAULT_SYS_OPS_RDWR, \ -- cgit v1.2.3 From 2d8c9dcf7158060fcec9f891c0292ffdb4397523 Mon Sep 17 00:00:00 2001 From: Wen Yang Date: Thu, 15 Jun 2023 02:40:28 +0800 Subject: eventfd: add a uapi header for eventfd userspace APIs Create a uapi header include/uapi/linux/eventfd.h, move the associated flags to the uapi header, and include it from linux/eventfd.h. Suggested-by: Christian Brauner Signed-off-by: Wen Yang Reviewed-by: Matthew Wilcox (Oracle) Cc: Alexander Viro Cc: Jens Axboe Cc: Christian Brauner Cc: Christoph Hellwig Cc: Dylan Yudaken Cc: David Woodhouse Cc: Matthew Wilcox Cc: Eric Biggers Cc: linux-fsdevel@vger.kernel.org Cc: linux-kernel@vger.kernel.org Message-Id: Signed-off-by: Christian Brauner --- include/linux/eventfd.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/eventfd.h b/include/linux/eventfd.h index 98d31cdaca40..b9d83652c097 100644 --- a/include/linux/eventfd.h +++ b/include/linux/eventfd.h @@ -9,12 +9,12 @@ #ifndef _LINUX_EVENTFD_H #define _LINUX_EVENTFD_H -#include #include #include #include #include #include +#include /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining @@ -23,10 +23,6 @@ * from eventfd, in order to leave a free define-space for * shared O_* flags. */ -#define EFD_SEMAPHORE (1 << 0) -#define EFD_CLOEXEC O_CLOEXEC -#define EFD_NONBLOCK O_NONBLOCK - #define EFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) #define EFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS | EFD_SEMAPHORE) -- cgit v1.2.3 From b4a11fa3331e163e177e76098fe1d8b12b87cf6b Mon Sep 17 00:00:00 2001 From: Wyes Karny Date: Mon, 29 May 2023 14:25:51 +0000 Subject: cpufreq: Fail driver register if it has adjust_perf without fast_switch If fast_switch_possible flag is set by the scaling driver, the governor is free to select fast_switch function even if adjust_perf is set. Some scaling drivers which use adjust_perf don't set fast_switch thinking that the governor would never fall back to fast_switch. But the governor can fall back to fast_switch even in runtime if frequency invariance is disabled due to some reason. This could crash the kernel if the driver didn't set the fast_switch function pointer. Therefore, fail driver registration if it has adjust_perf without fast_switch. Suggested-by: Rafael J. Wysocki Suggested-by: Viresh Kumar Signed-off-by: Wyes Karny Signed-off-by: Rafael J. Wysocki --- include/linux/cpufreq.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 26e2eb399484..172ff51c1b2a 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -340,7 +340,10 @@ struct cpufreq_driver { /* * ->fast_switch() replacement for drivers that use an internal * representation of performance levels and can pass hints other than - * the target performance level to the hardware. + * the target performance level to the hardware. This can only be set + * if ->fast_switch is set too, because in those cases (under specific + * conditions) scale invariance can be disabled, which causes the + * schedutil governor to fall back to the latter. */ void (*adjust_perf)(unsigned int cpu, unsigned long min_perf, -- cgit v1.2.3 From e1d001fa5b477c4da46a29be1fcece91db7c7c6f Mon Sep 17 00:00:00 2001 From: Breno Leitao Date: Fri, 9 Jun 2023 08:27:42 -0700 Subject: net: ioctl: Use kernel memory on protocol ioctl callbacks Most of the ioctls to net protocols operates directly on userspace argument (arg). Usually doing get_user()/put_user() directly in the ioctl callback. This is not flexible, because it is hard to reuse these functions without passing userspace buffers. Change the "struct proto" ioctls to avoid touching userspace memory and operate on kernel buffers, i.e., all protocol's ioctl callbacks is adapted to operate on a kernel memory other than on userspace (so, no more {put,get}_user() and friends being called in the ioctl callback). This changes the "struct proto" ioctl format in the following way: int (*ioctl)(struct sock *sk, int cmd, - unsigned long arg); + int *karg); (Important to say that this patch does not touch the "struct proto_ops" protocols) So, the "karg" argument, which is passed to the ioctl callback, is a pointer allocated to kernel space memory (inside a function wrapper). This buffer (karg) may contain input argument (copied from userspace in a prep function) and it might return a value/buffer, which is copied back to userspace if necessary. There is not one-size-fits-all format (that is I am using 'may' above), but basically, there are three type of ioctls: 1) Do not read from userspace, returns a result to userspace 2) Read an input parameter from userspace, and does not return anything to userspace 3) Read an input from userspace, and return a buffer to userspace. The default case (1) (where no input parameter is given, and an "int" is returned to userspace) encompasses more than 90% of the cases, but there are two other exceptions. Here is a list of exceptions: * Protocol RAW: * cmd = SIOCGETVIFCNT: * input and output = struct sioc_vif_req * cmd = SIOCGETSGCNT * input and output = struct sioc_sg_req * Explanation: for the SIOCGETVIFCNT case, userspace passes the input argument, which is struct sioc_vif_req. Then the callback populates the struct, which is copied back to userspace. * Protocol RAW6: * cmd = SIOCGETMIFCNT_IN6 * input and output = struct sioc_mif_req6 * cmd = SIOCGETSGCNT_IN6 * input and output = struct sioc_sg_req6 * Protocol PHONET: * cmd == SIOCPNADDRESOURCE | SIOCPNDELRESOURCE * input int (4 bytes) * Nothing is copied back to userspace. For the exception cases, functions sock_sk_ioctl_inout() will copy the userspace input, and copy it back to kernel space. The wrapper that prepare the buffer and put the buffer back to user is sk_ioctl(), so, instead of calling sk->sk_prot->ioctl(), the callee now calls sk_ioctl(), which will handle all cases. Signed-off-by: Breno Leitao Reviewed-by: Willem de Bruijn Reviewed-by: David Ahern Reviewed-by: Kuniyuki Iwashima Link: https://lore.kernel.org/r/20230609152800.830401-1-leitao@debian.org Signed-off-by: Jakub Kicinski --- include/linux/icmpv6.h | 6 ++++++ include/linux/mroute.h | 22 ++++++++++++++++++++-- include/linux/mroute6.h | 31 +++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index db0f4fcfdaf4..1fe33e6741cc 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -111,4 +111,10 @@ static inline bool icmpv6_is_err(int type) return false; } +static inline int sk_is_icmpv6(struct sock *sk) +{ + return sk->sk_family == AF_INET6 && + inet_sk(sk)->inet_num == IPPROTO_ICMPV6; +} + #endif diff --git a/include/linux/mroute.h b/include/linux/mroute.h index 80b8400ab8b2..94c6e6f549f0 100644 --- a/include/linux/mroute.h +++ b/include/linux/mroute.h @@ -16,12 +16,19 @@ static inline int ip_mroute_opt(int opt) return opt >= MRT_BASE && opt <= MRT_MAX; } +static inline int sk_is_ipmr(struct sock *sk) +{ + return sk->sk_family == AF_INET && + inet_sk(sk)->inet_num == IPPROTO_IGMP; +} + int ip_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int); int ip_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t); -int ipmr_ioctl(struct sock *sk, int cmd, void __user *arg); +int ipmr_ioctl(struct sock *sk, int cmd, void *arg); int ipmr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg); int ip_mr_init(void); bool ipmr_rule_default(const struct fib_rule *rule); +int ipmr_sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg); #else static inline int ip_mroute_setsockopt(struct sock *sock, int optname, sockptr_t optval, unsigned int optlen) @@ -35,7 +42,7 @@ static inline int ip_mroute_getsockopt(struct sock *sk, int optname, return -ENOPROTOOPT; } -static inline int ipmr_ioctl(struct sock *sk, int cmd, void __user *arg) +static inline int ipmr_ioctl(struct sock *sk, int cmd, void *arg) { return -ENOIOCTLCMD; } @@ -50,10 +57,21 @@ static inline int ip_mroute_opt(int opt) return 0; } +static inline int sk_is_ipmr(struct sock *sk) +{ + return 0; +} + static inline bool ipmr_rule_default(const struct fib_rule *rule) { return true; } + +static inline int ipmr_sk_ioctl(struct sock *sk, unsigned int cmd, + void __user *arg) +{ + return 1; +} #endif #define VIFF_STATIC 0x8000 diff --git a/include/linux/mroute6.h b/include/linux/mroute6.h index 8f2b307fb124..2f95d5b4e47a 100644 --- a/include/linux/mroute6.h +++ b/include/linux/mroute6.h @@ -29,10 +29,10 @@ struct sock; extern int ip6_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int); extern int ip6_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t); extern int ip6_mr_input(struct sk_buff *skb); -extern int ip6mr_ioctl(struct sock *sk, int cmd, void __user *arg); extern int ip6mr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg); extern int ip6_mr_init(void); extern void ip6_mr_cleanup(void); +int ip6mr_ioctl(struct sock *sk, int cmd, void *arg); #else static inline int ip6_mroute_setsockopt(struct sock *sock, int optname, sockptr_t optval, unsigned int optlen) @@ -48,7 +48,7 @@ int ip6_mroute_getsockopt(struct sock *sock, } static inline -int ip6mr_ioctl(struct sock *sk, int cmd, void __user *arg) +int ip6mr_ioctl(struct sock *sk, int cmd, void *arg) { return -ENOIOCTLCMD; } @@ -100,6 +100,27 @@ extern int ip6mr_get_route(struct net *net, struct sk_buff *skb, #ifdef CONFIG_IPV6_MROUTE bool mroute6_is_socket(struct net *net, struct sk_buff *skb); extern int ip6mr_sk_done(struct sock *sk); +static inline int ip6mr_sk_ioctl(struct sock *sk, unsigned int cmd, + void __user *arg) +{ + switch (cmd) { + /* These userspace buffers will be consumed by ip6mr_ioctl() */ + case SIOCGETMIFCNT_IN6: { + struct sioc_mif_req6 buffer; + + return sock_ioctl_inout(sk, cmd, arg, &buffer, + sizeof(buffer)); + } + case SIOCGETSGCNT_IN6: { + struct sioc_mif_req6 buffer; + + return sock_ioctl_inout(sk, cmd, arg, &buffer, + sizeof(buffer)); + } + } + + return 1; +} #else static inline bool mroute6_is_socket(struct net *net, struct sk_buff *skb) { @@ -109,5 +130,11 @@ static inline int ip6mr_sk_done(struct sock *sk) { return 0; } + +static inline int ip6mr_sk_ioctl(struct sock *sk, unsigned int cmd, + void __user *arg) +{ + return 1; +} #endif #endif -- cgit v1.2.3 From 7725acaa4f0c04fbefb0e0d342635b967bb7d414 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 14 Jun 2023 01:39:22 +0200 Subject: init: Provide arch_cpu_finalize_init() check_bugs() has become a dumping ground for all sorts of activities to finalize the CPU initialization before running the rest of the init code. Most are empty, a few do actual bug checks, some do alternative patching and some cobble a CPU advertisement string together.... Aside of that the current implementation requires duplicated function declaration and mostly empty header files for them. Provide a new function arch_cpu_finalize_init(). Provide a generic declaration if CONFIG_ARCH_HAS_CPU_FINALIZE_INIT is selected and a stub inline otherwise. This requires a temporary #ifdef in start_kernel() which will be removed along with check_bugs() once the architectures are converted over. Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20230613224544.957805717@linutronix.de --- include/linux/cpu.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 8582a7142623..4893c4ac026d 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -184,6 +184,12 @@ void arch_cpu_idle_enter(void); void arch_cpu_idle_exit(void); void __noreturn arch_cpu_idle_dead(void); +#ifdef CONFIG_ARCH_HAS_CPU_FINALIZE_INIT +void arch_cpu_finalize_init(void); +#else +static inline void arch_cpu_finalize_init(void) { } +#endif + int cpu_report_state(int cpu); int cpu_check_up_prepare(int cpu); void cpu_set_state_online(int cpu); -- cgit v1.2.3 From 0cfb4a1af386427cdaba98f18f501eb074985cfd Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Jun 2023 14:58:52 +0100 Subject: genirq: Use BIT() for the IRQD_* state flags As we're about to use the last bit available in the IRQD_* state flags, rewrite these flags with BIT(), which ensures that these constant do not represent a signed value. Signed-off-by: Marc Zyngier --- include/linux/irq.h | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/irq.h b/include/linux/irq.h index b1b28affb32a..d9c86db69982 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -226,29 +226,29 @@ struct irq_data { */ enum { IRQD_TRIGGER_MASK = 0xf, - IRQD_SETAFFINITY_PENDING = (1 << 8), - IRQD_ACTIVATED = (1 << 9), - IRQD_NO_BALANCING = (1 << 10), - IRQD_PER_CPU = (1 << 11), - IRQD_AFFINITY_SET = (1 << 12), - IRQD_LEVEL = (1 << 13), - IRQD_WAKEUP_STATE = (1 << 14), - IRQD_MOVE_PCNTXT = (1 << 15), - IRQD_IRQ_DISABLED = (1 << 16), - IRQD_IRQ_MASKED = (1 << 17), - IRQD_IRQ_INPROGRESS = (1 << 18), - IRQD_WAKEUP_ARMED = (1 << 19), - IRQD_FORWARDED_TO_VCPU = (1 << 20), - IRQD_AFFINITY_MANAGED = (1 << 21), - IRQD_IRQ_STARTED = (1 << 22), - IRQD_MANAGED_SHUTDOWN = (1 << 23), - IRQD_SINGLE_TARGET = (1 << 24), - IRQD_DEFAULT_TRIGGER_SET = (1 << 25), - IRQD_CAN_RESERVE = (1 << 26), - IRQD_MSI_NOMASK_QUIRK = (1 << 27), - IRQD_HANDLE_ENFORCE_IRQCTX = (1 << 28), - IRQD_AFFINITY_ON_ACTIVATE = (1 << 29), - IRQD_IRQ_ENABLED_ON_SUSPEND = (1 << 30), + IRQD_SETAFFINITY_PENDING = BIT(8), + IRQD_ACTIVATED = BIT(9), + IRQD_NO_BALANCING = BIT(10), + IRQD_PER_CPU = BIT(11), + IRQD_AFFINITY_SET = BIT(12), + IRQD_LEVEL = BIT(13), + IRQD_WAKEUP_STATE = BIT(14), + IRQD_MOVE_PCNTXT = BIT(15), + IRQD_IRQ_DISABLED = BIT(16), + IRQD_IRQ_MASKED = BIT(17), + IRQD_IRQ_INPROGRESS = BIT(18), + IRQD_WAKEUP_ARMED = BIT(19), + IRQD_FORWARDED_TO_VCPU = BIT(20), + IRQD_AFFINITY_MANAGED = BIT(21), + IRQD_IRQ_STARTED = BIT(22), + IRQD_MANAGED_SHUTDOWN = BIT(23), + IRQD_SINGLE_TARGET = BIT(24), + IRQD_DEFAULT_TRIGGER_SET = BIT(25), + IRQD_CAN_RESERVE = BIT(26), + IRQD_MSI_NOMASK_QUIRK = BIT(27), + IRQD_HANDLE_ENFORCE_IRQCTX = BIT(28), + IRQD_AFFINITY_ON_ACTIVATE = BIT(29), + IRQD_IRQ_ENABLED_ON_SUSPEND = BIT(30), }; #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) -- cgit v1.2.3 From 9c15eeb5362c48dd27d51bd72e8873341fa9383c Mon Sep 17 00:00:00 2001 From: James Gowans Date: Thu, 8 Jun 2023 14:00:20 +0200 Subject: genirq: Allow fasteoi handler to resend interrupts on concurrent handling There is a class of interrupt controllers out there that, once they have signalled a given interrupt number, will still signal incoming instances of the *same* interrupt despite the original interrupt not having been EOIed yet. As long as the new interrupt reaches the *same* CPU, nothing bad happens, as that CPU still has its interrupts globally disabled, and we will only take the new interrupt once the interrupt has been EOIed. However, things become more "interesting" if an affinity change comes in while the interrupt is being handled. More specifically, while the per-irq lock is being dropped. This results in the affinity change taking place immediately. At this point, there is nothing that prevents the interrupt from firing on the new target CPU. We end-up with the interrupt running concurrently on two CPUs, which isn't a good thing. And that's where things become worse: the new CPU notices that the interrupt handling is in progress (irq_may_run() return false), and *drops the interrupt on the floor*. The whole race looks like this: CPU 0 | CPU 1 -----------------------------|----------------------------- interrupt start | handle_fasteoi_irq | set_affinity(CPU 1) handler | ... | interrupt start ... | handle_fasteoi_irq -> early out handle_fasteoi_irq return | interrupt end interrupt end | If the interrupt was an edge, too bad. The interrupt is lost, and the system will eventually die one way or another. Not great. A way to avoid this situation is to detect this problem at the point we handle the interrupt on the new target. Instead of dropping the interrupt, use the resend mechanism to force it to be replayed. Also, in order to limit the impact of this workaround to the pathetic architectures that require it, gate it behind a new irq flag aptly named IRQD_RESEND_WHEN_IN_PROGRESS. Suggested-by: Marc Zyngier Signed-off-by: James Gowans Cc: Thomas Gleixner Cc: Marc Zyngier Cc: KarimAllah Raslan Cc: Yipeng Zou Cc: Zhang Jianhua [maz: reworded commit mesage] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230608120021.3273400-3-jgowans@amazon.com --- include/linux/irq.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/irq.h b/include/linux/irq.h index d9c86db69982..d8a6fdce9373 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -223,6 +223,8 @@ struct irq_data { * irq_chip::irq_set_affinity() when deactivated. * IRQD_IRQ_ENABLED_ON_SUSPEND - Interrupt is enabled on suspend by irq pm if * irqchip have flag IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND set. + * IRQD_RESEND_WHEN_IN_PROGRESS - Interrupt may fire when already in progress in which + * case it must be resent at the next available opportunity. */ enum { IRQD_TRIGGER_MASK = 0xf, @@ -249,6 +251,7 @@ enum { IRQD_HANDLE_ENFORCE_IRQCTX = BIT(28), IRQD_AFFINITY_ON_ACTIVATE = BIT(29), IRQD_IRQ_ENABLED_ON_SUSPEND = BIT(30), + IRQD_RESEND_WHEN_IN_PROGRESS = BIT(31), }; #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) @@ -448,6 +451,16 @@ static inline bool irqd_affinity_on_activate(struct irq_data *d) return __irqd_to_state(d) & IRQD_AFFINITY_ON_ACTIVATE; } +static inline void irqd_set_resend_when_in_progress(struct irq_data *d) +{ + __irqd_to_state(d) |= IRQD_RESEND_WHEN_IN_PROGRESS; +} + +static inline bool irqd_needs_resend_when_in_progress(struct irq_data *d) +{ + return __irqd_to_state(d) & IRQD_RESEND_WHEN_IN_PROGRESS; +} + #undef __irqd_to_state static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d) -- cgit v1.2.3 From b33eb50a92b0a298fa8a6ac350e741c3ec100f6d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 15 Jun 2023 14:27:34 +0100 Subject: locking/atomic: scripts: fix ${atomic}_dec_if_positive() kerneldoc The ${atomic}_dec_if_positive() ops are unlike all the other conditional atomic ops. Rather than returning a boolean success value, these return the value that the atomic variable would be updated to, even when no update is performed. We missed this when adding kerneldoc comments, and the documentation for ${atomic}_dec_if_positive() erroneously states: | Return: @true if @v was updated, @false otherwise. Ideally we'd clean this up by aligning ${atomic}_dec_if_positive() with the usual atomic op conventions: with ${atomic}_fetch_dec_if_positive() for those who care about the value of the varaible, and ${atomic}_dec_if_positive() returning a boolean success value. In the mean time, align the documentation with the current reality. Fixes: ad8110706f381170 ("locking/atomic: scripts: generate kerneldoc comments") Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paul E. McKenney Link: https://lore.kernel.org/r/20230615132734.1119765-1-mark.rutland@arm.com --- include/linux/atomic/atomic-arch-fallback.h | 6 +++--- include/linux/atomic/atomic-instrumented.h | 8 ++++---- include/linux/atomic/atomic-long.h | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 8cded57dd7a6..18f5744dfb5d 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -2520,7 +2520,7 @@ raw_atomic_dec_unless_positive(atomic_t *v) * * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere. * - * Return: @true if @v was updated, @false otherwise. + * Return: The old value of (@v - 1), regardless of whether @v was updated. */ static __always_inline int raw_atomic_dec_if_positive(atomic_t *v) @@ -4636,7 +4636,7 @@ raw_atomic64_dec_unless_positive(atomic64_t *v) * * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere. * - * Return: @true if @v was updated, @false otherwise. + * Return: The old value of (@v - 1), regardless of whether @v was updated. */ static __always_inline s64 raw_atomic64_dec_if_positive(atomic64_t *v) @@ -4657,4 +4657,4 @@ raw_atomic64_dec_if_positive(atomic64_t *v) } #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 3916f02c038baa3f5190d275f68b9211667fcc9d +// 202b45c7db600ce36198eb1f1fc2c2d5268ace2d diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index ebfc795f921b..d401b406ef7c 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -1570,7 +1570,7 @@ atomic_dec_unless_positive(atomic_t *v) * * Unsafe to use in noinstr code; use raw_atomic_dec_if_positive() there. * - * Return: @true if @v was updated, @false otherwise. + * Return: The old value of (@v - 1), regardless of whether @v was updated. */ static __always_inline int atomic_dec_if_positive(atomic_t *v) @@ -3134,7 +3134,7 @@ atomic64_dec_unless_positive(atomic64_t *v) * * Unsafe to use in noinstr code; use raw_atomic64_dec_if_positive() there. * - * Return: @true if @v was updated, @false otherwise. + * Return: The old value of (@v - 1), regardless of whether @v was updated. */ static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) @@ -4698,7 +4698,7 @@ atomic_long_dec_unless_positive(atomic_long_t *v) * * Unsafe to use in noinstr code; use raw_atomic_long_dec_if_positive() there. * - * Return: @true if @v was updated, @false otherwise. + * Return: The old value of (@v - 1), regardless of whether @v was updated. */ static __always_inline long atomic_long_dec_if_positive(atomic_long_t *v) @@ -5000,4 +5000,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// 06cec02e676a484857aee38b0071a1d846ec9457 +// 1568f875fef72097413caab8339120c065a39aa4 diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h index f6df2adadf99..c82947170ddc 100644 --- a/include/linux/atomic/atomic-long.h +++ b/include/linux/atomic/atomic-long.h @@ -1782,7 +1782,7 @@ raw_atomic_long_dec_unless_positive(atomic_long_t *v) * * Safe to use in noinstr code; prefer atomic_long_dec_if_positive() elsewhere. * - * Return: @true if @v was updated, @false otherwise. + * Return: The old value of (@v - 1), regardless of whether @v was updated. */ static __always_inline long raw_atomic_long_dec_if_positive(atomic_long_t *v) @@ -1795,4 +1795,4 @@ raw_atomic_long_dec_if_positive(atomic_long_t *v) } #endif /* _LINUX_ATOMIC_LONG_H */ -// 029d2e3a493086671e874a4c2e0e42084be42403 +// 4ef23f98c73cff96d239896175fd26b10b88899e -- cgit v1.2.3 From 0cce0fde499a92c726cd2e24f7763644f7c9f971 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Sat, 3 Jun 2023 15:36:45 +0800 Subject: sched/topology: Mark set_sched_topology() __init All callers of set_sched_topology() are within __init section. Mark it __init too. Signed-off-by: Miaohe Lin Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Valentin Schneider Link: https://lore.kernel.org/r/20230603073645.1173332-1-linmiaohe@huawei.com --- include/linux/sched/topology.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 816df6cc444e..67b573d5bf28 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -203,7 +203,7 @@ struct sched_domain_topology_level { #endif }; -extern void set_sched_topology(struct sched_domain_topology_level *tl); +extern void __init set_sched_topology(struct sched_domain_topology_level *tl); #ifdef CONFIG_SCHED_DEBUG # define SD_INIT_NAME(type) .name = #type -- cgit v1.2.3 From ef73d6a4ef0b35524125c3cfc6deafc26a0c966a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arve=20Hj=C3=B8nnev=C3=A5g?= Date: Fri, 2 Jun 2023 21:23:46 +0000 Subject: sched/wait: Fix a kthread_park race with wait_woken() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kthread_park and wait_woken have a similar race that kthread_stop and wait_woken used to have before it was fixed in commit cb6538e740d7 ("sched/wait: Fix a kthread race with wait_woken()"). Extend that fix to also cover kthread_park. [jstultz: Made changes suggested by Peter to optimize memory loads] Signed-off-by: Arve Hjønnevåg Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Valentin Schneider Link: https://lore.kernel.org/r/20230602212350.535358-1-jstultz@google.com --- include/linux/kthread.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 30e5bec81d2b..f1f95a71a4bc 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -89,6 +89,7 @@ int kthread_stop(struct task_struct *k); bool kthread_should_stop(void); bool kthread_should_park(void); bool __kthread_should_park(struct task_struct *k); +bool kthread_should_stop_or_park(void); bool kthread_freezable_should_stop(bool *was_frozen); void *kthread_func(struct task_struct *k); void *kthread_data(struct task_struct *k); -- cgit v1.2.3 From e4cc64657becbd073c3ecc9d5938a1fe0d59913f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 14 Jun 2023 16:03:40 +0200 Subject: block: remove BIO_PAGE_REFFED Now that all block direct I/O helpers use page pinning, this flag is unused. Signed-off-by: Christoph Hellwig Reviewed-by: Christian Brauner Reviewed-by: Johannes Thumshirn Reviewed-by: David Howells Link: https://lore.kernel.org/r/20230614140341.521331-4-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/bio.h | 3 +-- include/linux/blk_types.h | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bio.h b/include/linux/bio.h index 617522928964..c4f5b5228105 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -492,8 +492,7 @@ void zero_fill_bio(struct bio *bio); static inline void bio_release_pages(struct bio *bio, bool mark_dirty) { - if (bio_flagged(bio, BIO_PAGE_REFFED) || - bio_flagged(bio, BIO_PAGE_PINNED)) + if (bio_flagged(bio, BIO_PAGE_PINNED)) __bio_release_pages(bio, mark_dirty); } diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index deb69eeab6bd..752a54e3284b 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -326,7 +326,6 @@ struct bio { */ enum { BIO_PAGE_PINNED, /* Unpin pages in bio_release_pages() */ - BIO_PAGE_REFFED, /* put pages in bio_release_pages() */ BIO_CLONED, /* doesn't own data */ BIO_BOUNCED, /* bio is a bounce bio */ BIO_QUIET, /* Make BIO Quiet */ -- cgit v1.2.3 From 84bd06c632c6d5279849f5f8ab47d9517d259422 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 14 Jun 2023 16:03:41 +0200 Subject: iov_iter: remove iov_iter_get_pages and iov_iter_get_pages_alloc Now that the direct I/O helpers have switched to use iov_iter_extract_pages, these helpers are unused. Signed-off-by: Christoph Hellwig Reviewed-by: Christian Brauner Reviewed-by: David Howells Link: https://lore.kernel.org/r/20230614140341.521331-5-hch@lst.de Signed-off-by: Jens Axboe --- include/linux/uio.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/uio.h b/include/linux/uio.h index 60c342bb7ab8..8e7d2c425340 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -277,14 +277,8 @@ void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_ void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count); void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, loff_t start, size_t count); -ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, - size_t maxsize, unsigned maxpages, size_t *start, - iov_iter_extraction_t extraction_flags); ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages, size_t maxsize, unsigned maxpages, size_t *start); -ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, - struct page ***pages, size_t maxsize, size_t *start, - iov_iter_extraction_t extraction_flags); ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages, size_t maxsize, size_t *start); int iov_iter_npages(const struct iov_iter *i, int maxpages); -- cgit v1.2.3 From 8bb42ed4210e342631f63d32f7ed87b722968da6 Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Wed, 17 May 2023 11:53:50 +0300 Subject: net/mlx5: Expose timeout for sync reset unload stage Expose new timoueout in Default Timeouts Register to be used on sync reset flow running on smart NIC. In this flow the driver should know how much time to wait from getting unload request till firmware will ask the PF to continue to next stage of the flow. Signed-off-by: Moshe Shemesh Reviewed-by: Shay Drory Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 1f4f62cb9f34..14892e795808 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -3117,7 +3117,9 @@ struct mlx5_ifc_dtor_reg_bits { struct mlx5_ifc_default_timeout_bits reclaim_vfs_pages_to; - u8 reserved_at_1c0[0x40]; + struct mlx5_ifc_default_timeout_bits reset_unload_to; + + u8 reserved_at_1c0[0x20]; }; enum { -- cgit v1.2.3 From 7a9770f1bfeaeddf5afabd3244e2c4c4966be37d Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Wed, 17 May 2023 16:07:40 +0300 Subject: net/mlx5: Handle sync reset unload event Added a new event handler to firmware sync reset, which is used to support firmware sync reset flow on smart NIC. Adding this new stage to the flow enables the firmware to ensure host PFs unload before ECPFs unload, to avoid race of PFs recovery. If firmware sends sync_reset_unload event to driver the driver should unload and close all HW resources of the function. Once the driver finishes unloading part, it can't get any more events from firmware as event queues are closed, so it polls the reset state field to know when to continue to next stage of the sync reset flow. Added capability bit for supporting sync_reset_unload event. Signed-off-by: Moshe Shemesh Reviewed-by: Shay Drory Signed-off-by: Saeed Mahameed --- include/linux/mlx5/device.h | 1 + include/linux/mlx5/mlx5_ifc.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index c0af74efd3cb..80cc12a9a531 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -716,6 +716,7 @@ enum sync_rst_state_type { MLX5_SYNC_RST_STATE_RESET_REQUEST = 0x0, MLX5_SYNC_RST_STATE_RESET_NOW = 0x1, MLX5_SYNC_RST_STATE_RESET_ABORT = 0x2, + MLX5_SYNC_RST_STATE_RESET_UNLOAD = 0x3, }; struct mlx5_eqe_sync_fw_update { diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 14892e795808..d61dcb5d7cd5 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1755,7 +1755,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_328[0x2]; u8 relaxed_ordering_read[0x1]; u8 log_max_pd[0x5]; - u8 reserved_at_330[0x7]; + u8 reserved_at_330[0x6]; + u8 pci_sync_for_fw_update_with_driver_unload[0x1]; u8 vnic_env_cnt_steering_fail[0x1]; u8 reserved_at_338[0x1]; u8 q_counter_aggregation[0x1]; -- cgit v1.2.3 From 0bd2e6fc78fddf83b9a71a61bdf0c4caca83abe7 Mon Sep 17 00:00:00 2001 From: Or Har-Toov Date: Thu, 23 Mar 2023 17:52:03 +0200 Subject: net/mlx5: Expose bits for local loopback counter Add needed HW bits for querying local loopback counter and the HCA capability for it. Signed-off-by: Or Har-Toov Reviewed-by: Avihai Horon Reviewed-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index d61dcb5d7cd5..354c7e326eab 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1758,7 +1758,7 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_330[0x6]; u8 pci_sync_for_fw_update_with_driver_unload[0x1]; u8 vnic_env_cnt_steering_fail[0x1]; - u8 reserved_at_338[0x1]; + u8 vport_counter_local_loopback[0x1]; u8 q_counter_aggregation[0x1]; u8 q_counter_other_vport[0x1]; u8 log_max_xrcd[0x5]; @@ -5190,7 +5190,9 @@ struct mlx5_ifc_query_vport_counter_out_bits { struct mlx5_ifc_traffic_counter_bits transmitted_eth_multicast; - u8 reserved_at_680[0xa00]; + struct mlx5_ifc_traffic_counter_bits local_loopback; + + u8 reserved_at_700[0x980]; }; enum { -- cgit v1.2.3 From f1771b85e3086c9506c3de81e993330bca568ba5 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 22:05:05 +0200 Subject: irqchip/mmp: Remove non-DT codepath Building with "W=1" warns about missing declarations for two functions in the mmp irqchip driver: drivers/irqchip/irq-mmp.c:248:13: error: no previous prototype for 'icu_init_irq' drivers/irqchip/irq-mmp.c:271:13: error: no previous prototype for 'mmp2_init_icu' The declarations are present in an unused header, but since there is no caller, it's best to just remove the functions and the header completely, making the driver DT-only to match the state of the platform. Fixes: 77acc85ce797 ("ARM: mmp: remove device definitions") Signed-off-by: Arnd Bergmann Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230516200516.554663-2-arnd@kernel.org --- include/linux/irqchip/mmp.h | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 include/linux/irqchip/mmp.h (limited to 'include/linux') diff --git a/include/linux/irqchip/mmp.h b/include/linux/irqchip/mmp.h deleted file mode 100644 index aa1813749a4f..000000000000 --- a/include/linux/irqchip/mmp.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __IRQCHIP_MMP_H -#define __IRQCHIP_MMP_H - -extern struct irq_chip icu_irq_chip; - -extern void icu_init_irq(void); -extern void mmp2_init_icu(void); - -#endif /* __IRQCHIP_MMP_H */ -- cgit v1.2.3 From 415e84294798d1cb041c902168393054cc4ad211 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 22:05:08 +0200 Subject: irqchip/gicv3: Add a iort_pmsi_get_dev_id() prototype iort_pmsi_get_dev_id() has a __weak definition in the driver, and an override in arm64 specific code, but the declaration is conditional and not always seen when the copy in the driver gets built: drivers/irqchip/irq-gic-v3-its-platform-msi.c:41:12: error: no previous prototype for 'iort_pmsi_get_dev_id' [-Werror=missing-prototypes] Move the existing declaration out of the #ifdef block to ensure it can be seen in all configurations. Signed-off-by: Arnd Bergmann Reviewed-by: Hanjun Guo Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20230516200516.554663-5-arnd@kernel.org --- include/linux/acpi_iort.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index b43be0987b19..6b70d02bc5f9 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -26,13 +26,14 @@ int iort_register_domain_token(int trans_id, phys_addr_t base, struct fwnode_handle *fw_node); void iort_deregister_domain_token(int trans_id); struct fwnode_handle *iort_find_domain_token(int trans_id); +int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); + #ifdef CONFIG_ACPI_IORT void acpi_iort_init(void); u32 iort_msi_map_id(struct device *dev, u32 id); struct irq_domain *iort_get_device_domain(struct device *dev, u32 id, enum irq_domain_bus_token bus_token); void acpi_configure_pmsi_domain(struct device *dev); -int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); void iort_get_rmr_sids(struct fwnode_handle *iommu_fwnode, struct list_head *head); void iort_put_rmr_sids(struct fwnode_handle *iommu_fwnode, -- cgit v1.2.3 From fc669e922ecff02c173a8484f7a5ed4810089209 Mon Sep 17 00:00:00 2001 From: JuenKit Yip Date: Sat, 17 Jun 2023 00:00:12 +0800 Subject: hwmon: (sht3x) remove sht3x_platform_data Since no in-tree driver supports it, sht3x_platform_data has been removed and the relevant properties have been moved to sht3x_data. Signed-off-by: JuenKit Yip Link: https://lore.kernel.org/r/DB4PR10MB626126FB7226D5AF341197449258A@DB4PR10MB6261.EURPRD10.PROD.OUTLOOK.COM Signed-off-by: Guenter Roeck --- include/linux/platform_data/sht3x.h | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 include/linux/platform_data/sht3x.h (limited to 'include/linux') diff --git a/include/linux/platform_data/sht3x.h b/include/linux/platform_data/sht3x.h deleted file mode 100644 index 14680d2a98f7..000000000000 --- a/include/linux/platform_data/sht3x.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2016 Sensirion AG, Switzerland - * Author: David Frey - * Author: Pascal Sachs - */ - -#ifndef __SHT3X_H_ -#define __SHT3X_H_ - -struct sht3x_platform_data { - bool blocking_io; - bool high_precision; -}; -#endif /* __SHT3X_H_ */ -- cgit v1.2.3 From 2f2665c13af4895b26761107c2f637c2f112d8e9 Mon Sep 17 00:00:00 2001 From: Joel Granados Date: Fri, 16 Jun 2023 10:59:22 +0200 Subject: sysctl: replace child with an enumeration This is part of the effort to remove the empty element at the end of ctl_table structs. "child" was a deprecated elem in this struct and was being used to differentiate between two types of ctl_tables: "normal" and "permanently emtpy". What changed?: * Replace "child" with an enumeration that will have two values: the default (0) and the permanently empty (1). The latter is left at zero so when struct ctl_table is created with kzalloc or in a local context, it will have the zero value by default. We document the new enum with kdoc. * Remove the "empty child" check from sysctl_check_table * Remove count_subheaders function as there is no longer a need to calculate how many headers there are for every child * Remove the recursive call to unregister_sysctl_table as there is no need to traverse down the child tree any longer * Add a new SYSCTL_PERM_EMPTY_DIR binary flag * Remove the last remanence of child from partport/procfs.c Signed-off-by: Joel Granados Signed-off-by: Luis Chamberlain --- include/linux/sysctl.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 653b66c762b1..59d451f455bf 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -137,7 +137,17 @@ struct ctl_table { void *data; int maxlen; umode_t mode; - struct ctl_table *child; /* Deprecated */ + /** + * enum type - Enumeration to differentiate between ctl target types + * @SYSCTL_TABLE_TYPE_DEFAULT: ctl target with no special considerations + * @SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY: Used to identify a permanently + * empty directory target to serve + * as mount point. + */ + enum { + SYSCTL_TABLE_TYPE_DEFAULT, + SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY + } type; proc_handler *proc_handler; /* Callback for text formatting */ struct ctl_table_poll *poll; void *extra1; @@ -229,7 +239,7 @@ extern int unaligned_enabled; extern int unaligned_dump_stack; extern int no_unaligned_warning; -extern struct ctl_table sysctl_mount_point[]; +#define SYSCTL_PERM_EMPTY_DIR (1 << 0) #else /* CONFIG_SYSCTL */ -- cgit v1.2.3 From 7a7f094635349a7d0314364ad50bdeb770b6df4f Mon Sep 17 00:00:00 2001 From: Arjun Roy Date: Fri, 16 Jun 2023 12:34:27 -0700 Subject: tcp: Use per-vma locking for receive zerocopy Per-VMA locking allows us to lock a struct vm_area_struct without taking the process-wide mmap lock in read mode. Consider a process workload where the mmap lock is taken constantly in write mode. In this scenario, all zerocopy receives are periodically blocked during that period of time - though in principle, the memory ranges being used by TCP are not touched by the operations that need the mmap write lock. This results in performance degradation. Now consider another workload where the mmap lock is never taken in write mode, but there are many TCP connections using receive zerocopy that are concurrently receiving. These connections all take the mmap lock in read mode, but this does induce a lot of contention and atomic ops for this process-wide lock. This results in additional CPU overhead caused by contending on the cache line for this lock. However, with per-vma locking, both of these problems can be avoided. As a test, I ran an RPC-style request/response workload with 4KB payloads and receive zerocopy enabled, with 100 simultaneous TCP connections. I measured perf cycles within the find_tcp_vma/mmap_read_lock/mmap_read_unlock codepath, with and without per-vma locking enabled. When using process-wide mmap semaphore read locking, about 1% of measured perf cycles were within this path. With per-VMA locking, this value dropped to about 0.45%. Signed-off-by: Arjun Roy Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/net_mm.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 include/linux/net_mm.h (limited to 'include/linux') diff --git a/include/linux/net_mm.h b/include/linux/net_mm.h new file mode 100644 index 000000000000..b298998bd5a0 --- /dev/null +++ b/include/linux/net_mm.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifdef CONFIG_MMU + +#ifdef CONFIG_INET +extern const struct vm_operations_struct tcp_vm_ops; +static inline bool vma_is_tcp(const struct vm_area_struct *vma) +{ + return vma->vm_ops == &tcp_vm_ops; +} +#else +static inline bool vma_is_tcp(const struct vm_area_struct *vma) +{ + return false; +} +#endif /* CONFIG_INET*/ + +#endif /* CONFIG_MMU */ -- cgit v1.2.3 From 8ce8849dd1e78dadcee0ec9acbd259d239b7069f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 1 Jun 2023 20:58:47 +0200 Subject: posix-timers: Ensure timer ID search-loop limit is valid posix_timer_add() tries to allocate a posix timer ID by starting from the cached ID which was stored by the last successful allocation. This is done in a loop searching the ID space for a free slot one by one. The loop has to terminate when the search wrapped around to the starting point. But that's racy vs. establishing the starting point. That is read out lockless, which leads to the following problem: CPU0 CPU1 posix_timer_add() start = sig->posix_timer_id; lock(hash_lock); ... posix_timer_add() if (++sig->posix_timer_id < 0) start = sig->posix_timer_id; sig->posix_timer_id = 0; So CPU1 can observe a negative start value, i.e. -1, and the loop break never happens because the condition can never be true: if (sig->posix_timer_id == start) break; While this is unlikely to ever turn into an endless loop as the ID space is huge (INT_MAX), the racy read of the start value caught the attention of KCSAN and Dmitry unearthed that incorrectness. Rewrite it so that all id operations are under the hash lock. Reported-by: syzbot+5c54bd3eb218bb595aa9@syzkaller.appspotmail.com Reported-by: Dmitry Vyukov Signed-off-by: Thomas Gleixner Reviewed-by: Frederic Weisbecker Link: https://lore.kernel.org/r/87bkhzdn6g.ffs@tglx --- include/linux/sched/signal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 20099268fa25..669e8cff40c7 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -135,7 +135,7 @@ struct signal_struct { #ifdef CONFIG_POSIX_TIMERS /* POSIX.1b Interval Timers */ - int posix_timer_id; + unsigned int next_posix_timer_id; struct list_head posix_timers; /* ITIMER_REAL timer for the process */ -- cgit v1.2.3 From 892f439ea17cbf56a36e57c584d583649a64b404 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 7 Jun 2023 16:28:45 +0200 Subject: posix-timers: Add sys_ni_posix_timers() prototype The sys_ni_posix_timers() definition causes a warning when the declaration is missing, so this needs to be added along with the normal syscalls, outside of the #ifdef. kernel/time/posix-stubs.c:26:17: error: no previous prototype for 'sys_ni_posix_timers' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Signed-off-by: Thomas Gleixner Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20230607142925.3126422-1-arnd@kernel.org --- include/linux/syscalls.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 33a0ee3bcb2e..24871f8ec8bb 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1280,6 +1280,7 @@ asmlinkage long sys_ni_syscall(void); #endif /* CONFIG_ARCH_HAS_SYSCALL_WRAPPER */ +asmlinkage long sys_ni_posix_timers(void); /* * Kernel code should not call syscalls (i.e., sys_xyzyyz()) directly. -- cgit v1.2.3 From 568c69ae2fea27e0152e4ffeee7c6f354c61810f Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 18 Jun 2023 22:52:28 +0200 Subject: video/hdmi: Reorder fields in 'struct hdmi_avi_infoframe' Group some variables based on their sizes to reduce hole and avoid padding. On x86_64, this shrinks the size of 'struct hdmi_avi_infoframe' from 68 to 60 bytes. It saves a few bytes of memory and is more cache-line friendly. This also reduces the union hdmi_infoframe the same way. Signed-off-by: Christophe JAILLET Signed-off-by: Helge Deller --- include/linux/hdmi.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h index 2f4dcc8d060e..3bb87bf6bc65 100644 --- a/include/linux/hdmi.h +++ b/include/linux/hdmi.h @@ -170,19 +170,19 @@ struct hdmi_avi_infoframe { enum hdmi_infoframe_type type; unsigned char version; unsigned char length; + bool itc; + unsigned char pixel_repeat; enum hdmi_colorspace colorspace; enum hdmi_scan_mode scan_mode; enum hdmi_colorimetry colorimetry; enum hdmi_picture_aspect picture_aspect; enum hdmi_active_aspect active_aspect; - bool itc; enum hdmi_extended_colorimetry extended_colorimetry; enum hdmi_quantization_range quantization_range; enum hdmi_nups nups; unsigned char video_code; enum hdmi_ycc_quantization_range ycc_quantization_range; enum hdmi_content_type content_type; - unsigned char pixel_repeat; unsigned short top_bar; unsigned short bottom_bar; unsigned short left_bar; -- cgit v1.2.3 From d4313a68ec913f2705b337e2d332813a72cb2de9 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 13 Jun 2023 08:33:14 +0200 Subject: fbdev/media: Use GPIO descriptors for VIA GPIO The VIA fbdev exposes a custom GPIO chip for its GPIOs, these are in turn looked up the camera driver using a custom API. Drop the custom API, provide a look-up table and convert to GPIO descriptors. Note proper polarity on the RESET line. Cc: Jonathan Corbet Signed-off-by: Linus Walleij Signed-off-by: Helge Deller --- include/linux/via-gpio.h | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 include/linux/via-gpio.h (limited to 'include/linux') diff --git a/include/linux/via-gpio.h b/include/linux/via-gpio.h deleted file mode 100644 index ac34668fd442..000000000000 --- a/include/linux/via-gpio.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Support for viafb GPIO ports. - * - * Copyright 2009 Jonathan Corbet - */ - -#ifndef __VIA_GPIO_H__ -#define __VIA_GPIO_H__ - -extern int viafb_gpio_lookup(const char *name); -extern int viafb_gpio_init(void); -extern void viafb_gpio_exit(void); -#endif -- cgit v1.2.3 From 5bb578a0c1b86d6eb95f8d08ed6444b227fb674c Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 9 May 2023 13:57:28 +0100 Subject: ARM: 9298/1: Drop custom mdesc->handle_irq() ARM exclusively uses GENERIC_IRQ_MULTI_HANDLER, so at some point set_handle_irq() needs to be called to handle system-wide interrupts. For all DT-enabled boards, this call happens down in the drivers/irqchip subsystem, after locating the target irqchip driver from the device tree. We still have a few instances of the boardfiles with machine descriptors passing a machine-specific .handle_irq() to the ARM kernel core. Get rid of this by letting the few remaining machines consistently call set_handle_irq() from the end of the .init_irq() callback instead and diet down one member from the machine descriptor. Cc: Marc Zyngier Reviewed-by: Arnd Bergmann Acked-by: Mark Rutland Signed-off-by: Linus Walleij Signed-off-by: Russell King (Oracle) --- include/linux/irqchip/mxs.h | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 include/linux/irqchip/mxs.h (limited to 'include/linux') diff --git a/include/linux/irqchip/mxs.h b/include/linux/irqchip/mxs.h deleted file mode 100644 index 4f447e3f0f3a..000000000000 --- a/include/linux/irqchip/mxs.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2013 Freescale Semiconductor, Inc. - */ - -#ifndef __LINUX_IRQCHIP_MXS_H -#define __LINUX_IRQCHIP_MXS_H - -extern void icoll_handle_irq(struct pt_regs *); - -#endif -- cgit v1.2.3 From 03e7e493f1a3697eba115f3f69e296f7e47500ee Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Fri, 16 Jun 2023 09:54:02 +0300 Subject: wifi: cfg80211: ignore invalid TBTT info field types The TBTT information field type must be zero. This is only changed in the 802.11be draft specification where the value 1 is used to indicate that only the MLD parameters are included. Signed-off-by: Benjamin Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230616094949.7865606ffe94.I7ff28afb875d1b4c39acd497df8490a7d3628e3f@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 5dfed1a6625c..47ddc65b443b 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4481,6 +4481,8 @@ static inline bool for_each_element_completed(const struct element *element, #define IEEE80211_AP_INFO_TBTT_HDR_FILTERED 0x04 #define IEEE80211_AP_INFO_TBTT_HDR_COLOC 0x08 #define IEEE80211_AP_INFO_TBTT_HDR_COUNT 0xF0 +#define IEEE80211_TBTT_INFO_TYPE_TBTT 0 +#define IEEE80211_TBTT_INFO_TYPE_MLD 1 #define IEEE80211_TBTT_INFO_OFFSET_BSSID_BSS_PARAM 9 #define IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM 13 -- cgit v1.2.3 From e2efec97c3ad503042db27baaf7c8cb5d1348a83 Mon Sep 17 00:00:00 2001 From: Ilan Peer Date: Fri, 16 Jun 2023 09:54:09 +0300 Subject: wifi: mac80211: Rename ieee80211_mle_sta_prof_size_ok() Rename it to ieee80211_mle_basic_sta_prof_size_ok() as it validates the size of the station profile included in Basic Multi-Link element. Signed-off-by: Ilan Peer Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230616094949.9bdfd263974f.I7bebd26894f33716e93cc7da576ef3215e0ba727@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 47ddc65b443b..aeedd49e5101 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4798,11 +4798,13 @@ struct ieee80211_mle_per_sta_profile { } __packed; /** - * ieee80211_mle_sta_prof_size_ok - validate multi-link element sta profile size + * ieee80211_mle_basic_sta_prof_size_ok - validate basic multi-link element sta + * profile size * @data: pointer to the sub element data * @len: length of the containing sub element */ -static inline bool ieee80211_mle_sta_prof_size_ok(const u8 *data, size_t len) +static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, + size_t len) { const struct ieee80211_mle_per_sta_profile *prof = (const void *)data; u16 control; -- cgit v1.2.3 From ce6e1f600b0cfc563a7d607de702262a58cd835d Mon Sep 17 00:00:00 2001 From: Ilan Peer Date: Sun, 18 Jun 2023 21:49:45 +0300 Subject: wifi: ieee80211: Fix the common size calculation for reconfiguration ML The common information length is found in the first octet of the common information. Fixes: 0f48b8b88aa9 ("wifi: ieee80211: add definitions for multi-link element") Signed-off-by: Ilan Peer Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214435.3c7ed4817338.I42ef706cb827b4dade6e4ffbb6e7f341eaccd398@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index aeedd49e5101..97edc3b404dd 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4617,15 +4617,12 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) case IEEE80211_ML_CONTROL_TYPE_BASIC: case IEEE80211_ML_CONTROL_TYPE_PREQ: case IEEE80211_ML_CONTROL_TYPE_TDLS: + case IEEE80211_ML_CONTROL_TYPE_RECONF: /* * The length is the first octet pointed by mle->variable so no * need to add anything */ break; - case IEEE80211_ML_CONTROL_TYPE_RECONF: - if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR) - common += ETH_ALEN; - return common; case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS: if (control & IEEE80211_MLC_PRIO_ACCESS_PRES_AP_MLD_MAC_ADDR) common += ETH_ALEN; -- cgit v1.2.3 From eeec7574ec3c03c69adc99492df74dc1cc0ebd63 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Sun, 18 Jun 2023 21:49:46 +0300 Subject: wifi: ieee80211: add helper to validate ML element type and size The helper functions to retrieve the EML capabilities and medium synchronization delay both assume that the type is correct. Instead of assuming the length is correct and still checking the type, add a new helper to check both and don't do any verification. Signed-off-by: Benjamin Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214435.1b50e7a3b3cf.I9385514d8eb6d6d3c82479a6fa732ef65313e554@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 50 +++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 97edc3b404dd..b107f21e1233 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4639,10 +4639,10 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) * ieee80211_mle_get_eml_sync_delay - returns the medium sync delay * @data: pointer to the multi link EHT IE * - * The element is assumed to be big enough. This must be checked by - * ieee80211_mle_size_ok(). - * If the medium synchronization can't be found (the type is not basic, or - * the medium sync presence bit is clear), 0 will be returned. + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + * + * If the medium synchronization is not present, then 0 is returned. */ static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data) { @@ -4650,13 +4650,7 @@ static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data) u16 control = le16_to_cpu(mle->control); const u8 *common = mle->variable; - if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) != - IEEE80211_ML_CONTROL_TYPE_BASIC) - return 0; - - /* common points now at the beginning of - * ieee80211_mle_basic_common_info - */ + /* common points now at the beginning of ieee80211_mle_basic_common_info */ common += sizeof(struct ieee80211_mle_basic_common_info); if (!(control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)) @@ -4674,10 +4668,10 @@ static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data) * ieee80211_mle_get_eml_cap - returns the EML capability * @data: pointer to the multi link EHT IE * - * The element is assumed to be big enough. This must be checked by - * ieee80211_mle_size_ok(). - * If the EML capability can't be found (the type is not basic, or - * the EML capability presence bit is clear), 0 will be returned. + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + * + * If the EML capability is not present, 0 will be returned. */ static inline u16 ieee80211_mle_get_eml_cap(const u8 *data) { @@ -4685,10 +4679,6 @@ static inline u16 ieee80211_mle_get_eml_cap(const u8 *data) u16 control = le16_to_cpu(mle->control); const u8 *common = mle->variable; - if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) != - IEEE80211_ML_CONTROL_TYPE_BASIC) - return 0; - /* common points now at the beginning of ieee80211_mle_basic_common_info */ common += sizeof(struct ieee80211_mle_basic_common_info); @@ -4773,6 +4763,28 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) return mle->variable[0] >= common; } +/** + * ieee80211_mle_type_ok - validate multi-link element type and size + * @data: pointer to the element data + * @type: expected type of the element + * @len: length of the containing element + */ +static inline bool ieee80211_mle_type_ok(const u8 *data, u8 type, size_t len) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control; + + if (!ieee80211_mle_size_ok(data, len)) + return false; + + control = le16_to_cpu(mle->control); + + if (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE) == type) + return true; + + return false; +} + enum ieee80211_mle_subelems { IEEE80211_MLE_SUBELEM_PER_STA_PROFILE = 0, IEEE80211_MLE_SUBELEM_FRAGMENT = 254, -- cgit v1.2.3 From 39bcc5b8e16e75cfccc36fca3d425c64eef3df04 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Sun, 18 Jun 2023 21:49:47 +0300 Subject: wifi: ieee80211: use default for medium synchronization delay Default values are defined for the information included in the Medium Synchronization Delay Information subfield. The spec says to initialize the values to these defaults and only change them when included. Return the default value instead of zero so that the defaults are used when the field is not included in the association response. Signed-off-by: Benjamin Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214435.a7725bef3795.I2d3528cf4af021c5b37f97fbe64ae9116ce9bef1@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index b107f21e1233..251998be24d0 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4535,6 +4535,14 @@ struct ieee80211_multi_link_elem { #define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH 0x0f00 #define IEEE80211_MED_SYNC_DELAY_SYNC_MAX_NUM_TXOPS 0xf000 +/* + * Described in P802.11be_D3.0 + * dot11MSDTimerDuration should default to 5484 (i.e. 171.375) + * dot11MSDOFDMEDthreshold defaults to -72 (i.e. 0) + * dot11MSDTXOPMAX defaults to 1 + */ +#define IEEE80211_MED_SYNC_DELAY_DEFAULT 0x10ac + #define IEEE80211_EML_CAP_EMLSR_SUPP 0x0001 #define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY 0x000e #define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_0US 0 @@ -4642,7 +4650,8 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) * The element is assumed to be of the correct type (BASIC) and big enough, * this must be checked using ieee80211_mle_type_ok(). * - * If the medium synchronization is not present, then 0 is returned. + * If the medium synchronization is not present, then the default value is + * returned. */ static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data) { @@ -4654,7 +4663,7 @@ static inline u16 ieee80211_mle_get_eml_med_sync_delay(const u8 *data) common += sizeof(struct ieee80211_mle_basic_common_info); if (!(control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY)) - return 0; + return IEEE80211_MED_SYNC_DELAY_DEFAULT; if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) common += 1; -- cgit v1.2.3 From 66d9c573fbb992f11d29a907c339792ee0de82ee Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Sun, 18 Jun 2023 21:49:49 +0300 Subject: wifi: ieee80211: add definitions for RNR MLD params Add the definitions necessary to parse the MLD parameters included in an RNR element. Signed-off-by: Benjamin Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.9999842237c0.I80f00a90cb4e43071432b4158f206c73ba799618@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 251998be24d0..7afd08d2de2f 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4485,6 +4485,7 @@ static inline bool for_each_element_completed(const struct element *element, #define IEEE80211_TBTT_INFO_TYPE_MLD 1 #define IEEE80211_TBTT_INFO_OFFSET_BSSID_BSS_PARAM 9 #define IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM 13 +#define IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM_MLD_PARAM 16 #define IEEE80211_RNR_TBTT_PARAMS_OCT_RECOMMENDED 0x01 #define IEEE80211_RNR_TBTT_PARAMS_SAME_SSID 0x02 @@ -4508,6 +4509,20 @@ enum ieee80211_range_params_max_total_ltf { IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_UNSPECIFIED, }; +/* + * reduced neighbor report, based on Draft P802.11be_D3.0, + * section 9.4.2.170.2. + */ +struct ieee80211_rnr_mld_params { + u8 mld_id; + __le16 params; +} __packed; + +#define IEEE80211_RNR_MLD_PARAMS_LINK_ID 0x000F +#define IEEE80211_RNR_MLD_PARAMS_BSS_CHANGE_COUNT 0x0FF0 +#define IEEE80211_RNR_MLD_PARAMS_UPDATES_INCLUDED 0x1000 +#define IEEE80211_RNR_MLD_PARAMS_DISABLED_LINK 0x2000 + /* multi-link device */ #define IEEE80211_MLD_MAX_NUM_LINKS 15 -- cgit v1.2.3 From 50181fe4f59dcfae391523def6f62e32d86c46b1 Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Sun, 18 Jun 2023 21:49:51 +0300 Subject: wifi: ieee80211: add structs for TBTT information access The TBTT information can have various lengths with different elements thare are present. Add definitions for the two types that we are interested in (i.e. the ones that contain the BSSID). Signed-off-by: Benjamin Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.2a6f8766a3ec.Ic962e28492212cc8ee1eb602b8f07a4ea172fc4a@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 7afd08d2de2f..5a27c232afdb 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4523,6 +4523,28 @@ struct ieee80211_rnr_mld_params { #define IEEE80211_RNR_MLD_PARAMS_UPDATES_INCLUDED 0x1000 #define IEEE80211_RNR_MLD_PARAMS_DISABLED_LINK 0x2000 +/* Format of the TBTT information element if it has 7, 8 or 9 bytes */ +struct ieee80211_tbtt_info_7_8_9 { + u8 tbtt_offset; + u8 bssid[ETH_ALEN]; + + /* The following element is optional, structure may not grow */ + u8 bss_params; + u8 psd_20; +} __packed; + +/* Format of the TBTT information element if it has >= 11 bytes */ +struct ieee80211_tbtt_info_ge_11 { + u8 tbtt_offset; + u8 bssid[ETH_ALEN]; + __le32 short_ssid; + + /* The following elements are optional, structure may grow */ + u8 bss_params; + u8 psd_20; + struct ieee80211_rnr_mld_params mld_params; +} __packed; + /* multi-link device */ #define IEEE80211_MLD_MAX_NUM_LINKS 15 -- cgit v1.2.3 From dc92e54c30c4bc9d30e674a445dfe1afdca991cf Mon Sep 17 00:00:00 2001 From: Benjamin Berg Date: Sun, 18 Jun 2023 21:49:52 +0300 Subject: wifi: cfg80211: use structs for TBTT information access Make the data access a bit nicer overall by using structs. There is a small change here to also accept a TBTT information length of eight bytes as we do not require the 20 MHz PSD information. This also fixes a bug reading the short SSID on big endian machines. Signed-off-by: Benjamin Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.4c3f8901c1bc.Ic3e94fd6e1bccff7948a252ad3bb87e322690a17@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 5a27c232afdb..e145af7448a3 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4483,9 +4483,6 @@ static inline bool for_each_element_completed(const struct element *element, #define IEEE80211_AP_INFO_TBTT_HDR_COUNT 0xF0 #define IEEE80211_TBTT_INFO_TYPE_TBTT 0 #define IEEE80211_TBTT_INFO_TYPE_MLD 1 -#define IEEE80211_TBTT_INFO_OFFSET_BSSID_BSS_PARAM 9 -#define IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM 13 -#define IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM_MLD_PARAM 16 #define IEEE80211_RNR_TBTT_PARAMS_OCT_RECOMMENDED 0x01 #define IEEE80211_RNR_TBTT_PARAMS_SAME_SSID 0x02 -- cgit v1.2.3 From 8eb8dd2ffbbb6b0b8843b66754ee9f129f1b2d6c Mon Sep 17 00:00:00 2001 From: Ilan Peer Date: Sun, 18 Jun 2023 21:49:55 +0300 Subject: wifi: mac80211: Support link removal using Reconfiguration ML element Add support for handling link removal indicated by the Reconfiguration Multi-Link element. Signed-off-by: Ilan Peer Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.d8a046dc0c1a.I4dcf794da2a2d9f4e5f63a4b32158075d27c0660@changeid [use cfg80211_links_removed() API instead] Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index e145af7448a3..98223b665456 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4891,6 +4891,39 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, fixed + prof->sta_info_len <= len; } +#define IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID 0x000f +#define IEEE80211_MLE_STA_RECONF_CONTROL_COMPLETE_PROFILE 0x0010 +#define IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT 0x0020 +#define IEEE80211_MLE_STA_RECONF_CONTROL_DELETE_TIMER_PRESENT 0x0040 + +/** + * ieee80211_mle_reconf_sta_prof_size_ok - validate reconfiguration multi-link + * element sta profile size. + * @data: pointer to the sub element data + * @len: length of the containing sub element + */ +static inline bool ieee80211_mle_reconf_sta_prof_size_ok(const u8 *data, + size_t len) +{ + const struct ieee80211_mle_per_sta_profile *prof = (const void *)data; + u16 control; + u8 fixed = sizeof(*prof); + u8 info_len = 1; + + if (len < fixed) + return false; + + control = le16_to_cpu(prof->control); + + if (control & IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT) + info_len += ETH_ALEN; + if (control & IEEE80211_MLE_STA_RECONF_CONTROL_DELETE_TIMER_PRESENT) + info_len += 2; + + return prof->sta_info_len >= info_len && + fixed + prof->sta_info_len - 1 <= len; +} + #define for_each_mle_subelement(_elem, _data, _len) \ if (ieee80211_mle_size_ok(_data, _len)) \ for_each_element(_elem, \ -- cgit v1.2.3 From 888a325fe0a7d149828600c663869636ffdbe81f Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sun, 18 Jun 2023 21:49:56 +0300 Subject: wifi: ieee80211: reorder presence checks in MLE per-STA profile In ieee80211_mle_sta_prof_size_ok(), the presence checks aren't ordered by field order, so that's a bit confusing. Reorder them. Signed-off-by: Johannes Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.fdbf17320a37.I517cf27fdc3f6e5d6a2615182da47ba4bdf14039@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 98223b665456..fc3c26f1b718 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4876,9 +4876,6 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, info_len += 8; if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT) info_len += 2; - if (control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT) - info_len += 1; - if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE && control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) { if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) @@ -4886,6 +4883,8 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, else info_len += 1; } + if (control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT) + info_len += 1; return prof->sta_info_len >= info_len && fixed + prof->sta_info_len <= len; -- cgit v1.2.3 From c870d66f1b7f51fa3401771ff6c41fd78adb869e Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sun, 18 Jun 2023 21:49:59 +0300 Subject: wifi: update multi-link element STA reconfig Update the MLE STA reconfig sub-type to 802.11be D3.0 format, which includes the operation update field. Signed-off-by: Johannes Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.2e1383b31f07.I8055a111c8fcf22e833e60f5587a4d8d21caca5b@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index fc3c26f1b718..d2025c986b0f 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4893,7 +4893,9 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, #define IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID 0x000f #define IEEE80211_MLE_STA_RECONF_CONTROL_COMPLETE_PROFILE 0x0010 #define IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT 0x0020 -#define IEEE80211_MLE_STA_RECONF_CONTROL_DELETE_TIMER_PRESENT 0x0040 +#define IEEE80211_MLE_STA_RECONF_CONTROL_AP_REM_TIMER_PRESENT 0x0040 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_UPDATE_TYPE 0x0780 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_PARAMS_PRESENT 0x0800 /** * ieee80211_mle_reconf_sta_prof_size_ok - validate reconfiguration multi-link @@ -4916,7 +4918,9 @@ static inline bool ieee80211_mle_reconf_sta_prof_size_ok(const u8 *data, if (control & IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT) info_len += ETH_ALEN; - if (control & IEEE80211_MLE_STA_RECONF_CONTROL_DELETE_TIMER_PRESENT) + if (control & IEEE80211_MLE_STA_RECONF_CONTROL_AP_REM_TIMER_PRESENT) + info_len += 2; + if (control & IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_PARAMS_PRESENT) info_len += 2; return prof->sta_info_len >= info_len && -- cgit v1.2.3 From cf0b045ebf6bba7764151e1759c874c43964445a Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sun, 18 Jun 2023 21:50:02 +0300 Subject: wifi: mac80211: check EHT basic MCS/NSS set Check that all the NSS in the EHT basic MCS/NSS set are actually supported, otherwise disable EHT for the connection. Signed-off-by: Johannes Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230618214436.737827c906c9.I0c11a3cd46ab4dcb774c11a5bbc30aecfb6fce11@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index d2025c986b0f..fa679613c562 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -1996,12 +1996,18 @@ struct ieee80211_mu_edca_param_set { * @rx_tx_mcs13_max_nss: indicates the maximum number of spatial streams * supported for reception and the maximum number of spatial streams * supported for transmission for MCS 12 - 13. + * @rx_tx_max_nss: array of the previous fields for easier loop access */ struct ieee80211_eht_mcs_nss_supp_20mhz_only { - u8 rx_tx_mcs7_max_nss; - u8 rx_tx_mcs9_max_nss; - u8 rx_tx_mcs11_max_nss; - u8 rx_tx_mcs13_max_nss; + union { + struct { + u8 rx_tx_mcs7_max_nss; + u8 rx_tx_mcs9_max_nss; + u8 rx_tx_mcs11_max_nss; + u8 rx_tx_mcs13_max_nss; + }; + u8 rx_tx_max_nss[4]; + }; }; /** @@ -2021,11 +2027,17 @@ struct ieee80211_eht_mcs_nss_supp_20mhz_only { * @rx_tx_mcs13_max_nss: indicates the maximum number of spatial streams * supported for reception and the maximum number of spatial streams * supported for transmission for MCS 12 - 13. + * @rx_tx_max_nss: array of the previous fields for easier loop access */ struct ieee80211_eht_mcs_nss_supp_bw { - u8 rx_tx_mcs9_max_nss; - u8 rx_tx_mcs11_max_nss; - u8 rx_tx_mcs13_max_nss; + union { + struct { + u8 rx_tx_mcs9_max_nss; + u8 rx_tx_mcs11_max_nss; + u8 rx_tx_mcs13_max_nss; + }; + u8 rx_tx_max_nss[3]; + }; }; /** @@ -2078,7 +2090,7 @@ struct ieee80211_eht_cap_elem { */ struct ieee80211_eht_operation { u8 params; - __le32 basic_mcs_nss; + struct ieee80211_eht_mcs_nss_supp_20mhz_only basic_mcs_nss; u8 optional[]; } __packed; -- cgit v1.2.3 From a48b3f7be9c5e507ca07bd93d769798f4e5e68b1 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 16 Jun 2023 16:53:13 +0300 Subject: gpiolib: Drop unused domain_ops memeber of GPIO IRQ chip It seems there is no driver that requires custom IRQ chip domain options. Drop the member and respective code. Signed-off-by: Andy Shevchenko Reviewed-by: Linus Walleij Acked-by: Marc Zyngier Signed-off-by: Bartosz Golaszewski --- include/linux/gpio/driver.h | 7 ------- 1 file changed, 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 5c6db5533be6..6879b5436480 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -61,13 +61,6 @@ struct gpio_irq_chip { */ struct irq_domain *domain; - /** - * @domain_ops: - * - * Table of interrupt domain operations for this IRQ chip. - */ - const struct irq_domain_ops *domain_ops; - #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY /** * @fwnode: -- cgit v1.2.3 From d56e0ddb8fc35a7aa13ab8f21c499a34f45dda05 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 15 Jun 2023 14:22:25 +0300 Subject: fs: rename {vfs,kernel}_tmpfile_open() Overlayfs and cachefiles use vfs_open_tmpfile() to open a tmpfile without accounting for nr_files. Rename this helper to kernel_tmpfile_open() to better reflect this helper is used for kernel internal users. Signed-off-by: Amir Goldstein Reviewed-by: Christoph Hellwig Message-Id: <20230615112229.2143178-2-amir73il@gmail.com> Signed-off-by: Christian Brauner --- include/linux/fs.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 21a981680856..62237beeac2a 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1672,9 +1672,10 @@ static inline int vfs_whiteout(struct mnt_idmap *idmap, WHITEOUT_DEV); } -struct file *vfs_tmpfile_open(struct mnt_idmap *idmap, - const struct path *parentpath, - umode_t mode, int open_flag, const struct cred *cred); +struct file *kernel_tmpfile_open(struct mnt_idmap *idmap, + const struct path *parentpath, + umode_t mode, int open_flag, + const struct cred *cred); int vfs_mkobj(struct dentry *, umode_t, int (*f)(struct dentry *, umode_t, void *), -- cgit v1.2.3 From cbb0b9d4bbcfa96e7872808a63be03202536f1bc Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 15 Jun 2023 14:22:26 +0300 Subject: fs: use a helper for opening kernel internal files cachefiles uses kernel_open_tmpfile() to open kernel internal tmpfile without accounting for nr_files. cachefiles uses open_with_fake_path() for the same reason without the need for a fake path. Fork open_with_fake_path() to kernel_file_open() which only does the noaccount part and use it in cachefiles. Signed-off-by: Amir Goldstein Reviewed-by: Christoph Hellwig Message-Id: <20230615112229.2143178-3-amir73il@gmail.com> Signed-off-by: Christian Brauner --- include/linux/fs.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 62237beeac2a..1f8486e773af 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1676,6 +1676,8 @@ struct file *kernel_tmpfile_open(struct mnt_idmap *idmap, const struct path *parentpath, umode_t mode, int open_flag, const struct cred *cred); +struct file *kernel_file_open(const struct path *path, int flags, + struct inode *inode, const struct cred *cred); int vfs_mkobj(struct dentry *, umode_t, int (*f)(struct dentry *, umode_t, void *), -- cgit v1.2.3 From 62d53c4a1dfe347bd87ede46ffad38c9a3870338 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 15 Jun 2023 14:22:28 +0300 Subject: fs: use backing_file container for internal files with "fake" f_path Overlayfs uses open_with_fake_path() to allocate internal kernel files, with a "fake" path - whose f_path is not on the same fs as f_inode. Allocate a container struct backing_file for those internal files, that is used to hold the "fake" ovl path along with the real path. backing_file_real_path() can be used to access the stored real path. Signed-off-by: Amir Goldstein Message-Id: <20230615112229.2143178-5-amir73il@gmail.com> Signed-off-by: Christian Brauner --- include/linux/fs.h | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 1f8486e773af..24e1be1c13ca 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -171,6 +171,9 @@ typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset, /* File supports non-exclusive O_DIRECT writes from multiple threads */ #define FMODE_DIO_PARALLEL_WRITE ((__force fmode_t)0x1000000) +/* File is embedded in backing_file object */ +#define FMODE_BACKING ((__force fmode_t)0x2000000) + /* File was opened by fanotify and shouldn't generate fanotify events */ #define FMODE_NONOTIFY ((__force fmode_t)0x4000000) @@ -2352,11 +2355,31 @@ static inline struct file *file_open_root_mnt(struct vfsmount *mnt, return file_open_root(&(struct path){.mnt = mnt, .dentry = mnt->mnt_root}, name, flags, mode); } -extern struct file * dentry_open(const struct path *, int, const struct cred *); -extern struct file *dentry_create(const struct path *path, int flags, - umode_t mode, const struct cred *cred); -extern struct file * open_with_fake_path(const struct path *, int, - struct inode*, const struct cred *); +struct file *dentry_open(const struct path *path, int flags, + const struct cred *creds); +struct file *dentry_create(const struct path *path, int flags, umode_t mode, + const struct cred *cred); +struct file *backing_file_open(const struct path *path, int flags, + const struct path *real_path, + const struct cred *cred); +struct path *backing_file_real_path(struct file *f); + +/* + * file_real_path - get the path corresponding to f_inode + * + * When opening a backing file for a stackable filesystem (e.g., + * overlayfs) f_path may be on the stackable filesystem and f_inode on + * the underlying filesystem. When the path associated with f_inode is + * needed, this helper should be used instead of accessing f_path + * directly. +*/ +static inline const struct path *file_real_path(struct file *f) +{ + if (unlikely(f->f_mode & FMODE_BACKING)) + return backing_file_real_path(f); + return &f->f_path; +} + static inline struct file *file_clone_open(struct file *file) { return dentry_open(&file->f_path, file->f_flags, file->f_cred); -- cgit v1.2.3 From bc2473c90fca55bf95b2ab6af1dacee26a4f92f6 Mon Sep 17 00:00:00 2001 From: Amir Goldstein Date: Thu, 15 Jun 2023 14:22:29 +0300 Subject: ovl: enable fsnotify events on underlying real files Overlayfs creates the real underlying files with fake f_path, whose f_inode is on the underlying fs and f_path on overlayfs. Those real files were open with FMODE_NONOTIFY, because fsnotify code was not prapared to handle fsnotify hooks on files with fake path correctly and fanotify would report unexpected event->fd with fake overlayfs path, when the underlying fs was being watched. Teach fsnotify to handle events on the real files, and do not set real files to FMODE_NONOTIFY to allow operations on real file (e.g. open, access, modify, close) to generate async and permission events. Because fsnotify does not have notifications on address space operations, we do not need to worry about ->vm_file not reporting events to a watched overlayfs when users are accessing a mapped overlayfs file. Acked-by: Jan Kara Signed-off-by: Amir Goldstein Message-Id: <20230615112229.2143178-6-amir73il@gmail.com> Signed-off-by: Christian Brauner --- include/linux/fsnotify.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index bb8467cd11ae..ed48e4f1e755 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -91,11 +91,13 @@ static inline void fsnotify_dentry(struct dentry *dentry, __u32 mask) static inline int fsnotify_file(struct file *file, __u32 mask) { - const struct path *path = &file->f_path; + const struct path *path; if (file->f_mode & FMODE_NONOTIFY) return 0; + /* Overlayfs internal files have fake f_path */ + path = file_real_path(file); return fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH); } -- cgit v1.2.3 From ce5df7764b3b2abaf3687c460a9a1922daaed5b7 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Fri, 19 May 2023 13:16:52 +0200 Subject: mm: page_isolation: write proper kerneldoc And remove the incorrect header comments. [akpm@linux-foundation.org: s/lower/first/, s/upper/last/, per Mike] Link: https://lkml.kernel.org/r/20230519111652.40658-1-hannes@cmpxchg.org Signed-off-by: Johannes Weiner Cc: Mike Rapoport Signed-off-by: Andrew Morton --- include/linux/page-isolation.h | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 5456b7be38ae..0ab089e89db4 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -37,24 +37,12 @@ void set_pageblock_migratetype(struct page *page, int migratetype); int move_freepages_block(struct zone *zone, struct page *page, int migratetype, int *num_movable); -/* - * Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE. - */ -int -start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, - int migratetype, int flags, gfp_t gfp_flags); - -/* - * Changes MIGRATE_ISOLATE to MIGRATE_MOVABLE. - * target range is [start_pfn, end_pfn) - */ -void -undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, - int migratetype); - -/* - * Test all pages in [start_pfn, end_pfn) are isolated or not. - */ +int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + int migratetype, int flags, gfp_t gfp_flags); + +void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + int migratetype); + int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, int isol_flags); -- cgit v1.2.3 From e52ee4cc8fa87a75ab0cfc7bf51c0715a880a08e Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Sat, 3 Jun 2023 22:25:13 +0800 Subject: mm: remove obsolete alloc_migrate_target() There's only declaration left in the header file. Remove it. Link: https://lkml.kernel.org/r/20230603142513.787000-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: David Hildenbrand Cc: Johannes Weiner Signed-off-by: Andrew Morton --- include/linux/page-isolation.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 0ab089e89db4..4ac34392823a 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -45,7 +45,4 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, int isol_flags); - -struct page *alloc_migrate_target(struct page *page, unsigned long private); - #endif -- cgit v1.2.3 From e5797dc011182f8b25420bc977f37cd92fc6e755 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Tue, 6 Jun 2023 20:18:13 +0800 Subject: mm: vmscan: mark kswapd_run() and kswapd_stop() __meminit Add __meminit to kswapd_run() and kswapd_stop() to ensure they're default to __init when memory hotplug is not enabled. Link: https://lkml.kernel.org/r/20230606121813.242163-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Acked-by: Yu Zhao Acked-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/swap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 2ddbfd85f6c7..b5f6f2916de1 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -460,8 +460,8 @@ static inline bool node_reclaim_enabled(void) void check_move_unevictable_folios(struct folio_batch *fbatch); void check_move_unevictable_pages(struct pagevec *pvec); -extern void kswapd_run(int nid); -extern void kswapd_stop(int nid); +extern void __meminit kswapd_run(int nid); +extern void __meminit kswapd_stop(int nid); #ifdef CONFIG_SWAP -- cgit v1.2.3 From bd5f79ab39367665f40e10c2486aa15e7a841490 Mon Sep 17 00:00:00 2001 From: Yajun Deng Date: Wed, 7 Jun 2023 10:39:52 +0800 Subject: mm/sparse: remove unused parameters in sparse_remove_section() These parameters ms and map_offset are not used in sparse_remove_section(), so remove them. The __remove_section() is only called by __remove_pages(), remove it. And put the WARN_ON_ONCE() in sparse_remove_section(). Link: https://lkml.kernel.org/r/20230607023952.2247489-1-yajun.deng@linux.dev Signed-off-by: Yajun Deng Reviewed-by: David Hildenbrand Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/memory_hotplug.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 04bc286eed42..013c69753c91 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -344,9 +344,8 @@ extern void remove_pfn_range_from_zone(struct zone *zone, extern int sparse_add_section(int nid, unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); -extern void sparse_remove_section(struct mem_section *ms, - unsigned long pfn, unsigned long nr_pages, - unsigned long map_offset, struct vmem_altmap *altmap); +extern void sparse_remove_section(unsigned long pfn, unsigned long nr_pages, + struct vmem_altmap *altmap); extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum); extern struct zone *zone_for_pfn_range(int online_type, int nid, -- cgit v1.2.3 From 36ce9d76b0a93bae799e27e4f5ac35478c676592 Mon Sep 17 00:00:00 2001 From: Roberto Sassu Date: Wed, 7 Jun 2023 18:15:23 +0200 Subject: shmem: use ramfs_kill_sb() for kill_sb method of ramfs-based tmpfs As the ramfs-based tmpfs uses ramfs_init_fs_context() for the init_fs_context method, which allocates fc->s_fs_info, use ramfs_kill_sb() to free it and avoid a memory leak. Link: https://lkml.kernel.org/r/20230607161523.2876433-1-roberto.sassu@huaweicloud.com Fixes: c3b1b1cbf002 ("ramfs: add support for "mode=" mount option") Signed-off-by: Roberto Sassu Cc: Hugh Dickins Cc: David Howells Cc: Al Viro Cc: Signed-off-by: Andrew Morton --- include/linux/ramfs.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/ramfs.h b/include/linux/ramfs.h index 917528d102c4..d506dc63dd47 100644 --- a/include/linux/ramfs.h +++ b/include/linux/ramfs.h @@ -7,6 +7,7 @@ struct inode *ramfs_get_inode(struct super_block *sb, const struct inode *dir, umode_t mode, dev_t dev); extern int ramfs_init_fs_context(struct fs_context *fc); +extern void ramfs_kill_sb(struct super_block *sb); #ifdef CONFIG_MMU static inline int -- cgit v1.2.3 From a668968f84265e698a122656c433809ab9f023fa Mon Sep 17 00:00:00 2001 From: Haifeng Xu Date: Wed, 7 Jun 2023 02:45:48 +0000 Subject: mm/memory_hotplug: remove reset_node_managed_pages() in hotadd_init_pgdat() managed pages has already been set to 0 in free_area_init_core_hotplug(), via zone_init_internals() on each zone. It's pointless to reset again. Furthermore, reset_node_managed_pages() no longer needs to be exposed outside of mm/memblock.c. Remove declaration in include/linux/memblock.h and define it as static. In addtion to this, the only caller of reset_node_managed_pages() is reset_all_zones_managed_pages(), which is annotated with __init, so it should be safe to also mark reset_node_managed_pages() as __init. Link: https://lkml.kernel.org/r/20230607024548.1240-1-haifeng.xu@shopee.com Signed-off-by: Haifeng Xu Suggested-by: David Hildenbrand Cc: Michal Hocko Cc: Mike Rapoport (IBM) Cc: Oscar Salvador Signed-off-by: Andrew Morton --- include/linux/memblock.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/memblock.h b/include/linux/memblock.h index f82ee3fac1cd..f71ff9f0ec81 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -128,7 +128,6 @@ int memblock_clear_nomap(phys_addr_t base, phys_addr_t size); void memblock_free_all(void); void memblock_free(void *ptr, size_t size); -void reset_node_managed_pages(pg_data_t *pgdat); void reset_all_zones_managed_pages(void); /* Low level functions */ -- cgit v1.2.3 From b9c91c43412f2e07a5287dfe7027acdd8fb0b1ef Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Wed, 7 Jun 2023 19:51:43 +0000 Subject: mm: zswap: support exclusive loads Commit 71024cb4a0bf ("frontswap: remove frontswap_tmem_exclusive_gets") removed support for exclusive loads from frontswap as it was not used. Bring back exclusive loads support to frontswap by adding an "exclusive" output parameter to frontswap_ops->load. On the zswap side, add a module parameter to enable/disable exclusive loads, and a config option to control the boot default value. Refactor zswap entry invalidation in zswap_frontswap_invalidate_page() into zswap_invalidate_entry() to reuse it in zswap_frontswap_load() if exclusive loads are enabled. With exclusive loads, we avoid having two copies of the same page in memory (compressed & uncompressed) after faulting it in from zswap. On the other hand, if the page is to be reclaimed again without being dirtied, it will be re-compressed. Compression is not usually slow, and a page that was just faulted in is less likely to be reclaimed again soon. Link: https://lkml.kernel.org/r/20230607195143.1473802-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Suggested-by: Yu Zhao Acked-by: Johannes Weiner Cc: Dan Streetman Cc: Domenico Cerasuolo Cc: Konrad Rzeszutek Wilk Cc: Nhat Pham Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton --- include/linux/frontswap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h index a631bac12220..eaa0ac5f9003 100644 --- a/include/linux/frontswap.h +++ b/include/linux/frontswap.h @@ -10,7 +10,7 @@ struct frontswap_ops { void (*init)(unsigned); /* this swap type was just swapon'ed */ int (*store)(unsigned, pgoff_t, struct page *); /* store a page */ - int (*load)(unsigned, pgoff_t, struct page *); /* load a page */ + int (*load)(unsigned, pgoff_t, struct page *, bool *); /* load a page */ void (*invalidate_page)(unsigned, pgoff_t); /* page no longer needed */ void (*invalidate_area)(unsigned); /* swap type just swapoff'ed */ }; -- cgit v1.2.3 From 26e1a0c3277d7f43856ec424902423be212cc178 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Thu, 8 Jun 2023 18:06:53 -0700 Subject: mm: use pmdp_get_lockless() without surplus barrier() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch series "mm: allow pte_offset_map[_lock]() to fail", v2. What is it all about? Some mmap_lock avoidance i.e. latency reduction. Initially just for the case of collapsing shmem or file pages to THPs; but likely to be relied upon later in other contexts e.g. freeing of empty page tables (but that's not work I'm doing). mmap_write_lock avoidance when collapsing to anon THPs? Perhaps, but again that's not work I've done: a quick attempt was not as easy as the shmem/file case. I would much prefer not to have to make these small but wide-ranging changes for such a niche case; but failed to find another way, and have heard that shmem MADV_COLLAPSE's usefulness is being limited by that mmap_write_lock it currently requires. These changes (though of course not these exact patches) have been in Google's data centre kernel for three years now: we do rely upon them. What is this preparatory series about? The current mmap locking will not be enough to guard against that tricky transition between pmd entry pointing to page table, and empty pmd entry, and pmd entry pointing to huge page: pte_offset_map() will have to validate the pmd entry for itself, returning NULL if no page table is there. What to do about that varies: sometimes nearby error handling indicates just to skip it; but in many cases an ACTION_AGAIN or "goto again" is appropriate (and if that risks an infinite loop, then there must have been an oops, or pfn 0 mistaken for page table, before). Given the likely extension to freeing empty page tables, I have not limited this set of changes to a THP config; and it has been easier, and sets a better example, if each site is given appropriate handling: even where deeper study might prove that failure could only happen if the pmd table were corrupted. Several of the patches are, or include, cleanup on the way; and by the end, pmd_trans_unstable() and suchlike are deleted: pte_offset_map() and pte_offset_map_lock() then handle those original races and more. Most uses of pte_lockptr() are deprecated, with pte_offset_map_nolock() taking its place. This patch (of 32): Use pmdp_get_lockless() in preference to READ_ONCE(*pmdp), to get a more reliable result with PAE (or READ_ONCE as before without PAE); and remove the unnecessary extra barrier()s which got left behind in its callers. HOWEVER: Note the small print in linux/pgtable.h, where it was designed specifically for fast GUP, and depends on interrupts being disabled for its full guarantee: most callers which have been added (here and before) do NOT have interrupts disabled, so there is still some need for caution. Link: https://lkml.kernel.org/r/f35279a9-9ac0-de22-d245-591afbfb4dc@google.com Signed-off-by: Hugh Dickins Acked-by: Yu Zhao Acked-by: Peter Xu Cc: Alistair Popple Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Ira Weiny Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox Cc: Mel Gorman Cc: Miaohe Lin Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Minchan Kim Cc: Naoya Horiguchi Cc: Pavel Tatashin Cc: Peter Zijlstra Cc: Qi Zheng Cc: Ralph Campbell Cc: Ryan Roberts Cc: SeongJae Park Cc: Song Liu Cc: Steven Price Cc: Suren Baghdasaryan Cc: Thomas Hellström Cc: Will Deacon Cc: Yang Shi Cc: Zack Rusin Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index c5a51481bbb9..8ec27fe69dc8 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1344,23 +1344,6 @@ static inline int pud_trans_unstable(pud_t *pud) static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) { pmd_t pmdval = pmdp_get_lockless(pmd); - /* - * The barrier will stabilize the pmdval in a register or on - * the stack so that it will stop changing under the code. - * - * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, - * pmdp_get_lockless is allowed to return a not atomic pmdval - * (for example pointing to an hugepage that has never been - * mapped in the pmd). The below checks will only care about - * the low part of the pmd with 32bit PAE x86 anyway, with the - * exception of pmd_none(). So the important thing is that if - * the low part of the pmd is found null, the high part will - * be also null or the pmd_none() check below would be - * confused. - */ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - barrier(); -#endif /* * !pmd_present() checks for pmd migration entries * -- cgit v1.2.3 From 0cb8fd4d14165a7e654048e43983d86f75b90879 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Thu, 8 Jun 2023 18:08:20 -0700 Subject: mm/migrate: remove cruft from migration_entry_wait()s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit migration_entry_wait_on_locked() does not need to take a mapped pte pointer, its callers can do the unmap first. Annotate it with __releases(ptl) to reduce sparse warnings. Fold __migration_entry_wait_huge() into migration_entry_wait_huge(). Fold __migration_entry_wait() into migration_entry_wait(), preferring the tighter pte_offset_map_lock() to pte_offset_map() and pte_lockptr(). Link: https://lkml.kernel.org/r/b0e2a532-cdf2-561b-e999-f3b13b8d6d3@google.com Signed-off-by: Hugh Dickins Reviewed-by: Alistair Popple Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Ira Weiny Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox Cc: Mel Gorman Cc: Miaohe Lin Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Minchan Kim Cc: Naoya Horiguchi Cc: Pavel Tatashin Cc: Peter Xu Cc: Peter Zijlstra Cc: Qi Zheng Cc: Ralph Campbell Cc: Ryan Roberts Cc: SeongJae Park Cc: Song Liu Cc: Steven Price Cc: Suren Baghdasaryan Cc: Thomas Hellström Cc: Will Deacon Cc: Yang Shi Cc: Yu Zhao Cc: Zack Rusin Signed-off-by: Andrew Morton --- include/linux/migrate.h | 4 ++-- include/linux/swapops.h | 17 +++-------------- 2 files changed, 5 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 6de5756d8533..711dd9412561 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -75,8 +75,8 @@ bool isolate_movable_page(struct page *page, isolate_mode_t mode); int migrate_huge_page_move_mapping(struct address_space *mapping, struct folio *dst, struct folio *src); -void migration_entry_wait_on_locked(swp_entry_t entry, pte_t *ptep, - spinlock_t *ptl); +void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) + __releases(ptl); void folio_migrate_flags(struct folio *newfolio, struct folio *folio); void folio_migrate_copy(struct folio *newfolio, struct folio *folio); int folio_migrate_mapping(struct address_space *mapping, diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 3a451b7afcb3..4c932cb45e0b 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -332,15 +332,9 @@ static inline bool is_migration_entry_dirty(swp_entry_t entry) return false; } -extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, - spinlock_t *ptl); extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address); -#ifdef CONFIG_HUGETLB_PAGE -extern void __migration_entry_wait_huge(struct vm_area_struct *vma, - pte_t *ptep, spinlock_t *ptl); extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); -#endif /* CONFIG_HUGETLB_PAGE */ #else /* CONFIG_MIGRATION */ static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) { @@ -362,15 +356,10 @@ static inline int is_migration_entry(swp_entry_t swp) return 0; } -static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, - spinlock_t *ptl) { } static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, - unsigned long address) { } -#ifdef CONFIG_HUGETLB_PAGE -static inline void __migration_entry_wait_huge(struct vm_area_struct *vma, - pte_t *ptep, spinlock_t *ptl) { } -static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { } -#endif /* CONFIG_HUGETLB_PAGE */ + unsigned long address) { } +static inline void migration_entry_wait_huge(struct vm_area_struct *vma, + pte_t *pte) { } static inline int is_writable_migration_entry(swp_entry_t entry) { return 0; -- cgit v1.2.3 From 46c475bd676bb05077c8a38b37f175552f035406 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Thu, 8 Jun 2023 18:09:25 -0700 Subject: mm/pgtable: kmap_local_page() instead of kmap_atomic() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pte_offset_map() was still using kmap_atomic(): update it to the preferred kmap_local_page() before making further changes there, in case we need this as a bisection point; but I doubt it can cause any trouble. Link: https://lkml.kernel.org/r/d74dc4b3-6a76-446f-8f5-52ae271fa07d@google.com Signed-off-by: Hugh Dickins Cc: Alistair Popple Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Ira Weiny Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox Cc: Mel Gorman Cc: Miaohe Lin Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Minchan Kim Cc: Naoya Horiguchi Cc: Pavel Tatashin Cc: Peter Xu Cc: Peter Zijlstra Cc: Qi Zheng Cc: Ralph Campbell Cc: Ryan Roberts Cc: SeongJae Park Cc: Song Liu Cc: Steven Price Cc: Suren Baghdasaryan Cc: Thomas Hellström Cc: Will Deacon Cc: Yang Shi Cc: Yu Zhao Cc: Zack Rusin Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8ec27fe69dc8..94235ff2706e 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -96,9 +96,9 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) #if defined(CONFIG_HIGHPTE) #define pte_offset_map(dir, address) \ - ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ + ((pte_t *)kmap_local_page(pmd_page(*(dir))) + \ pte_index((address))) -#define pte_unmap(pte) kunmap_atomic((pte)) +#define pte_unmap(pte) kunmap_local((pte)) #else #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) #define pte_unmap(pte) ((void)(pte)) /* NOP */ -- cgit v1.2.3 From 0d940a9b270b9220dcff74d8e9123c9788365751 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Thu, 8 Jun 2023 18:10:32 -0700 Subject: mm/pgtable: allow pte_offset_map[_lock]() to fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make pte_offset_map() a wrapper for __pte_offset_map() (optionally outputs pmdval), pte_offset_map_lock() a sparse __cond_lock wrapper for __pte_offset_map_lock(): those __funcs added in mm/pgtable-generic.c. __pte_offset_map() do pmdval validation (including pmd_clear_bad() when pmd_bad()), returning NULL if pmdval is not for a page table. __pte_offset_map_lock() verify pmdval unchanged after getting the lock, trying again if it changed. No #ifdef CONFIG_TRANSPARENT_HUGEPAGE around them: that could be done to cover the imminent case, but we expect to generalize it later, and it makes a mess of where to do the pmd_bad() clearing. Add pte_offset_map_nolock(): outputs ptl like pte_offset_map_lock(), without actually taking the lock. This will be preferred to open uses of pte_lockptr(), because (when split ptlock is in page table's struct page) it points to the right lock for the returned pte pointer, even if *pmd gets changed racily afterwards. Update corresponding Documentation. Do not add the anticipated rcu_read_lock() and rcu_read_unlock()s yet: they have to wait until all architectures are balancing pte_offset_map()s with pte_unmap()s (as in the arch series posted earlier). But comment where they will go, so that it's easy to add them for experiments. And only when those are in place can transient racy failure cases be enabled. Add more safety for the PAE mismatched pmd_low pmd_high case at that time. Link: https://lkml.kernel.org/r/2929bfd-9893-a374-e463-4c3127ff9b9d@google.com Signed-off-by: Hugh Dickins Cc: Alistair Popple Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Ira Weiny Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox Cc: Mel Gorman Cc: Miaohe Lin Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Minchan Kim Cc: Naoya Horiguchi Cc: Pavel Tatashin Cc: Peter Xu Cc: Peter Zijlstra Cc: Qi Zheng Cc: Ralph Campbell Cc: Ryan Roberts Cc: SeongJae Park Cc: Song Liu Cc: Steven Price Cc: Suren Baghdasaryan Cc: Thomas Hellström Cc: Will Deacon Cc: Yang Shi Cc: Yu Zhao Cc: Zack Rusin Signed-off-by: Andrew Morton --- include/linux/mm.h | 27 +++++++++++++++++++-------- include/linux/pgtable.h | 22 +++++++++++++++------- 2 files changed, 34 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 66032f0d515c..a08dc8cc48fb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2827,14 +2827,25 @@ static inline void pgtable_pte_page_dtor(struct page *page) dec_lruvec_page_state(page, NR_PAGETABLE); } -#define pte_offset_map_lock(mm, pmd, address, ptlp) \ -({ \ - spinlock_t *__ptl = pte_lockptr(mm, pmd); \ - pte_t *__pte = pte_offset_map(pmd, address); \ - *(ptlp) = __ptl; \ - spin_lock(__ptl); \ - __pte; \ -}) +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); +static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) +{ + return __pte_offset_map(pmd, addr, NULL); +} + +pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, spinlock_t **ptlp); +static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, spinlock_t **ptlp) +{ + pte_t *pte; + + __cond_lock(*ptlp, pte = __pte_offset_map_lock(mm, pmd, addr, ptlp)); + return pte; +} + +pte_t *pte_offset_map_nolock(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, spinlock_t **ptlp); #define pte_unmap_unlock(pte, ptl) do { \ spin_unlock(ptl); \ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 94235ff2706e..3fabbb018557 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -94,14 +94,22 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) #define pte_offset_kernel pte_offset_kernel #endif -#if defined(CONFIG_HIGHPTE) -#define pte_offset_map(dir, address) \ - ((pte_t *)kmap_local_page(pmd_page(*(dir))) + \ - pte_index((address))) -#define pte_unmap(pte) kunmap_local((pte)) +#ifdef CONFIG_HIGHPTE +#define __pte_map(pmd, address) \ + ((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address))) +#define pte_unmap(pte) do { \ + kunmap_local((pte)); \ + /* rcu_read_unlock() to be added later */ \ +} while (0) #else -#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) -#define pte_unmap(pte) ((void)(pte)) /* NOP */ +static inline pte_t *__pte_map(pmd_t *pmd, unsigned long address) +{ + return pte_offset_kernel(pmd, address); +} +static inline void pte_unmap(pte_t *pte) +{ + /* rcu_read_unlock() to be added later */ +} #endif /* Find an entry in the second-level page table.. */ -- cgit v1.2.3 From feda5c393a6c843c7bf1fc49e1381e2d3822b564 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Thu, 8 Jun 2023 18:50:37 -0700 Subject: mm/pgtable: delete pmd_trans_unstable() and friends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete pmd_trans_unstable, pmd_none_or_trans_huge_or_clear_bad() and pmd_devmap_trans_unstable(), all now unused. With mixed feelings, delete all the comments on pmd_trans_unstable(). That was very good documentation of a subtle state, and this series does not even eliminate that state: but rather, normalizes and extends it, asking pte_offset_map[_lock]() callers to anticipate failure, without regard for whether mmap_read_lock() or mmap_write_lock() is held. Retain pud_trans_unstable(), which has one use in __handle_mm_fault(), but delete its equivalent pud_none_or_trans_huge_or_dev_or_clear_bad(). While there, move the default arch_needs_pgtable_deposit() definition up near where pgtable_trans_huge_deposit() and withdraw() are declared. Link: https://lkml.kernel.org/r/5abdab3-3136-b42e-274d-9c6281bfb79@google.com Signed-off-by: Hugh Dickins Cc: Alistair Popple Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Ira Weiny Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox Cc: Mel Gorman Cc: Miaohe Lin Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Minchan Kim Cc: Naoya Horiguchi Cc: Pavel Tatashin Cc: Peter Xu Cc: Peter Zijlstra Cc: Qi Zheng Cc: Ralph Campbell Cc: Ryan Roberts Cc: SeongJae Park Cc: Song Liu Cc: Steven Price Cc: Suren Baghdasaryan Cc: Thomas Hellström Cc: Will Deacon Cc: Yang Shi Cc: Yu Zhao Cc: Zack Rusin Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 103 ++++-------------------------------------------- 1 file changed, 7 insertions(+), 96 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 3fabbb018557..a1326e61d7ee 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -599,6 +599,10 @@ extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); #endif +#ifndef arch_needs_pgtable_deposit +#define arch_needs_pgtable_deposit() (false) +#endif + #ifdef CONFIG_TRANSPARENT_HUGEPAGE /* * This is an implementation of pmdp_establish() that is only suitable for an @@ -1300,9 +1304,10 @@ static inline int pud_trans_huge(pud_t pud) } #endif -/* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ -static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) +static inline int pud_trans_unstable(pud_t *pud) { +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ + defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) pud_t pudval = READ_ONCE(*pud); if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) @@ -1311,104 +1316,10 @@ static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) pud_clear_bad(pud); return 1; } - return 0; -} - -/* See pmd_trans_unstable for discussion. */ -static inline int pud_trans_unstable(pud_t *pud) -{ -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ - defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) - return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); -#else - return 0; #endif -} - -#ifndef arch_needs_pgtable_deposit -#define arch_needs_pgtable_deposit() (false) -#endif -/* - * This function is meant to be used by sites walking pagetables with - * the mmap_lock held in read mode to protect against MADV_DONTNEED and - * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd - * into a null pmd and the transhuge page fault can convert a null pmd - * into an hugepmd or into a regular pmd (if the hugepage allocation - * fails). While holding the mmap_lock in read mode the pmd becomes - * stable and stops changing under us only if it's not null and not a - * transhuge pmd. When those races occurs and this function makes a - * difference vs the standard pmd_none_or_clear_bad, the result is - * undefined so behaving like if the pmd was none is safe (because it - * can return none anyway). The compiler level barrier() is critically - * important to compute the two checks atomically on the same pmdval. - * - * For 32bit kernels with a 64bit large pmd_t this automatically takes - * care of reading the pmd atomically to avoid SMP race conditions - * against pmd_populate() when the mmap_lock is hold for reading by the - * caller (a special atomic read not done by "gcc" as in the generic - * version above, is also needed when THP is disabled because the page - * fault can populate the pmd from under us). - */ -static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) -{ - pmd_t pmdval = pmdp_get_lockless(pmd); - /* - * !pmd_present() checks for pmd migration entries - * - * The complete check uses is_pmd_migration_entry() in linux/swapops.h - * But using that requires moving current function and pmd_trans_unstable() - * to linux/swapops.h to resolve dependency, which is too much code move. - * - * !pmd_present() is equivalent to is_pmd_migration_entry() currently, - * because !pmd_present() pages can only be under migration not swapped - * out. - * - * pmd_none() is preserved for future condition checks on pmd migration - * entries and not confusing with this function name, although it is - * redundant with !pmd_present(). - */ - if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || - (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) - return 1; - if (unlikely(pmd_bad(pmdval))) { - pmd_clear_bad(pmd); - return 1; - } return 0; } -/* - * This is a noop if Transparent Hugepage Support is not built into - * the kernel. Otherwise it is equivalent to - * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in - * places that already verified the pmd is not none and they want to - * walk ptes while holding the mmap sem in read mode (write mode don't - * need this). If THP is not enabled, the pmd can't go away under the - * code even if MADV_DONTNEED runs, but if THP is enabled we need to - * run a pmd_trans_unstable before walking the ptes after - * split_huge_pmd returns (because it may have run when the pmd become - * null, but then a page fault can map in a THP and not a regular page). - */ -static inline int pmd_trans_unstable(pmd_t *pmd) -{ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - return pmd_none_or_trans_huge_or_clear_bad(pmd); -#else - return 0; -#endif -} - -/* - * the ordering of these checks is important for pmds with _page_devmap set. - * if we check pmd_trans_unstable() first we will trip the bad_pmd() check - * inside of pmd_none_or_trans_huge_or_clear_bad(). this will end up correctly - * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. - */ -static inline int pmd_devmap_trans_unstable(pmd_t *pmd) -{ - return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); -} - #ifndef CONFIG_NUMA_BALANCING /* * Technically a PTE can be PROTNONE even when not doing NUMA balancing but -- cgit v1.2.3 From 4f8fcf4ced0b7184149045818dcc2f9e2689b775 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Thu, 8 Jun 2023 18:52:17 -0700 Subject: mm/swap: swap_vma_readahead() do the pte_offset_map() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit swap_vma_readahead() has been proceeding in an unconventional way, its preliminary swap_ra_info() doing the pte_offset_map() and pte_unmap(), then relying on that pte pointer even after the pte_unmap() - in its CONFIG_64BIT case (I think !CONFIG_HIGHPTE was intended; whereas 32-bit copied ptes to stack while they were mapped, but had to limit how many). Though it would be difficult to construct a failing testcase, accessing page table after pte_unmap() will become bad practice, even on 64-bit: an rcu_read_unlock() in pte_unmap() will allow page table to be freed. Move relevant definitions from include/linux/swap.h to mm/swap_state.c, nothing else used them. Delete the CONFIG_64BIT distinction and buffer, delete all reference to ptes from swap_ra_info(), use pte_offset_map() repeatedly in swap_vma_readahead(), breaking from the loop if it fails. (Will the repeated "map" and "unmap" show up as a slowdown anywhere? If so, maybe modify __read_swap_cache_async() to do the pte_unmap() only when it does not find the page already in the swapcache.) Use ptep_get_lockless(), mainly for its READ_ONCE(). Correctly advance the address passed down to each call of __read__swap_cache_async(). Link: https://lkml.kernel.org/r/b7c64ab3-9e44-aac0-d2b-c57de578af1c@google.com Signed-off-by: Hugh Dickins Reviewed-by: "Huang, Ying" Cc: Alistair Popple Cc: Anshuman Khandual Cc: Axel Rasmussen Cc: Christophe Leroy Cc: Christoph Hellwig Cc: David Hildenbrand Cc: Ira Weiny Cc: Jason Gunthorpe Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Matthew Wilcox Cc: Mel Gorman Cc: Miaohe Lin Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Minchan Kim Cc: Naoya Horiguchi Cc: Pavel Tatashin Cc: Peter Xu Cc: Peter Zijlstra Cc: Qi Zheng Cc: Ralph Campbell Cc: Ryan Roberts Cc: SeongJae Park Cc: Song Liu Cc: Steven Price Cc: Suren Baghdasaryan Cc: Thomas Hellström Cc: Will Deacon Cc: Yang Shi Cc: Yu Zhao Cc: Zack Rusin Signed-off-by: Andrew Morton --- include/linux/swap.h | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index b5f6f2916de1..ce7e82cf787f 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -337,25 +337,6 @@ struct swap_info_struct { */ }; -#ifdef CONFIG_64BIT -#define SWAP_RA_ORDER_CEILING 5 -#else -/* Avoid stack overflow, because we need to save part of page table */ -#define SWAP_RA_ORDER_CEILING 3 -#define SWAP_RA_PTE_CACHE_SIZE (1 << SWAP_RA_ORDER_CEILING) -#endif - -struct vma_swap_readahead { - unsigned short win; - unsigned short offset; - unsigned short nr_pte; -#ifdef CONFIG_64BIT - pte_t *ptes; -#else - pte_t ptes[SWAP_RA_PTE_CACHE_SIZE]; -#endif -}; - static inline swp_entry_t folio_swap_entry(struct folio *folio) { swp_entry_t entry = { .val = page_private(&folio->page) }; -- cgit v1.2.3 From b95826c9aa48b2997b3973b42a8716ba132b920e Mon Sep 17 00:00:00 2001 From: Sidhartha Kumar Date: Mon, 12 Jun 2023 09:34:05 -0700 Subject: mm: remove set_compound_page_dtor() All users can use the folio equivalent so this function can be safely removed. Link: https://lkml.kernel.org/r/20230612163405.99345-1-sidhartha.kumar@oracle.com Signed-off-by: Sidhartha Kumar Cc: Aneesh Kumar K.V Cc: Matthew Wilcox Cc: Tarun Sahu Signed-off-by: Andrew Morton --- include/linux/mm.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index a08dc8cc48fb..8f40bf17d597 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1223,16 +1223,6 @@ enum compound_dtor_id { }; extern compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS]; -static inline void set_compound_page_dtor(struct page *page, - enum compound_dtor_id compound_dtor) -{ - struct folio *folio = (struct folio *)page; - - VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page); - VM_BUG_ON_PAGE(!PageHead(page), page); - folio->_folio_dtor = compound_dtor; -} - static inline void folio_set_compound_dtor(struct folio *folio, enum compound_dtor_id compound_dtor) { -- cgit v1.2.3 From 4ab5f8ec7d71aea5fe13a48248242130f84ac6bb Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 12 Jun 2023 16:31:45 +0100 Subject: mm/slab: decouple ARCH_KMALLOC_MINALIGN from ARCH_DMA_MINALIGN Patch series "mm, dma, arm64: Reduce ARCH_KMALLOC_MINALIGN to 8", v7. A series reducing the kmalloc() minimum alignment on arm64 to 8 (from 128). This patch (of 17): In preparation for supporting a kmalloc() minimum alignment smaller than the arch DMA alignment, decouple the two definitions. This requires that either the kmalloc() caches are aligned to a (run-time) cache-line size or the DMA API bounces unaligned kmalloc() allocations. Subsequent patches will implement both options. After this patch, ARCH_DMA_MINALIGN is expected to be used in static alignment annotations and defined by an architecture to be the maximum alignment for all supported configurations/SoCs in a single Image. Architectures opting in to a smaller ARCH_KMALLOC_MINALIGN will need to define its value in the arch headers. Since ARCH_DMA_MINALIGN is now always defined, adjust the #ifdef in dma_get_cache_alignment() so that there is no change for architectures not requiring a minimum DMA alignment. Link: https://lkml.kernel.org/r/20230612153201.554742-1-catalin.marinas@arm.com Link: https://lkml.kernel.org/r/20230612153201.554742-2-catalin.marinas@arm.com Signed-off-by: Catalin Marinas Tested-by: Isaac J. Manjarres Cc: Vlastimil Babka Cc: Christoph Hellwig Cc: Robin Murphy Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joerg Roedel Cc: Jonathan Cameron Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: Rafael J. Wysocki Cc: Saravana Kannan Cc: Will Deacon Cc: Jerry Snitselaar Cc: Jonathan Cameron Cc: Lars-Peter Clausen Cc: Logan Gunthorpe Signed-off-by: Andrew Morton --- include/linux/cache.h | 6 ++++++ include/linux/dma-mapping.h | 3 ++- include/linux/slab.h | 14 ++++++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cache.h b/include/linux/cache.h index 5da1bbd96154..9900d20b76c2 100644 --- a/include/linux/cache.h +++ b/include/linux/cache.h @@ -98,4 +98,10 @@ struct cacheline_padding { #define CACHELINE_PADDING(name) #endif +#ifdef ARCH_DMA_MINALIGN +#define ARCH_HAS_DMA_MINALIGN +#else +#define ARCH_DMA_MINALIGN __alignof__(unsigned long long) +#endif + #endif /* __LINUX_CACHE_H */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 0ee20b764000..a50375331eac 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -2,6 +2,7 @@ #ifndef _LINUX_DMA_MAPPING_H #define _LINUX_DMA_MAPPING_H +#include #include #include #include @@ -545,7 +546,7 @@ static inline int dma_set_min_align_mask(struct device *dev, static inline int dma_get_cache_alignment(void) { -#ifdef ARCH_DMA_MINALIGN +#ifdef ARCH_HAS_DMA_MINALIGN return ARCH_DMA_MINALIGN; #endif return 1; diff --git a/include/linux/slab.h b/include/linux/slab.h index 6b3e155b70bf..ca53425e9b32 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -12,6 +12,7 @@ #ifndef _LINUX_SLAB_H #define _LINUX_SLAB_H +#include #include #include #include @@ -235,12 +236,17 @@ void kmem_dump_obj(void *object); * alignment larger than the alignment of a 64-bit integer. * Setting ARCH_DMA_MINALIGN in arch headers allows that. */ -#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 +#ifdef ARCH_HAS_DMA_MINALIGN +#if ARCH_DMA_MINALIGN > 8 && !defined(ARCH_KMALLOC_MINALIGN) #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN -#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN -#define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) -#else +#endif +#endif + +#ifndef ARCH_KMALLOC_MINALIGN #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) +#elif ARCH_KMALLOC_MINALIGN > 8 +#define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN +#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) #endif /* -- cgit v1.2.3 From 8c57da28dc3df4e091474a004b5596c9b88a3be0 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 12 Jun 2023 16:31:46 +0100 Subject: dma: allow dma_get_cache_alignment() to be overridden by the arch code On arm64, ARCH_DMA_MINALIGN is larger than most cache line size configurations deployed. Allow an architecture to override dma_get_cache_alignment() in order to return a run-time probed value (e.g. cache_line_size()). Link: https://lkml.kernel.org/r/20230612153201.554742-3-catalin.marinas@arm.com Signed-off-by: Catalin Marinas Reviewed-by: Christoph Hellwig Tested-by: Isaac J. Manjarres Cc: Robin Murphy Cc: Will Deacon Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Jerry Snitselaar Cc: Joerg Roedel Cc: Jonathan Cameron Cc: Jonathan Cameron Cc: Lars-Peter Clausen Cc: Logan Gunthorpe Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: "Rafael J. Wysocki" Cc: Saravana Kannan Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- include/linux/dma-mapping.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index a50375331eac..e13050eb9777 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -544,6 +544,7 @@ static inline int dma_set_min_align_mask(struct device *dev, return 0; } +#ifndef dma_get_cache_alignment static inline int dma_get_cache_alignment(void) { #ifdef ARCH_HAS_DMA_MINALIGN @@ -551,6 +552,7 @@ static inline int dma_get_cache_alignment(void) #endif return 1; } +#endif static inline void *dmam_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp) -- cgit v1.2.3 From 88b216d339691888ef98644a5eae62c3d9c8ddf0 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 12 Jun 2023 16:31:54 +0100 Subject: iio: core: use ARCH_DMA_MINALIGN instead of ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN represents the minimum (static) alignment for safe DMA operations while ARCH_KMALLOC_MINALIGN is the minimum kmalloc() objects alignment. Link: https://lkml.kernel.org/r/20230612153201.554742-11-catalin.marinas@arm.com Signed-off-by: Catalin Marinas Acked-by: Jonathan Cameron Tested-by: Isaac J. Manjarres Cc: Lars-Peter Clausen Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Christoph Hellwig Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Jerry Snitselaar Cc: Joerg Roedel Cc: Jonathan Cameron Cc: Logan Gunthorpe Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: "Rafael J. Wysocki" Cc: Robin Murphy Cc: Saravana Kannan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/iio/iio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 81413cd3a3e7..d28a5e8097e4 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -722,7 +722,7 @@ static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev) * must not share cachelines with the rest of the structure, thus making * them safe for use with non-coherent DMA. */ -#define IIO_DMA_MINALIGN ARCH_KMALLOC_MINALIGN +#define IIO_DMA_MINALIGN ARCH_DMA_MINALIGN struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv); /* The information at the returned address is guaranteed to be cacheline aligned */ -- cgit v1.2.3 From af2880ec44021d32cc72a5aa7c5d7d7beaa722d3 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jun 2023 16:31:56 +0100 Subject: scatterlist: add dedicated config for DMA flags The DMA flags field will be useful for users beyond PCI P2P, so upgrade to its own dedicated config option. [catalin.marinas@arm.com: use #ifdef CONFIG_NEED_SG_DMA_FLAGS in scatterlist.h] [catalin.marinas@arm.com: update PCI_P2PDMA dma_flags comment in scatterlist.h] Link: https://lkml.kernel.org/r/20230612153201.554742-13-catalin.marinas@arm.com Signed-off-by: Robin Murphy Signed-off-by: Catalin Marinas Reviewed-by: Christoph Hellwig Tested-by: Isaac J. Manjarres Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Jerry Snitselaar Cc: Joerg Roedel Cc: Jonathan Cameron Cc: Jonathan Cameron Cc: Lars-Peter Clausen Cc: Logan Gunthorpe Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: "Rafael J. Wysocki" Cc: Saravana Kannan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/scatterlist.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index 375a5e90d86a..19833fd4113b 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -16,7 +16,7 @@ struct scatterlist { #ifdef CONFIG_NEED_SG_DMA_LENGTH unsigned int dma_length; #endif -#ifdef CONFIG_PCI_P2PDMA +#ifdef CONFIG_NEED_SG_DMA_FLAGS unsigned int dma_flags; #endif }; @@ -249,12 +249,11 @@ static inline void sg_unmark_end(struct scatterlist *sg) } /* - * CONFGI_PCI_P2PDMA depends on CONFIG_64BIT which means there is 4 bytes - * in struct scatterlist (assuming also CONFIG_NEED_SG_DMA_LENGTH is set). - * Use this padding for DMA flags bits to indicate when a specific - * dma address is a bus address. + * One 64-bit architectures there is a 4-byte padding in struct scatterlist + * (assuming also CONFIG_NEED_SG_DMA_LENGTH is set). Use this padding for DMA + * flags bits to indicate when a specific dma address is a bus address. */ -#ifdef CONFIG_PCI_P2PDMA +#ifdef CONFIG_NEED_SG_DMA_FLAGS #define SG_DMA_BUS_ADDRESS (1 << 0) @@ -312,7 +311,7 @@ static inline void sg_dma_unmark_bus_address(struct scatterlist *sg) { } -#endif +#endif /* CONFIG_NEED_SG_DMA_FLAGS */ /** * sg_phys - Return physical address of an sg entry -- cgit v1.2.3 From cb147bbe22d2be9b49021c2e5dacdf2935745d1c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jun 2023 16:31:57 +0100 Subject: dma-mapping: name SG DMA flag helpers consistently sg_is_dma_bus_address() is inconsistent with the naming pattern of its corresponding setters and its own kerneldoc, so take the majority vote and rename it sg_dma_is_bus_address() (and fix up the missing underscores in the kerneldoc too). This gives us a nice clear pattern where SG DMA flags are SG_DMA_, and the helpers for acting on them are sg_dma__(). Link: https://lkml.kernel.org/r/20230612153201.554742-14-catalin.marinas@arm.com Signed-off-by: Robin Murphy Signed-off-by: Catalin Marinas Reviewed-by: Christoph Hellwig Reviewed-by: Jerry Snitselaar Reviewed-by: Logan Gunthorpe Link: https://lore.kernel.org/r/fa2eca2862c7ffc41b50337abffb2dfd2864d3ea.1685036694.git.robin.murphy@arm.com Tested-by: Isaac J. Manjarres Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Joerg Roedel Cc: Jonathan Cameron Cc: Jonathan Cameron Cc: Lars-Peter Clausen Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: "Rafael J. Wysocki" Cc: Saravana Kannan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/scatterlist.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index 19833fd4113b..2f06178996ba 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -258,7 +258,7 @@ static inline void sg_unmark_end(struct scatterlist *sg) #define SG_DMA_BUS_ADDRESS (1 << 0) /** - * sg_dma_is_bus address - Return whether a given segment was marked + * sg_dma_is_bus_address - Return whether a given segment was marked * as a bus address * @sg: SG entry * @@ -266,13 +266,13 @@ static inline void sg_unmark_end(struct scatterlist *sg) * Returns true if sg_dma_mark_bus_address() has been called on * this segment. **/ -static inline bool sg_is_dma_bus_address(struct scatterlist *sg) +static inline bool sg_dma_is_bus_address(struct scatterlist *sg) { return sg->dma_flags & SG_DMA_BUS_ADDRESS; } /** - * sg_dma_mark_bus address - Mark the scatterlist entry as a bus address + * sg_dma_mark_bus_address - Mark the scatterlist entry as a bus address * @sg: SG entry * * Description: @@ -300,7 +300,7 @@ static inline void sg_dma_unmark_bus_address(struct scatterlist *sg) #else -static inline bool sg_is_dma_bus_address(struct scatterlist *sg) +static inline bool sg_dma_is_bus_address(struct scatterlist *sg) { return false; } -- cgit v1.2.3 From 370645f41e6e2fdd2fb6f6982530b04612c9793c Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 12 Jun 2023 16:31:58 +0100 Subject: dma-mapping: force bouncing if the kmalloc() size is not cache-line-aligned For direct DMA, if the size is small enough to have originated from a kmalloc() cache below ARCH_DMA_MINALIGN, check its alignment against dma_get_cache_alignment() and bounce if necessary. For larger sizes, it is the responsibility of the DMA API caller to ensure proper alignment. At this point, the kmalloc() caches are properly aligned but this will change in a subsequent patch. Architectures can opt in by selecting DMA_BOUNCE_UNALIGNED_KMALLOC. Link: https://lkml.kernel.org/r/20230612153201.554742-15-catalin.marinas@arm.com Signed-off-by: Catalin Marinas Reviewed-by: Christoph Hellwig Reviewed-by: Robin Murphy Tested-by: Isaac J. Manjarres Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Jerry Snitselaar Cc: Joerg Roedel Cc: Jonathan Cameron Cc: Jonathan Cameron Cc: Lars-Peter Clausen Cc: Logan Gunthorpe Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: "Rafael J. Wysocki" Cc: Saravana Kannan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/dma-map-ops.h | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 31f114f486c4..9bf19b5bf755 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -8,6 +8,7 @@ #include #include +#include struct cma; @@ -277,6 +278,66 @@ static inline bool dev_is_dma_coherent(struct device *dev) } #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */ +/* + * Check whether potential kmalloc() buffers are safe for non-coherent DMA. + */ +static inline bool dma_kmalloc_safe(struct device *dev, + enum dma_data_direction dir) +{ + /* + * If DMA bouncing of kmalloc() buffers is disabled, the kmalloc() + * caches have already been aligned to a DMA-safe size. + */ + if (!IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC)) + return true; + + /* + * kmalloc() buffers are DMA-safe irrespective of size if the device + * is coherent or the direction is DMA_TO_DEVICE (non-desctructive + * cache maintenance and benign cache line evictions). + */ + if (dev_is_dma_coherent(dev) || dir == DMA_TO_DEVICE) + return true; + + return false; +} + +/* + * Check whether the given size, assuming it is for a kmalloc()'ed buffer, is + * sufficiently aligned for non-coherent DMA. + */ +static inline bool dma_kmalloc_size_aligned(size_t size) +{ + /* + * Larger kmalloc() sizes are guaranteed to be aligned to + * ARCH_DMA_MINALIGN. + */ + if (size >= 2 * ARCH_DMA_MINALIGN || + IS_ALIGNED(kmalloc_size_roundup(size), dma_get_cache_alignment())) + return true; + + return false; +} + +/* + * Check whether the given object size may have originated from a kmalloc() + * buffer with a slab alignment below the DMA-safe alignment and needs + * bouncing for non-coherent DMA. The pointer alignment is not considered and + * in-structure DMA-safe offsets are the responsibility of the caller. Such + * code should use the static ARCH_DMA_MINALIGN for compiler annotations. + * + * The heuristics can have false positives, bouncing unnecessarily, though the + * buffers would be small. False negatives are theoretically possible if, for + * example, multiple small kmalloc() buffers are coalesced into a larger + * buffer that passes the alignment check. There are no such known constructs + * in the kernel. + */ +static inline bool dma_kmalloc_needs_bounce(struct device *dev, size_t size, + enum dma_data_direction dir) +{ + return !dma_kmalloc_safe(dev, dir) && !dma_kmalloc_size_aligned(size); +} + void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, -- cgit v1.2.3 From 861370f49ce484cd6ef2e9b3ad06d137f3cb0ca3 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 12 Jun 2023 16:31:59 +0100 Subject: iommu/dma: force bouncing if the size is not cacheline-aligned Similarly to the direct DMA, bounce small allocations as they may have originated from a kmalloc() cache not safe for DMA. Unlike the direct DMA, iommu_dma_map_sg() cannot call iommu_dma_map_sg_swiotlb() for all non-coherent devices as this would break some cases where the iova is expected to be contiguous (dmabuf). Instead, scan the scatterlist for any small sizes and only go the swiotlb path if any element of the list needs bouncing (note that iommu_dma_map_page() would still only bounce those buffers which are not DMA-aligned). To avoid scanning the scatterlist on the 'sync' operations, introduce an SG_DMA_SWIOTLB flag set by iommu_dma_map_sg_swiotlb(). The dev_use_swiotlb() function together with the newly added dev_use_sg_swiotlb() now check for both untrusted devices and unaligned kmalloc() buffers (suggested by Robin Murphy). Link: https://lkml.kernel.org/r/20230612153201.554742-16-catalin.marinas@arm.com Signed-off-by: Catalin Marinas Reviewed-by: Robin Murphy Tested-by: Isaac J. Manjarres Cc: Joerg Roedel Cc: Christoph Hellwig Cc: Alasdair Kergon Cc: Ard Biesheuvel Cc: Arnd Bergmann Cc: Daniel Vetter Cc: Greg Kroah-Hartman Cc: Herbert Xu Cc: Jerry Snitselaar Cc: Jonathan Cameron Cc: Jonathan Cameron Cc: Lars-Peter Clausen Cc: Logan Gunthorpe Cc: Marc Zyngier Cc: Mark Brown Cc: Mike Snitzer Cc: "Rafael J. Wysocki" Cc: Saravana Kannan Cc: Vlastimil Babka Cc: Will Deacon Signed-off-by: Andrew Morton --- include/linux/scatterlist.h | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index 2f06178996ba..ec46d8e8e49d 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -251,11 +251,13 @@ static inline void sg_unmark_end(struct scatterlist *sg) /* * One 64-bit architectures there is a 4-byte padding in struct scatterlist * (assuming also CONFIG_NEED_SG_DMA_LENGTH is set). Use this padding for DMA - * flags bits to indicate when a specific dma address is a bus address. + * flags bits to indicate when a specific dma address is a bus address or the + * buffer may have been bounced via SWIOTLB. */ #ifdef CONFIG_NEED_SG_DMA_FLAGS -#define SG_DMA_BUS_ADDRESS (1 << 0) +#define SG_DMA_BUS_ADDRESS (1 << 0) +#define SG_DMA_SWIOTLB (1 << 1) /** * sg_dma_is_bus_address - Return whether a given segment was marked @@ -298,6 +300,34 @@ static inline void sg_dma_unmark_bus_address(struct scatterlist *sg) sg->dma_flags &= ~SG_DMA_BUS_ADDRESS; } +/** + * sg_dma_is_swiotlb - Return whether the scatterlist was marked for SWIOTLB + * bouncing + * @sg: SG entry + * + * Description: + * Returns true if the scatterlist was marked for SWIOTLB bouncing. Not all + * elements may have been bounced, so the caller would have to check + * individual SG entries with is_swiotlb_buffer(). + */ +static inline bool sg_dma_is_swiotlb(struct scatterlist *sg) +{ + return sg->dma_flags & SG_DMA_SWIOTLB; +} + +/** + * sg_dma_mark_swiotlb - Mark the scatterlist for SWIOTLB bouncing + * @sg: SG entry + * + * Description: + * Marks a a scatterlist for SWIOTLB bounce. Not all SG entries may be + * bounced. + */ +static inline void sg_dma_mark_swiotlb(struct scatterlist *sg) +{ + sg->dma_flags |= SG_DMA_SWIOTLB; +} + #else static inline bool sg_dma_is_bus_address(struct scatterlist *sg) @@ -310,6 +340,13 @@ static inline void sg_dma_mark_bus_address(struct scatterlist *sg) static inline void sg_dma_unmark_bus_address(struct scatterlist *sg) { } +static inline bool sg_dma_is_swiotlb(struct scatterlist *sg) +{ + return false; +} +static inline void sg_dma_mark_swiotlb(struct scatterlist *sg) +{ +} #endif /* CONFIG_NEED_SG_DMA_FLAGS */ -- cgit v1.2.3 From 6c1d2a073a1d850e79026411e79dff7ef997c90d Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 12 Jun 2023 16:15:44 +0100 Subject: mm: move ptep_get() and pmdp_get() helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are many call sites that directly dereference a pte_t pointer. This makes it very difficult to properly encapsulate a page table in the arch code without having to allocate shadow page tables. We will shortly solve this by replacing all the call sites with ptep_get() calls. But there are call sites above the function definition in the header file, so let's move ptep_get() to an earlier location to solve that problem. And move pmdp_get() at the same time to keep it close to ptep_get(). Link: https://lkml.kernel.org/r/20230612151545.3317766-3-ryan.roberts@arm.com Signed-off-by: Ryan Roberts Cc: Adrian Hunter Cc: Alexander Potapenko Cc: Alexander Shishkin Cc: Alex Williamson Cc: Al Viro Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Christian Brauner Cc: Christoph Hellwig Cc: Daniel Vetter Cc: Dave Airlie Cc: Dimitri Sivanich Cc: Dmitry Vyukov Cc: Ian Rogers Cc: Jason Gunthorpe Cc: Jérôme Glisse Cc: Jiri Olsa Cc: Johannes Weiner Cc: kernel test robot Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Mark Rutland Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Michal Hocko Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Namhyung Kim Cc: Naoya Horiguchi Cc: Oleksandr Tyshchenko Cc: Pavel Tatashin Cc: Roman Gushchin Cc: SeongJae Park Cc: Shakeel Butt Cc: Uladzislau Rezki (Sony) Cc: Vincenzo Frascino Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/pgtable.h | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index a1326e61d7ee..fc06f6419661 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -212,6 +212,20 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif +#ifndef ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + return READ_ONCE(*ptep); +} +#endif + +#ifndef pmdp_get +static inline pmd_t pmdp_get(pmd_t *pmdp) +{ + return READ_ONCE(*pmdp); +} +#endif + #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, @@ -317,20 +331,6 @@ static inline void ptep_clear(struct mm_struct *mm, unsigned long addr, ptep_get_and_clear(mm, addr, ptep); } -#ifndef ptep_get -static inline pte_t ptep_get(pte_t *ptep) -{ - return READ_ONCE(*ptep); -} -#endif - -#ifndef pmdp_get -static inline pmd_t pmdp_get(pmd_t *pmdp) -{ - return READ_ONCE(*pmdp); -} -#endif - #ifdef CONFIG_GUP_GET_PXX_LOW_HIGH /* * For walking the pagetables without holding any locks. Some architectures -- cgit v1.2.3 From c33c794828f21217f72ce6fc140e0d34e0d56bff Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Mon, 12 Jun 2023 16:15:45 +0100 Subject: mm: ptep_get() conversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert all instances of direct pte_t* dereferencing to instead use ptep_get() helper. This means that by default, the accesses change from a C dereference to a READ_ONCE(). This is technically the correct thing to do since where pgtables are modified by HW (for access/dirty) they are volatile and therefore we should always ensure READ_ONCE() semantics. But more importantly, by always using the helper, it can be overridden by the architecture to fully encapsulate the contents of the pte. Arch code is deliberately not converted, as the arch code knows best. It is intended that arch code (arm64) will override the default with its own implementation that can (e.g.) hide certain bits from the core code, or determine young/dirty status by mixing in state from another source. Conversion was done using Coccinelle: ---- // $ make coccicheck \ // COCCI=ptepget.cocci \ // SPFLAGS="--include-headers" \ // MODE=patch virtual patch @ depends on patch @ pte_t *v; @@ - *v + ptep_get(v) ---- Then reviewed and hand-edited to avoid multiple unnecessary calls to ptep_get(), instead opting to store the result of a single call in a variable, where it is correct to do so. This aims to negate any cost of READ_ONCE() and will benefit arch-overrides that may be more complex. Included is a fix for an issue in an earlier version of this patch that was pointed out by kernel test robot. The issue arose because config MMU=n elides definition of the ptep helper functions, including ptep_get(). HUGETLB_PAGE=n configs still define a simple huge_ptep_clear_flush() for linking purposes, which dereferences the ptep. So when both configs are disabled, this caused a build error because ptep_get() is not defined. Fix by continuing to do a direct dereference when MMU=n. This is safe because for this config the arch code cannot be trying to virtualize the ptes because none of the ptep helpers are defined. Link: https://lkml.kernel.org/r/20230612151545.3317766-4-ryan.roberts@arm.com Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202305120142.yXsNEo6H-lkp@intel.com/ Signed-off-by: Ryan Roberts Cc: Adrian Hunter Cc: Alexander Potapenko Cc: Alexander Shishkin Cc: Alex Williamson Cc: Al Viro Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Christian Brauner Cc: Christoph Hellwig Cc: Daniel Vetter Cc: Dave Airlie Cc: Dimitri Sivanich Cc: Dmitry Vyukov Cc: Ian Rogers Cc: Jason Gunthorpe Cc: Jérôme Glisse Cc: Jiri Olsa Cc: Johannes Weiner Cc: Kirill A. Shutemov Cc: Lorenzo Stoakes Cc: Mark Rutland Cc: Matthew Wilcox Cc: Miaohe Lin Cc: Michal Hocko Cc: Mike Kravetz Cc: Mike Rapoport (IBM) Cc: Muchun Song Cc: Namhyung Kim Cc: Naoya Horiguchi Cc: Oleksandr Tyshchenko Cc: Pavel Tatashin Cc: Roman Gushchin Cc: SeongJae Park Cc: Shakeel Butt Cc: Uladzislau Rezki (Sony) Cc: Vincenzo Frascino Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 4 ++++ include/linux/mm_inline.h | 2 +- include/linux/pgtable.h | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 21f942025fec..beb7c63d2871 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -1185,7 +1185,11 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { +#ifdef CONFIG_MMU + return ptep_get(ptep); +#else return *ptep; +#endif } static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 0e1d239a882c..08c2bcefcb2b 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -555,7 +555,7 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, bool arm_uffd_pte = false; /* The current status of the pte should be "cleared" before calling */ - WARN_ON_ONCE(!pte_none(*pte)); + WARN_ON_ONCE(!pte_none(ptep_get(pte))); /* * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index fc06f6419661..5063b482e34f 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -231,7 +231,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { - pte_t pte = *ptep; + pte_t pte = ptep_get(ptep); int r = 1; if (!pte_young(pte)) r = 0; @@ -318,7 +318,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, pte_t *ptep) { - pte_t pte = *ptep; + pte_t pte = ptep_get(ptep); pte_clear(mm, address, ptep); page_table_check_pte_clear(mm, address, pte); return pte; @@ -519,7 +519,7 @@ extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, struct mm_struct; static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) { - pte_t old_pte = *ptep; + pte_t old_pte = ptep_get(ptep); set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); } #endif -- cgit v1.2.3 From 35499e2b79ffc51ea704c3268a5830164825a43e Mon Sep 17 00:00:00 2001 From: Domenico Cerasuolo Date: Mon, 12 Jun 2023 11:38:13 +0200 Subject: mm: zswap: remove shrink from zpool interface Now that all three zswap backends have removed their shrink code, it is no longer necessary for the zpool interface to include shrink/writeback endpoints. Link: https://lkml.kernel.org/r/20230612093815.133504-6-cerasuolodomenico@gmail.com Signed-off-by: Domenico Cerasuolo Reviewed-by: Yosry Ahmed Acked-by: Nhat Pham Acked-by: Johannes Weiner Reviewed-by: Sergey Senozhatsky Cc: Dan Streetman Cc: Minchan Kim Cc: Seth Jennings Cc: Vitaly Wool Signed-off-by: Andrew Morton --- include/linux/zpool.h | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/zpool.h b/include/linux/zpool.h index e8997010612a..3296438eec06 100644 --- a/include/linux/zpool.h +++ b/include/linux/zpool.h @@ -14,10 +14,6 @@ struct zpool; -struct zpool_ops { - int (*evict)(struct zpool *pool, unsigned long handle); -}; - /* * Control how a handle is mapped. It will be ignored if the * implementation does not support it. Its use is optional. @@ -39,8 +35,7 @@ enum zpool_mapmode { bool zpool_has_pool(char *type); -struct zpool *zpool_create_pool(const char *type, const char *name, - gfp_t gfp, const struct zpool_ops *ops); +struct zpool *zpool_create_pool(const char *type, const char *name, gfp_t gfp); const char *zpool_get_type(struct zpool *pool); @@ -53,9 +48,6 @@ int zpool_malloc(struct zpool *pool, size_t size, gfp_t gfp, void zpool_free(struct zpool *pool, unsigned long handle); -int zpool_shrink(struct zpool *pool, unsigned int pages, - unsigned int *reclaimed); - void *zpool_map_handle(struct zpool *pool, unsigned long handle, enum zpool_mapmode mm); @@ -72,7 +64,6 @@ u64 zpool_get_total_size(struct zpool *pool); * @destroy: destroy a pool. * @malloc: allocate mem from a pool. * @free: free mem from a pool. - * @shrink: shrink the pool. * @sleep_mapped: whether zpool driver can sleep during map. * @map: map a handle. * @unmap: unmap a handle. @@ -87,10 +78,7 @@ struct zpool_driver { atomic_t refcount; struct list_head list; - void *(*create)(const char *name, - gfp_t gfp, - const struct zpool_ops *ops, - struct zpool *zpool); + void *(*create)(const char *name, gfp_t gfp); void (*destroy)(void *pool); bool malloc_support_movable; @@ -98,9 +86,6 @@ struct zpool_driver { unsigned long *handle); void (*free)(void *pool, unsigned long handle); - int (*shrink)(void *pool, unsigned int pages, - unsigned int *reclaimed); - bool sleep_mapped; void *(*map)(void *pool, unsigned long handle, enum zpool_mapmode mm); @@ -113,7 +98,6 @@ void zpool_register_driver(struct zpool_driver *driver); int zpool_unregister_driver(struct zpool_driver *driver); -bool zpool_evictable(struct zpool *pool); bool zpool_can_sleep_mapped(struct zpool *pool); #endif -- cgit v1.2.3 From 1e3be4856f49d55c60b6cd500297b06acfe216a9 Mon Sep 17 00:00:00 2001 From: Tarun Sahu Date: Mon, 12 Jun 2023 15:05:14 +0530 Subject: mm/folio: replace set_compound_order with folio_set_order The patch ("mm/folio: Avoid special handling for order value 0 in folio_set_order") [1] removed the need for special handling of order = 0 in folio_set_order. Now, folio_set_order and set_compound_order becomes similar function. This patch removes the set_compound_order and uses folio_set_order instead. [1] https://lore.kernel.org/all/20230609183032.13E08C433D2@smtp.kernel.org/ Link: https://lkml.kernel.org/r/20230612093514.689846-1-tsahu@linux.ibm.com Signed-off-by: Tarun Sahu Reviewed-by Sidhartha Kumar Reviewed-by: Muchun Song Cc: Aneesh Kumar K.V Cc: Gerald Schaefer Cc: Matthew Wilcox Cc: Mike Kravetz Signed-off-by: Andrew Morton --- include/linux/mm.h | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 8f40bf17d597..ab04756b2240 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1232,16 +1232,6 @@ static inline void folio_set_compound_dtor(struct folio *folio, void destroy_large_folio(struct folio *folio); -static inline void set_compound_order(struct page *page, unsigned int order) -{ - struct folio *folio = (struct folio *)page; - - folio->_folio_order = order; -#ifdef CONFIG_64BIT - folio->_folio_nr_pages = 1U << order; -#endif -} - /* Returns the number of bytes in this potentially compound page. */ static inline unsigned long page_size(struct page *page) { -- cgit v1.2.3 From 65ac132027a884c411b8f9f96d240ba2dde34dec Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Wed, 31 May 2023 21:54:02 -0400 Subject: userfaultfd: fix regression in userfaultfd_unmap_prep() Android reported a performance regression in the userfaultfd unmap path. A closer inspection on the userfaultfd_unmap_prep() change showed that a second tree walk would be necessary in the reworked code. Fix the regression by passing each VMA that will be unmapped through to the userfaultfd_unmap_prep() function as they are added to the unmap list, instead of re-walking the tree for the VMA. Link: https://lkml.kernel.org/r/20230601015402.2819343-1-Liam.Howlett@oracle.com Fixes: 69dbe6daf104 ("userfaultfd: use maple tree iterator to iterate VMAs") Signed-off-by: Liam R. Howlett Reported-by: Suren Baghdasaryan Suggested-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/userfaultfd_k.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index d78b01524349..ac7b0c96d351 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -188,8 +188,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma, unsigned long start, unsigned long end); -extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, - unsigned long end, struct list_head *uf); +extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, + unsigned long start, unsigned long end, struct list_head *uf); extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); @@ -271,7 +271,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma, return true; } -static inline int userfaultfd_unmap_prep(struct mm_struct *mm, +static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct list_head *uf) { -- cgit v1.2.3 From e4d86756159b5794edad5b0d0d19c6f3d9888240 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sat, 10 Jun 2023 18:19:56 +0800 Subject: mm: remove unused vma_init_lock() commit c7f8f31c00d1 ("mm: separate vma->lock from vm_area_struct") left this behind. Link: https://lkml.kernel.org/r/20230610101956.20592-1-yuehaibing@huawei.com Signed-off-by: YueHaibing Reviewed-by: David Hildenbrand Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index ab04756b2240..f20ac57b634d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -725,7 +725,6 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, #else /* CONFIG_PER_VMA_LOCK */ -static inline void vma_init_lock(struct vm_area_struct *vma) {} static inline bool vma_start_read(struct vm_area_struct *vma) { return false; } static inline void vma_end_read(struct vm_area_struct *vma) {} -- cgit v1.2.3 From 833dfc0090b3f8017ddac82d818b2d8e5ceb61db Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Sat, 10 Jun 2023 11:46:15 +0800 Subject: mm: compaction: mark kcompactd_run() and kcompactd_stop() __meminit Add __meminit to kcompactd_run() and kcompactd_stop() to ensure they're default to __init when memory hotplug is not enabled. Link: https://lkml.kernel.org/r/20230610034615.997813-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Reviewed-by: Baolin Wang Signed-off-by: Andrew Morton --- include/linux/compaction.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 57b16e69c19a..e94776496049 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -98,8 +98,8 @@ extern void compaction_defer_reset(struct zone *zone, int order, bool compaction_zonelist_suitable(struct alloc_context *ac, int order, int alloc_flags); -extern void kcompactd_run(int nid); -extern void kcompactd_stop(int nid); +extern void __meminit kcompactd_run(int nid); +extern void __meminit kcompactd_stop(int nid); extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx); #else -- cgit v1.2.3 From 53418a18fcbbb086dbfacbdd9b853c1071d3ec16 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 12 Jun 2023 22:01:31 +0100 Subject: buffer: convert __block_write_full_page() to __block_write_full_folio() Remove nine hidden calls to compound_head() by using a folio instead of a page. Link: https://lkml.kernel.org/r/20230612210141.730128-5-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Tested-by: Bob Peterson Reviewed-by: Bob Peterson Cc: Andreas Gruenbacher Cc: Hannes Reinecke Cc: Luis Chamberlain Signed-off-by: Andrew Morton --- include/linux/buffer_head.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 1520793c72da..a366e01f8bd4 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -263,7 +263,7 @@ extern int buffer_heads_over_limit; void block_invalidate_folio(struct folio *folio, size_t offset, size_t length); int block_write_full_page(struct page *page, get_block_t *get_block, struct writeback_control *wbc); -int __block_write_full_page(struct inode *inode, struct page *page, +int __block_write_full_folio(struct inode *inode, struct folio *folio, get_block_t *get_block, struct writeback_control *wbc, bh_end_io_t *handler); int block_read_full_folio(struct folio *, get_block_t *); -- cgit v1.2.3 From 4a9622f2fdaee84c373f3f285d898a3ea60ee9f2 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 12 Jun 2023 22:01:36 +0100 Subject: buffer: convert page_zero_new_buffers() to folio_zero_new_buffers() Most of the callers already have a folio; convert reiserfs_write_end() to have a folio. Removes a couple of hidden calls to compound_head(). Link: https://lkml.kernel.org/r/20230612210141.730128-10-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Cc: Andreas Gruenbacher Cc: Bob Peterson Cc: Hannes Reinecke Cc: Luis Chamberlain Signed-off-by: Andrew Morton --- include/linux/buffer_head.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index a366e01f8bd4..c794ea7096ba 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -278,7 +278,7 @@ int block_write_end(struct file *, struct address_space *, int generic_write_end(struct file *, struct address_space *, loff_t, unsigned, unsigned, struct page *, void *); -void page_zero_new_buffers(struct page *page, unsigned from, unsigned to); +void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to); void clean_page_buffers(struct page *page); int cont_write_begin(struct file *, struct address_space *, loff_t, unsigned, struct page **, void **, -- cgit v1.2.3 From 6c77b607ee26472fb945aa41734281c39d06d68f Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Wed, 14 Jun 2023 22:36:12 +0800 Subject: mm: kill lock|unlock_page_memcg() Since commit c7c3dec1c9db ("mm: rmap: remove lock_page_memcg()"), no more user, kill lock_page_memcg() and unlock_page_memcg(). Link: https://lkml.kernel.org/r/20230614143612.62575-1-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: Johannes Weiner Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 00a88cf947e1..c3d3a0c09315 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -419,7 +419,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) * * - the folio lock * - LRU isolation - * - lock_page_memcg() + * - folio_memcg_lock() * - exclusive reference * - mem_cgroup_trylock_pages() * @@ -949,8 +949,6 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); void folio_memcg_lock(struct folio *folio); void folio_memcg_unlock(struct folio *folio); -void lock_page_memcg(struct page *page); -void unlock_page_memcg(struct page *page); void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); @@ -1438,14 +1436,6 @@ mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg) { } -static inline void lock_page_memcg(struct page *page) -{ -} - -static inline void unlock_page_memcg(struct page *page) -{ -} - static inline void folio_memcg_lock(struct folio *folio) { } -- cgit v1.2.3 From 708ff4914dfb410761227a219c17c3e9dbd68c05 Mon Sep 17 00:00:00 2001 From: "Vishal Moola (Oracle)" Date: Tue, 13 Jun 2023 19:13:08 -0700 Subject: mmzone: introduce folio_is_zone_movable() Patch series "Replace is_longterm_pinnable_page()", v2. This patchset introduces some more helper functions for the folio conversions, and converts all callers of is_longterm_pinnable_page() to use folios. This patch (of 5): Introduce folio_is_zone_movable() to act as a folio equivalent for is_zone_movable_page(). This is to assist in later folio conversions. Link: https://lkml.kernel.org/r/20230614021312.34085-1-vishal.moola@gmail.com Link: https://lkml.kernel.org/r/20230614021312.34085-2-vishal.moola@gmail.com Signed-off-by: Vishal Moola (Oracle) Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 5a7ada0413da..f10902491ead 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1116,6 +1116,11 @@ static inline bool is_zone_movable_page(const struct page *page) { return page_zonenum(page) == ZONE_MOVABLE; } + +static inline bool folio_is_zone_movable(const struct folio *folio) +{ + return folio_zonenum(folio) == ZONE_MOVABLE; +} #endif /* -- cgit v1.2.3 From 28fb54f6a2fd6cc471165cce1650a57dfbf49746 Mon Sep 17 00:00:00 2001 From: "Vishal Moola (Oracle)" Date: Tue, 13 Jun 2023 19:13:09 -0700 Subject: mmzone: introduce folio_migratetype() Introduce folio_migratetype() as a folio equivalent for get_pageblock_migratetype(). This function intends to return the migratetype the folio is located in, hence the name choice. Link: https://lkml.kernel.org/r/20230614021312.34085-3-vishal.moola@gmail.com Signed-off-by: Vishal Moola (Oracle) Reviewed-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index f10902491ead..3e822335f214 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -105,6 +105,9 @@ extern int page_group_by_mobility_disabled; #define get_pageblock_migratetype(page) \ get_pfnblock_flags_mask(page, page_to_pfn(page), MIGRATETYPE_MASK) +#define folio_migratetype(folio) \ + get_pfnblock_flags_mask(&folio->page, folio_pfn(folio), \ + MIGRATETYPE_MASK) struct free_area { struct list_head free_list[MIGRATE_TYPES]; unsigned long nr_free; -- cgit v1.2.3 From 5d949953f841fd661a2a49df188426d5930ed723 Mon Sep 17 00:00:00 2001 From: "Vishal Moola (Oracle)" Date: Tue, 13 Jun 2023 19:13:12 -0700 Subject: mm: remove is_longterm_pinnable_page() and reimplement folio_is_longterm_pinnable() folio_is_longterm_pinnable() already exists as a wrapper function. Now that the whole implementation of is_longterm_pinnable_page() can be implemented using folios, folio_is_longterm_pinnable() can be made its own standalone function - and we can remove is_longterm_pinnable_page(). Link: https://lkml.kernel.org/r/20230614021312.34085-6-vishal.moola@gmail.com Signed-off-by: Vishal Moola (Oracle) Reviewed-by: Matthew Wilcox (Oracle) Reviewed-by: Lorenzo Stoakes Signed-off-by: Andrew Morton --- include/linux/mm.h | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index f20ac57b634d..a8baa34d0747 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1902,39 +1902,35 @@ static inline bool page_needs_cow_for_dma(struct vm_area_struct *vma, return page_maybe_dma_pinned(page); } -/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin pages */ +/* MIGRATE_CMA and ZONE_MOVABLE do not allow pin folios */ #ifdef CONFIG_MIGRATION -static inline bool is_longterm_pinnable_page(struct page *page) +static inline bool folio_is_longterm_pinnable(struct folio *folio) { #ifdef CONFIG_CMA - int mt = get_pageblock_migratetype(page); + int mt = folio_migratetype(folio); if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) return false; #endif /* The zero page may always be pinned */ - if (is_zero_pfn(page_to_pfn(page))) + if (is_zero_pfn(folio_pfn(folio))) return true; /* Coherent device memory must always allow eviction. */ - if (is_device_coherent_page(page)) + if (folio_is_device_coherent(folio)) return false; - /* Otherwise, non-movable zone pages can be pinned. */ - return !is_zone_movable_page(page); + /* Otherwise, non-movable zone folios can be pinned. */ + return !folio_is_zone_movable(folio); + } #else -static inline bool is_longterm_pinnable_page(struct page *page) +static inline bool folio_is_longterm_pinnable(struct folio *folio) { return true; } #endif -static inline bool folio_is_longterm_pinnable(struct folio *folio) -{ - return is_longterm_pinnable_page(&folio->page); -} - static inline void set_page_zone(struct page *page, enum zone_type zone) { page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); -- cgit v1.2.3 From 025b7799b35d32e46988ba0614ea2f91b85d6375 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Fri, 16 Jun 2023 14:30:30 +0800 Subject: mm/memcg: remove return value of mem_cgroup_scan_tasks() No user checks the return value of mem_cgroup_scan_tasks(). Make the return value void. Link: https://lkml.kernel.org/r/20230616063030.977586-1-zhangpeng362@huawei.com Signed-off-by: ZhangPeng Cc: Johannes Weiner Cc: Kefeng Wang Cc: Michal Hocko Cc: Muchun Song Cc: Nanyong Sun Cc: Roman Gushchin Cc: Shakeel Butt Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index c3d3a0c09315..5818af8eca5a 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -820,8 +820,8 @@ struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, struct mem_cgroup *, struct mem_cgroup_reclaim_cookie *); void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); -int mem_cgroup_scan_tasks(struct mem_cgroup *, - int (*)(struct task_struct *, void *), void *); +void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, + int (*)(struct task_struct *, void *), void *arg); static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) { @@ -1364,10 +1364,9 @@ static inline void mem_cgroup_iter_break(struct mem_cgroup *root, { } -static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg, +static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*fn)(struct task_struct *, void *), void *arg) { - return 0; } static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg) -- cgit v1.2.3 From c1753fd02a0058ea43cbb31ab26d25be2f6cfe08 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Mon, 15 May 2023 10:35:36 -0400 Subject: mm: move mm_count into its own cache line The mm_struct mm_count field is frequently updated by mmgrab/mmdrop performed by context switch. This causes false-sharing for surrounding mm_struct fields which are read-mostly. This has been observed on a 2sockets/112core/224cpu Intel Sapphire Rapids server running hackbench, and by the kernel test robot will-it-scale testcase. Move the mm_count field into its own cache line to prevent false-sharing with other mm_struct fields. Move mm_count to the first field of mm_struct to minimize the amount of padding required: rather than adding padding before and after the mm_count field, padding is only added after mm_count. Note that I noticed this odd comment in mm_struct: commit 2e3025434a6b ("mm: relocate 'write_protect_seq' in struct mm_struct") /* * With some kernel config, the current mmap_lock's offset * inside 'mm_struct' is at 0x120, which is very optimal, as * its two hot fields 'count' and 'owner' sit in 2 different * cachelines, and when mmap_lock is highly contended, both * of the 2 fields will be accessed frequently, current layout * will help to reduce cache bouncing. * * So please be careful with adding new fields before * mmap_lock, which can easily push the 2 fields into one * cacheline. */ struct rw_semaphore mmap_lock; This comment is rather odd for a few reasons: - It requires addition/removal of mm_struct fields to carefully consider field alignment of _other_ fields, - It expresses the wish to keep an "optimal" alignment for a specific kernel config. I suspect that the author of this comment may want to revisit this topic and perhaps introduce a split-struct approach for struct rw_semaphore, if the need is to place various fields of this structure in different cache lines. Link: https://lkml.kernel.org/r/20230515143536.114960-1-mathieu.desnoyers@efficios.com Fixes: 223baf9d17f2 ("sched: Fix performance regression introduced by mm_cid") Fixes: af7f588d8f73 ("sched: Introduce per-memory-map concurrency ID") Link: https://lore.kernel.org/lkml/7a0c1db1-103d-d518-ed96-1584a28fbf32@efficios.com Reported-by: kernel test robot Link: https://lore.kernel.org/oe-lkp/202305151017.27581d75-yujie.liu@intel.com Signed-off-by: Mathieu Desnoyers Reviewed-by: Aaron Lu Reviewed-by: John Hubbard Cc: Peter Zijlstra Cc: Olivier Dion Cc: Cc: Feng Tang Cc: Jason Gunthorpe Cc: Peter Xu Signed-off-by: Andrew Morton --- include/linux/mm_types.h | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 306a3d1a0fa6..de10fc797c8e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -583,6 +583,21 @@ struct mm_cid { struct kioctx_table; struct mm_struct { struct { + /* + * Fields which are often written to are placed in a separate + * cache line. + */ + struct { + /** + * @mm_count: The number of references to &struct + * mm_struct (@mm_users count as 1). + * + * Use mmgrab()/mmdrop() to modify. When this drops to + * 0, the &struct mm_struct is freed. + */ + atomic_t mm_count; + } ____cacheline_aligned_in_smp; + struct maple_tree mm_mt; #ifdef CONFIG_MMU unsigned long (*get_unmapped_area) (struct file *filp, @@ -620,14 +635,6 @@ struct mm_struct { */ atomic_t mm_users; - /** - * @mm_count: The number of references to &struct mm_struct - * (@mm_users count as 1). - * - * Use mmgrab()/mmdrop() to modify. When this drops to 0, the - * &struct mm_struct is freed. - */ - atomic_t mm_count; #ifdef CONFIG_SCHED_MM_CID /** * @pcpu_cid: Per-cpu current cid. -- cgit v1.2.3 From cf01724e2d73a90524450e3dd8798cfb9d7aca05 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Sat, 17 Jun 2023 11:46:22 +0800 Subject: mm: page_alloc: make compound_page_dtors static It's only used inside page_alloc.c now. So make it static and remove the declaration in mm.h. Link: https://lkml.kernel.org/r/20230617034622.1235913-1-linmiaohe@huawei.com Signed-off-by: Miaohe Lin Signed-off-by: Andrew Morton --- include/linux/mm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index a8baa34d0747..cf43deb25553 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1220,7 +1220,6 @@ enum compound_dtor_id { #endif NR_COMPOUND_DTORS, }; -extern compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS]; static inline void folio_set_compound_dtor(struct folio *folio, enum compound_dtor_id compound_dtor) -- cgit v1.2.3 From 9ec272c586b07d1abf73438524bd12b1df9c5f9b Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 26 May 2023 18:41:31 -0700 Subject: watchdog/hardlockup: keep kernel.nmi_watchdog sysctl as 0444 if probe fails Patch series "watchdog: Cleanup / fixes after buddy series v5 reviews". This patch series attempts to finish resolving the feedback received from Petr Mladek on the v5 series I posted. Probably the only thing that wasn't fully as clean as Petr requested was the Kconfig stuff. I couldn't find a better way to express it without a more major overhaul. In the very least, I renamed "NON_ARCH" to "PERF_OR_BUDDY" in the hopes that will make it marginally better. Nothing in this series is terribly critical and even the bugfixes are small. However, it does cleanup a few things that were pointed out in review. This patch (of 10): The permissions for the kernel.nmi_watchdog sysctl have always been set at compile time despite the fact that a watchdog can fail to probe. Let's fix this and set the permissions based on whether the hardlockup detector actually probed. Link: https://lkml.kernel.org/r/20230527014153.2793931-1-dianders@chromium.org Link: https://lkml.kernel.org/r/20230526184139.1.I0d75971cc52a7283f495aac0bd5c3041aadc734e@changeid Fixes: a994a3147e4c ("watchdog/hardlockup/perf: Implement init time detection of perf") Signed-off-by: Douglas Anderson Reported-by: Petr Mladek Closes: https://lore.kernel.org/r/ZHCn4hNxFpY5-9Ki@alley Reviewed-by: Petr Mladek Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 83577ae736cc..99b7d748ca21 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -95,12 +95,6 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); static inline void arch_touch_nmi_watchdog(void) { } #endif -#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR) -# define NMI_WATCHDOG_SYSCTL_PERM 0644 -#else -# define NMI_WATCHDOG_SYSCTL_PERM 0444 -#endif - #if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) extern void hardlockup_detector_perf_stop(void); extern void hardlockup_detector_perf_restart(void); -- cgit v1.2.3 From 05e7b558766114aa9c3d5d3af188a5c574809661 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 26 May 2023 18:41:35 -0700 Subject: watchdog/hardlockup: remove softlockup comment in touch_nmi_watchdog() In the patch ("watchdog/hardlockup: add comments to touch_nmi_watchdog()") we adjusted some comments for touch_nmi_watchdog(). The comment about the softlockup had a typo and were also felt to be too obvious. Remove it. Link: https://lkml.kernel.org/r/20230526184139.5.Ia593afc9eb12082d55ea6681dc2c5a89677f20a8@changeid Signed-off-by: Douglas Anderson Suggested-by: Petr Mladek Reviewed-by: Petr Mladek Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 4 ---- 1 file changed, 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 99b7d748ca21..3625d64da6db 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -140,10 +140,6 @@ static inline void touch_nmi_watchdog(void) */ arch_touch_nmi_watchdog(); - /* - * Touching the hardlock detector implicitly resets the - * softlockup detector too - */ touch_softlockup_watchdog(); } -- cgit v1.2.3 From d3b62ace0f097f1d863fb6c41df3c61503e4ec9e Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 26 May 2023 18:41:36 -0700 Subject: watchdog/buddy: cleanup how watchdog_buddy_check_hardlockup() is called In the patch ("watchdog/hardlockup: detect hard lockups using secondary (buddy) CPUs"), we added a call from the common watchdog.c file into the buddy. That call could be done more cleanly. Specifically: 1. If we move the call into watchdog_hardlockup_kick() then it keeps watchdog_timer_fn() simpler. 2. We don't need to pass an "unsigned long" to the buddy for the timer count. In the patch ("watchdog/hardlockup: add a "cpu" param to watchdog_hardlockup_check()") the count was changed to "atomic_t" which is backed by an int, so we should match types. Link: https://lkml.kernel.org/r/20230526184139.6.I006c7d958a1ea5c4e1e4dc44a25596d9bb5fd3ba@changeid Signed-off-by: Douglas Anderson Suggested-by: Petr Mladek Reviewed-by: Petr Mladek Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 3625d64da6db..d35393405b24 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -114,9 +114,9 @@ void watchdog_hardlockup_disable(unsigned int cpu); void lockup_detector_reconfigure(void); #ifdef CONFIG_HARDLOCKUP_DETECTOR_BUDDY -void watchdog_buddy_check_hardlockup(unsigned long hrtimer_interrupts); +void watchdog_buddy_check_hardlockup(int hrtimer_interrupts); #else -static inline void watchdog_buddy_check_hardlockup(unsigned long hrtimer_interrupts) {} +static inline void watchdog_buddy_check_hardlockup(int hrtimer_interrupts) {} #endif /** -- cgit v1.2.3 From 0c68bda69665307bf835b0c433363e5073608c95 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Fri, 16 Jun 2023 17:06:15 +0200 Subject: watchdog/hardlockup: declare arch_touch_nmi_watchdog() only in linux/nmi.h arch_touch_nmi_watchdog() needs a different implementation for various hardlockup detector implementations. And it does nothing when any hardlockup detector is not built at all. arch_touch_nmi_watchdog() is declared via linux/nmi.h. And it must be defined as an empty function when there is no hardlockup detector. It is done directly in this header file for the perf and buddy detectors. And it is done in the included asm/linux.h for arch specific detectors. The reason probably is that the arch specific variants build the code using another conditions. For example, powerpc64/sparc64 builds the code when CONFIG_PPC_WATCHDOG is enabled. Another reason might be that these architectures define more functions in asm/nmi.h anyway. However the generic code actually knows when the function will be implemented. It happens when some full featured or the sparc64-specific hardlockup detector is built. In particular, CONFIG_HARDLOCKUP_DETECTOR can be enabled only when a generic or arch-specific full featured hardlockup detector is available. The only exception is sparc64 which can be built even when the global HARDLOCKUP_DETECTOR switch is disabled. The information about sparc64 is a bit complicated. The hardlockup detector is built there when CONFIG_HAVE_NMI_WATCHDOG is set and CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH is not set. People might wonder whether this change really makes things easier. The motivation is: + The current logic in linux/nmi.h is far from obvious. For example, arch_touch_nmi_watchdog() is defined as {} when neither CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER nor CONFIG_HAVE_NMI_WATCHDOG is defined. + The change synchronizes the checks in lib/Kconfig.debug and in the generic code. + It is a step that will help cleaning HAVE_NMI_WATCHDOG related checks. The change should not change the existing behavior. Link: https://lkml.kernel.org/r/20230616150618.6073-4-pmladek@suse.com Signed-off-by: Petr Mladek Reviewed-by: Douglas Anderson Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index d35393405b24..07bf2b813463 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -7,6 +7,8 @@ #include #include + +/* Arch specific watchdogs might need to share extra watchdog-related APIs. */ #if defined(CONFIG_HAVE_NMI_WATCHDOG) #include #endif @@ -87,12 +89,17 @@ extern unsigned int hardlockup_panic; static inline void hardlockup_detector_disable(void) {} #endif -#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER) +/* Sparc64 has special implemetantion that is always enabled. */ +#if defined(CONFIG_HARDLOCKUP_DETECTOR) || \ + (defined(CONFIG_HAVE_NMI_WATCHDOG) && !defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH)) void arch_touch_nmi_watchdog(void); +#else +static inline void arch_touch_nmi_watchdog(void) { } +#endif + +#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER) void watchdog_hardlockup_touch_cpu(unsigned int cpu); void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs); -#elif !defined(CONFIG_HAVE_NMI_WATCHDOG) -static inline void arch_touch_nmi_watchdog(void) { } #endif #if defined(CONFIG_HARDLOCKUP_DETECTOR_PERF) -- cgit v1.2.3 From a5fcc2367e223c45c78a882438c2b8e13fe0f580 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Fri, 16 Jun 2023 17:06:16 +0200 Subject: watchdog/hardlockup: make HAVE_NMI_WATCHDOG sparc64-specific There are several hardlockup detector implementations and several Kconfig values which allow selection and build of the preferred one. CONFIG_HARDLOCKUP_DETECTOR was introduced by the commit 23637d477c1f53acb ("lockup_detector: Introduce CONFIG_HARDLOCKUP_DETECTOR") in v2.6.36. It was a preparation step for introducing the new generic perf hardlockup detector. The existing arch-specific variants did not support the to-be-created generic build configurations, sysctl interface, etc. This distinction was made explicit by the commit 4a7863cc2eb5f98 ("x86, nmi_watchdog: Remove ARCH_HAS_NMI_WATCHDOG and rely on CONFIG_HARDLOCKUP_DETECTOR") in v2.6.38. CONFIG_HAVE_NMI_WATCHDOG was introduced by the commit d314d74c695f967e105 ("nmi watchdog: do not use cpp symbol in Kconfig") in v3.4-rc1. It replaced the above mentioned ARCH_HAS_NMI_WATCHDOG. At that time, it was still used by three architectures, namely blackfin, mn10300, and sparc. The support for blackfin and mn10300 architectures has been completely dropped some time ago. And sparc is the only architecture with the historic NMI watchdog at the moment. And the old sparc implementation is really special. It is always built on sparc64. It used to be always enabled until the commit 7a5c8b57cec93196b ("sparc: implement watchdog_nmi_enable and watchdog_nmi_disable") added in v4.10-rc1. There are only few locations where the sparc64 NMI watchdog interacts with the generic hardlockup detectors code: + implements arch_touch_nmi_watchdog() which is called from the generic touch_nmi_watchdog() + implements watchdog_hardlockup_enable()/disable() to support /proc/sys/kernel/nmi_watchdog + is always preferred over other generic watchdogs, see CONFIG_HARDLOCKUP_DETECTOR + includes asm/nmi.h into linux/nmi.h because some sparc-specific functions are needed in sparc-specific code which includes only linux/nmi.h. The situation became more complicated after the commit 05a4a95279311c3 ("kernel/watchdog: split up config options") and commit 2104180a53698df5 ("powerpc/64s: implement arch-specific hardlockup watchdog") in v4.13-rc1. They introduced HAVE_HARDLOCKUP_DETECTOR_ARCH. It was used for powerpc specific hardlockup detector. It was compatible with the perf one regarding the general boot, sysctl, and programming interfaces. HAVE_HARDLOCKUP_DETECTOR_ARCH was defined as a superset of HAVE_NMI_WATCHDOG. It made some sense because all arch-specific detectors had some common requirements, namely: + implemented arch_touch_nmi_watchdog() + included asm/nmi.h into linux/nmi.h + defined the default value for /proc/sys/kernel/nmi_watchdog But it actually has made things pretty complicated when the generic buddy hardlockup detector was added. Before the generic perf detector was newer supported together with an arch-specific one. But the buddy detector could work on any SMP system. It means that an architecture could support both the arch-specific and buddy detector. As a result, there are few tricky dependencies. For example, CONFIG_HARDLOCKUP_DETECTOR depends on: ((HAVE_HARDLOCKUP_DETECTOR_PERF || HAVE_HARDLOCKUP_DETECTOR_BUDDY) && !HAVE_NMI_WATCHDOG) || HAVE_HARDLOCKUP_DETECTOR_ARCH The problem is that the very special sparc implementation is defined as: HAVE_NMI_WATCHDOG && !HAVE_HARDLOCKUP_DETECTOR_ARCH Another problem is that the meaning of HAVE_NMI_WATCHDOG is far from clear without reading understanding the history. Make the logic less tricky and more self-explanatory by making HAVE_NMI_WATCHDOG specific for the sparc64 implementation. And rename it to HAVE_HARDLOCKUP_DETECTOR_SPARC64. Note that HARDLOCKUP_DETECTOR_PREFER_BUDDY, HARDLOCKUP_DETECTOR_PERF, and HARDLOCKUP_DETECTOR_BUDDY may conflict only with HAVE_HARDLOCKUP_DETECTOR_ARCH. They depend on HARDLOCKUP_DETECTOR and it is not longer enabled when HAVE_NMI_WATCHDOG is set. Link: https://lkml.kernel.org/r/20230616150618.6073-5-pmladek@suse.com Signed-off-by: Petr Mladek Reviewed-by: Douglas Anderson Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 07bf2b813463..63acc6586774 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -9,7 +9,7 @@ #include /* Arch specific watchdogs might need to share extra watchdog-related APIs. */ -#if defined(CONFIG_HAVE_NMI_WATCHDOG) +#if defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH) || defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_SPARC64) #include #endif @@ -90,8 +90,7 @@ static inline void hardlockup_detector_disable(void) {} #endif /* Sparc64 has special implemetantion that is always enabled. */ -#if defined(CONFIG_HARDLOCKUP_DETECTOR) || \ - (defined(CONFIG_HAVE_NMI_WATCHDOG) && !defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH)) +#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_SPARC64) void arch_touch_nmi_watchdog(void); #else static inline void arch_touch_nmi_watchdog(void) { } -- cgit v1.2.3 From 47f4cb433923a08d81f1e5c065cb680215109db9 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Fri, 16 Jun 2023 17:06:17 +0200 Subject: watchdog/sparc64: define HARDLOCKUP_DETECTOR_SPARC64 The HAVE_ prefix means that the code could be enabled. Add another variable for HAVE_HARDLOCKUP_DETECTOR_SPARC64 without this prefix. It will be set when it should be built. It will make it compatible with the other hardlockup detectors. Before, it is far from obvious that the SPARC64 variant is actually used: $> make ARCH=sparc64 defconfig $> grep HARDLOCKUP_DETECTOR .config CONFIG_HAVE_HARDLOCKUP_DETECTOR_BUDDY=y CONFIG_HAVE_HARDLOCKUP_DETECTOR_SPARC64=y After, it is more clear: $> make ARCH=sparc64 defconfig $> grep HARDLOCKUP_DETECTOR .config CONFIG_HAVE_HARDLOCKUP_DETECTOR_BUDDY=y CONFIG_HAVE_HARDLOCKUP_DETECTOR_SPARC64=y CONFIG_HARDLOCKUP_DETECTOR_SPARC64=y Link: https://lkml.kernel.org/r/20230616150618.6073-6-pmladek@suse.com Signed-off-by: Petr Mladek Reviewed-by: Douglas Anderson Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 63acc6586774..e91a1f803d55 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -9,7 +9,7 @@ #include /* Arch specific watchdogs might need to share extra watchdog-related APIs. */ -#if defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH) || defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_SPARC64) +#if defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64) #include #endif @@ -90,7 +90,7 @@ static inline void hardlockup_detector_disable(void) {} #endif /* Sparc64 has special implemetantion that is always enabled. */ -#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_SPARC64) +#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64) void arch_touch_nmi_watchdog(void); #else static inline void arch_touch_nmi_watchdog(void) { } -- cgit v1.2.3 From 7ca8fe94aa92d9adcd7dcdf64371fc78eb2da3f9 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Fri, 16 Jun 2023 17:06:18 +0200 Subject: watchdog/hardlockup: define HARDLOCKUP_DETECTOR_ARCH The HAVE_ prefix means that the code could be enabled. Add another variable for HAVE_HARDLOCKUP_DETECTOR_ARCH without this prefix. It will be set when it should be built. It will make it compatible with the other hardlockup detectors. The change allows to clean up dependencies of PPC_WATCHDOG and HAVE_HARDLOCKUP_DETECTOR_PERF definitions for powerpc. As a result HAVE_HARDLOCKUP_DETECTOR_PERF has the same dependencies on arm, x86, powerpc architectures. Link: https://lkml.kernel.org/r/20230616150618.6073-7-pmladek@suse.com Signed-off-by: Petr Mladek Reviewed-by: Douglas Anderson Cc: Christophe Leroy Cc: "David S. Miller" Cc: Michael Ellerman Cc: Nicholas Piggin Signed-off-by: Andrew Morton --- include/linux/nmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/nmi.h b/include/linux/nmi.h index e91a1f803d55..e3e6a64b98e0 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -9,7 +9,7 @@ #include /* Arch specific watchdogs might need to share extra watchdog-related APIs. */ -#if defined(CONFIG_HAVE_HARDLOCKUP_DETECTOR_ARCH) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64) +#if defined(CONFIG_HARDLOCKUP_DETECTOR_ARCH) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64) #include #endif -- cgit v1.2.3 From a05d070a6164bd0578991e42181a52b9c7cf630c Mon Sep 17 00:00:00 2001 From: Rahul Rameshbabu Date: Mon, 12 Jun 2023 14:14:52 -0700 Subject: ptp: Clarify ptp_clock_info .adjphase expects an internal servo to be used .adjphase expects a PHC to use an internal servo algorithm to correct the provided phase offset target in the callback. Implementation of the internal servo algorithm are defined by the individual devices. Cc: Jakub Kicinski Cc: Richard Cochran Signed-off-by: Rahul Rameshbabu Acked-by: Richard Cochran Signed-off-by: David S. Miller --- include/linux/ptp_clock_kernel.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h index fdffa6a98d79..f8e8443a8b35 100644 --- a/include/linux/ptp_clock_kernel.h +++ b/include/linux/ptp_clock_kernel.h @@ -77,8 +77,10 @@ struct ptp_system_timestamp { * nominal frequency in parts per million, but with a * 16 bit binary fractional field. * - * @adjphase: Adjusts the phase offset of the hardware clock. - * parameter delta: Desired change in nanoseconds. + * @adjphase: Indicates that the PHC should use an internal servo + * algorithm to correct the provided phase offset. + * parameter delta: PHC servo phase adjustment target + * in nanoseconds. * * @adjtime: Shifts the time of the hardware clock. * parameter delta: Desired change in nanoseconds. -- cgit v1.2.3 From c3b60ab7a4dff6e6e608e685b70ddc3d6b2aca81 Mon Sep 17 00:00:00 2001 From: Rahul Rameshbabu Date: Mon, 12 Jun 2023 14:14:56 -0700 Subject: ptp: Add .getmaxphase callback to ptp_clock_info Enables advertisement of the maximum offset supported by the phase control functionality of PHCs. The callback is used to return an error if an offset not supported by the PHC is used in ADJ_OFFSET. The ioctls PTP_CLOCK_GETCAPS and PTP_CLOCK_GETCAPS2 now advertise the maximum offset a PHC's phase control functionality is capable of supporting. Introduce new sysfs node, max_phase_adjustment. Cc: Jakub Kicinski Cc: Shuah Khan Cc: Richard Cochran Cc: Maciek Machnikowski Signed-off-by: Rahul Rameshbabu Acked-by: Richard Cochran Signed-off-by: David S. Miller --- include/linux/ptp_clock_kernel.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h index f8e8443a8b35..1ef4e0f9bd2a 100644 --- a/include/linux/ptp_clock_kernel.h +++ b/include/linux/ptp_clock_kernel.h @@ -82,6 +82,10 @@ struct ptp_system_timestamp { * parameter delta: PHC servo phase adjustment target * in nanoseconds. * + * @getmaxphase: Advertises maximum offset that can be provided + * to the hardware clock's phase control functionality + * through adjphase. + * * @adjtime: Shifts the time of the hardware clock. * parameter delta: Desired change in nanoseconds. * @@ -171,6 +175,7 @@ struct ptp_clock_info { struct ptp_pin_desc *pin_config; int (*adjfine)(struct ptp_clock_info *ptp, long scaled_ppm); int (*adjphase)(struct ptp_clock_info *ptp, s32 phase); + s32 (*getmaxphase)(struct ptp_clock_info *ptp); int (*adjtime)(struct ptp_clock_info *ptp, s64 delta); int (*gettime64)(struct ptp_clock_info *ptp, struct timespec64 *ts); int (*gettimex64)(struct ptp_clock_info *ptp, struct timespec64 *ts, -- cgit v1.2.3 From c467c8f081859d4f4ca4eee4fba54bb5d85d6c97 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 20 Jun 2023 12:27:13 +0200 Subject: mmc: Add MMC_QUIRK_BROKEN_SD_CACHE for Kingston Canvas Go Plus from 11/2019 This microSD card never clears Flush Cache bit after cache flush has been started in sd_flush_cache(). This leads e.g. to failure to mount file system. Add a quirk which disables the SD cache for this specific card from specific manufacturing date of 11/2019, since on newer dated cards from 05/2023 the cache flush works correctly. Fixes: 08ebf903af57 ("mmc: core: Fixup support for writeback-cache for eMMC and SD") Signed-off-by: Marek Vasut Link: https://lore.kernel.org/r/20230620102713.7701-1-marex@denx.de Signed-off-by: Ulf Hansson --- include/linux/mmc/card.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index c726ea781255..daa2f40d9ce6 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -294,6 +294,7 @@ struct mmc_card { #define MMC_QUIRK_TRIM_BROKEN (1<<12) /* Skip trim */ #define MMC_QUIRK_BROKEN_HPI (1<<13) /* Disable broken HPI support */ #define MMC_QUIRK_BROKEN_SD_DISCARD (1<<14) /* Disable broken SD discard support */ +#define MMC_QUIRK_BROKEN_SD_CACHE (1<<15) /* Disable broken SD cache support */ bool reenable_cmdq; /* Re-enable Command Queue */ -- cgit v1.2.3 From d7439fb1f4338fffd0bc68bb62d78f7712725f26 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Tue, 20 Jun 2023 13:28:32 +0200 Subject: fs: Provide helpers for manipulating sb->s_readonly_remount Provide helpers to set and clear sb->s_readonly_remount including appropriate memory barriers. Also use this opportunity to document what the barriers pair with and why they are needed. Suggested-by: Dave Chinner Signed-off-by: Jan Kara Reviewed-by: Dave Chinner Message-Id: <20230620112832.5158-1-jack@suse.cz> Signed-off-by: Christian Brauner --- include/linux/fs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index 40bef9bf8749..4caac7fdc5d3 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1248,7 +1248,7 @@ struct super_block { */ atomic_long_t s_fsnotify_connectors; - /* Being remounted read-only */ + /* Read-only state of the superblock is being changed */ int s_readonly_remount; /* per-sb errseq_t for reporting writeback errors via syncfs */ -- cgit v1.2.3 From ebf51575c8418fcbabe489b3b4a4227c34ed256a Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Thu, 15 Jun 2023 13:19:31 +0300 Subject: clk: fix typo in clk_hw_register_fixed_rate_parent_data() macro clk_hw_register_fixed_rate_parent_data() 3rd parameter is parent_data not parent_hw. Inner function (__clk_hw_register_fixed_rate()) is called with parent_data parameter as valid. To have this parameter taken into account update the name of the 3rd parameter of clk_hw_register_fixed_rate_parent_data() macro to parent_data. Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20230615101931.581060-1-claudiu.beznea@microchip.com Signed-off-by: Stephen Boyd --- include/linux/clk-provider.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 28ff6f1a6ada..bbc1bf19250a 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -415,7 +415,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, * @flags: framework-specific flags * @fixed_rate: non-adjustable clock rate */ -#define clk_hw_register_fixed_rate_parent_data(dev, name, parent_hw, flags, \ +#define clk_hw_register_fixed_rate_parent_data(dev, name, parent_data, flags, \ fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ (parent_data), (flags), (fixed_rate), 0, \ -- cgit v1.2.3 From 3a4f0edbb7939a16abb54668642ceed1decdbf4a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 19 Jun 2023 07:27:40 +0000 Subject: ipv6: fix a typo in ip6mr_sk_ioctl() SIOCGETSGCNT_IN6 uses a "struct sioc_sg_req6 buffer". Unfortunately the blamed commit made hard to ensure type safety. syzbot reported: BUG: KASAN: stack-out-of-bounds in ip6mr_ioctl+0xba3/0xcb0 net/ipv6/ip6mr.c:1917 Read of size 16 at addr ffffc900039afb68 by task syz-executor937/5008 CPU: 1 PID: 5008 Comm: syz-executor937 Not tainted 6.4.0-rc6-syzkaller-01304-gc08afcdcf952 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 05/27/2023 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xd9/0x150 lib/dump_stack.c:106 print_address_description.constprop.0+0x2c/0x3c0 mm/kasan/report.c:351 print_report mm/kasan/report.c:462 [inline] kasan_report+0x11c/0x130 mm/kasan/report.c:572 ip6mr_ioctl+0xba3/0xcb0 net/ipv6/ip6mr.c:1917 rawv6_ioctl+0x4e/0x1e0 net/ipv6/raw.c:1143 sock_ioctl_out net/core/sock.c:4186 [inline] sk_ioctl+0x151/0x440 net/core/sock.c:4214 inet6_ioctl+0x1b8/0x290 net/ipv6/af_inet6.c:582 sock_do_ioctl+0xcc/0x230 net/socket.c:1189 sock_ioctl+0x1f8/0x680 net/socket.c:1306 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:870 [inline] __se_sys_ioctl fs/ioctl.c:856 [inline] __x64_sys_ioctl+0x197/0x210 fs/ioctl.c:856 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x39/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7f255849bad9 Code: 28 c3 e8 2a 14 00 00 66 2e 0f 1f 84 00 00 00 00 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 c0 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007ffd06792778 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f255849bad9 RDX: 0000000000000000 RSI: 00000000000089e1 RDI: 0000000000000003 RBP: 00007f255845fc80 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00007f255845fd10 R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 The buggy address belongs to stack of task syz-executor937/5008 and is located at offset 40 in frame: sk_ioctl+0x0/0x440 net/core/sock.c:4172 This frame has 2 objects: [32, 36) 'karg' [48, 88) 'buffer' Fixes: e1d001fa5b47 ("net: ioctl: Use kernel memory on protocol ioctl callbacks") Reported-by: syzbot Signed-off-by: Eric Dumazet Cc: Willem de Bruijn Cc: David Ahern Cc: Kuniyuki Iwashima Reviewed-by: Breno Leitao Link: https://lore.kernel.org/r/20230619072740.464528-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/mroute6.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mroute6.h b/include/linux/mroute6.h index 2f95d5b4e47a..63ef5191cc57 100644 --- a/include/linux/mroute6.h +++ b/include/linux/mroute6.h @@ -109,13 +109,13 @@ static inline int ip6mr_sk_ioctl(struct sock *sk, unsigned int cmd, struct sioc_mif_req6 buffer; return sock_ioctl_inout(sk, cmd, arg, &buffer, - sizeof(buffer)); + sizeof(buffer)); } case SIOCGETSGCNT_IN6: { - struct sioc_mif_req6 buffer; + struct sioc_sg_req6 buffer; return sock_ioctl_inout(sk, cmd, arg, &buffer, - sizeof(buffer)); + sizeof(buffer)); } } -- cgit v1.2.3 From 634236b34d7a8c9e11c12b0746b83b8942fc8f2e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 19 Jun 2023 12:43:35 +0000 Subject: net: remove sk_is_ipmr() and sk_is_icmpv6() helpers Blamed commit added these helpers for sake of detecting RAW sockets specific ioctl. syzbot complained about it [1]. Issue here is that RAW sockets could pretend there was no need to call ipmr_sk_ioctl() Regardless of inet_sk(sk)->inet_num, we must be prepared for ipmr_ioctl() being called later. This must happen from ipmr_sk_ioctl() context only. We could add a safety check in ipmr_ioctl() at the risk of breaking applications. Instead, remove sk_is_ipmr() and sk_is_icmpv6() because their name would be misleading, once we change their implementation. [1] BUG: KASAN: stack-out-of-bounds in ipmr_ioctl+0xb12/0xbd0 net/ipv4/ipmr.c:1654 Read of size 4 at addr ffffc90003aefae4 by task syz-executor105/5004 CPU: 0 PID: 5004 Comm: syz-executor105 Not tainted 6.4.0-rc6-syzkaller-01304-gc08afcdcf952 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 05/27/2023 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0xd9/0x150 lib/dump_stack.c:106 print_address_description.constprop.0+0x2c/0x3c0 mm/kasan/report.c:351 print_report mm/kasan/report.c:462 [inline] kasan_report+0x11c/0x130 mm/kasan/report.c:572 ipmr_ioctl+0xb12/0xbd0 net/ipv4/ipmr.c:1654 raw_ioctl+0x4e/0x1e0 net/ipv4/raw.c:881 sock_ioctl_out net/core/sock.c:4186 [inline] sk_ioctl+0x151/0x440 net/core/sock.c:4214 inet_ioctl+0x18c/0x380 net/ipv4/af_inet.c:1001 sock_do_ioctl+0xcc/0x230 net/socket.c:1189 sock_ioctl+0x1f8/0x680 net/socket.c:1306 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:870 [inline] __se_sys_ioctl fs/ioctl.c:856 [inline] __x64_sys_ioctl+0x197/0x210 fs/ioctl.c:856 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x39/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7f2944bf6ad9 Code: 28 c3 e8 2a 14 00 00 66 2e 0f 1f 84 00 00 00 00 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 c0 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007ffd8897a028 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f2944bf6ad9 RDX: 0000000000000000 RSI: 00000000000089e1 RDI: 0000000000000003 RBP: 00007f2944bbac80 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 00007f2944bbad10 R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 The buggy address belongs to stack of task syz-executor105/5004 and is located at offset 36 in frame: sk_ioctl+0x0/0x440 net/core/sock.c:4172 This frame has 2 objects: [32, 36) 'karg' [48, 88) 'buffer' Fixes: e1d001fa5b47 ("net: ioctl: Use kernel memory on protocol ioctl callbacks") Reported-by: syzbot Signed-off-by: Eric Dumazet Cc: Breno Leitao Cc: Kuniyuki Iwashima Reviewed-by: Jiri Pirko Reviewed-by: David Ahern Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/20230619124336.651528-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- include/linux/icmpv6.h | 6 ------ include/linux/mroute.h | 11 ----------- 2 files changed, 17 deletions(-) (limited to 'include/linux') diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index 1fe33e6741cc..db0f4fcfdaf4 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -111,10 +111,4 @@ static inline bool icmpv6_is_err(int type) return false; } -static inline int sk_is_icmpv6(struct sock *sk) -{ - return sk->sk_family == AF_INET6 && - inet_sk(sk)->inet_num == IPPROTO_ICMPV6; -} - #endif diff --git a/include/linux/mroute.h b/include/linux/mroute.h index 94c6e6f549f0..4c5003afee6c 100644 --- a/include/linux/mroute.h +++ b/include/linux/mroute.h @@ -16,12 +16,6 @@ static inline int ip_mroute_opt(int opt) return opt >= MRT_BASE && opt <= MRT_MAX; } -static inline int sk_is_ipmr(struct sock *sk) -{ - return sk->sk_family == AF_INET && - inet_sk(sk)->inet_num == IPPROTO_IGMP; -} - int ip_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int); int ip_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t); int ipmr_ioctl(struct sock *sk, int cmd, void *arg); @@ -57,11 +51,6 @@ static inline int ip_mroute_opt(int opt) return 0; } -static inline int sk_is_ipmr(struct sock *sk) -{ - return 0; -} - static inline bool ipmr_rule_default(const struct fib_rule *rule) { return true; -- cgit v1.2.3 From aa571b6275fb60da443c490ebeef021a6897d332 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Mon, 19 Jun 2023 11:24:00 +0200 Subject: net: stmmac: add new switch to struct plat_stmmacenet_data On some platforms, the PCS can be integrated in the MAC so the driver will not see any PCS link activity. Add a switch that allows the platform drivers to let the core code know. Signed-off-by: Bartosz Golaszewski Reviewed-by: Jose Abreu Signed-off-by: Jakub Kicinski --- include/linux/stmmac.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 225751a8fd8e..06090538fe2d 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -293,5 +293,6 @@ struct plat_stmmacenet_data { bool sph_disable; bool serdes_up_after_phy_linkup; const struct dwmac4_addrs *dwmac4_addrs; + bool has_integrated_pcs; }; #endif -- cgit v1.2.3 From 5c1f97537bfb9f358e0ecc88c3c8a913c51944cb Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Mon, 19 Jun 2023 16:26:48 +0300 Subject: wifi: mac80211: store BSS param change count from assoc response When receiving a multi-link association response, make sure to track the BSS parameter change count for each link, including the assoc link. Signed-off-by: Johannes Berg Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230619161906.1799c164e7e9.I8e2c1f5eec6eec3fab525ae2dead9f6f099a2427@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index fa679613c562..15c4e12b6fc7 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4689,6 +4689,34 @@ static inline u8 ieee80211_mle_common_size(const u8 *data) return sizeof(*mle) + common + mle->variable[0]; } +/** + * ieee80211_mle_get_bss_param_ch_cnt - returns the BSS parameter change count + * @mle: the basic multi link element + * + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + * + * If the BSS parameter change count value can't be found (the presence bit + * for it is clear), 0 will be returned. + */ +static inline u8 +ieee80211_mle_get_bss_param_ch_cnt(const struct ieee80211_multi_link_elem *mle) +{ + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + /* common points now at the beginning of ieee80211_mle_basic_common_info */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)) + return 0; + + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + + return *common; +} + /** * ieee80211_mle_get_eml_sync_delay - returns the medium sync delay * @data: pointer to the multi link EHT IE @@ -4902,6 +4930,42 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, fixed + prof->sta_info_len <= len; } +/** + * ieee80211_mle_basic_sta_prof_bss_param_ch_cnt - get per-STA profile BSS + * parameter change count + * @prof: the per-STA profile, having been checked with + * ieee80211_mle_basic_sta_prof_size_ok() for the correct length + * + * Return: The BSS parameter change count value if present, 0 otherwise. + */ +static inline u8 +ieee80211_mle_basic_sta_prof_bss_param_ch_cnt(const struct ieee80211_mle_per_sta_profile *prof) +{ + u16 control = le16_to_cpu(prof->control); + const u8 *pos = prof->variable; + + if (!(control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT)) + return 0; + + if (control & IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT) + pos += 6; + if (control & IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT) + pos += 2; + if (control & IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT) + pos += 8; + if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT) + pos += 2; + if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE && + control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) { + if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) + pos += 2; + else + pos += 1; + } + + return *pos; +} + #define IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID 0x000f #define IEEE80211_MLE_STA_RECONF_CONTROL_COMPLETE_PROFILE 0x0010 #define IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT 0x0020 -- cgit v1.2.3 From 4ef2f53e50cba9780057b51357ef45cb5f49859d Mon Sep 17 00:00:00 2001 From: Ilan Peer Date: Mon, 19 Jun 2023 16:26:52 +0300 Subject: wifi: cfg80211: Retrieve PSD information from RNR AP information Retrieve the Power Spectral Density (PSD) value from RNR AP information entry and store it so it could be used by the drivers. PSD value is explained in Section 9.4.2.170 of Draft P802.11Revme_D2.0. Signed-off-by: Ilan Peer Signed-off-by: Gregory Greenman Link: https://lore.kernel.org/r/20230619161906.067ded2b8fc3.I9f407ab5800cbb07045a0537a513012960ced740@changeid Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 15c4e12b6fc7..6f1747a9c106 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4504,6 +4504,9 @@ static inline bool for_each_element_completed(const struct element *element, #define IEEE80211_RNR_TBTT_PARAMS_PROBE_ACTIVE 0x20 #define IEEE80211_RNR_TBTT_PARAMS_COLOC_AP 0x40 +#define IEEE80211_RNR_TBTT_PARAMS_PSD_NO_LIMIT 127 +#define IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED -128 + struct ieee80211_neighbor_ap_info { u8 tbtt_info_hdr; u8 tbtt_info_len; @@ -4539,7 +4542,7 @@ struct ieee80211_tbtt_info_7_8_9 { /* The following element is optional, structure may not grow */ u8 bss_params; - u8 psd_20; + s8 psd_20; } __packed; /* Format of the TBTT information element if it has >= 11 bytes */ @@ -4550,7 +4553,7 @@ struct ieee80211_tbtt_info_ge_11 { /* The following elements are optional, structure may grow */ u8 bss_params; - u8 psd_20; + s8 psd_20; struct ieee80211_rnr_mld_params mld_params; } __packed; -- cgit v1.2.3 From c88ad30e3f861c7be4e3b4995554e2b0754059b7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 20 Jun 2023 12:24:33 -0500 Subject: cpufreq: amd-pstate: Add a kernel config option to set default mode Users are having more success with amd-pstate since the introduction of EPP and Guided modes. To expose the driver to more users by default introduce a kernel configuration option for setting the default mode. Users can use an integer to map out which default mode they want to use in lieu of a kernel command line option. This will default to EPP, but only if: 1) The CPU supports an MSR. 2) The system profile is identified 3) The system profile is identified as a non-server by the FADT. Link: https://gitlab.freedesktop.org/hadess/power-profiles-daemon/-/merge_requests/121 Acked-by: Huang Rui Reviewed-by: Gautham R. Shenoy Co-developed-by: Perry Yuan Signed-off-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- include/linux/amd-pstate.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index c10ebf8c42e6..446394f84606 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -94,7 +94,8 @@ struct amd_cpudata { * enum amd_pstate_mode - driver working mode of amd pstate */ enum amd_pstate_mode { - AMD_PSTATE_DISABLE = 0, + AMD_PSTATE_UNDEFINED = 0, + AMD_PSTATE_DISABLE, AMD_PSTATE_PASSIVE, AMD_PSTATE_ACTIVE, AMD_PSTATE_GUIDED, @@ -102,6 +103,7 @@ enum amd_pstate_mode { }; static const char * const amd_pstate_mode_string[] = { + [AMD_PSTATE_UNDEFINED] = "undefined", [AMD_PSTATE_DISABLE] = "disable", [AMD_PSTATE_PASSIVE] = "passive", [AMD_PSTATE_ACTIVE] = "active", -- cgit v1.2.3 From d5e01266e7f5fa12400d4c8aa4e86fe89dcc61e9 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Mon, 19 Jun 2023 22:46:58 +0200 Subject: leds: trigger: netdev: add additional specific link speed mode Add additional modes for specific link speed. Use ethtool APIs to get the current link speed and enable the LED accordingly. Under netdev event handler the rtnl lock is already held and is not needed to be set to access ethtool APIs. This is especially useful for PHY and Switch that supports LEDs hw control for specific link speed. (example scenario a PHY that have 2 LED connected one green and one orange where the green is turned on with 1000mbps speed and orange is turned on with 10mpbs speed) On mode set from sysfs we check if we have enabled split link speed mode and reject enabling generic link mode to prevent wrong and redundant configuration. Rework logic on the set baseline state to support these new modes to select if we need to turn on or off the LED. Add additional modes: - link_10: Turn on LED when link speed is 10mbps - link_100: Turn on LED when link speed is 100mbps - link_1000: Turn on LED when link speed is 1000mbps Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Acked-by: Lee Jones Signed-off-by: Jakub Kicinski --- include/linux/leds.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index 8af62ff431f0..126b79019429 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -555,6 +555,9 @@ static inline void *led_get_trigger_data(struct led_classdev *led_cdev) /* Trigger specific enum */ enum led_trigger_netdev_modes { TRIGGER_NETDEV_LINK = 0, + TRIGGER_NETDEV_LINK_10, + TRIGGER_NETDEV_LINK_100, + TRIGGER_NETDEV_LINK_1000, TRIGGER_NETDEV_TX, TRIGGER_NETDEV_RX, -- cgit v1.2.3 From f22f95b9ff1551c9bab13104131929f33d51f23f Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Mon, 19 Jun 2023 22:46:59 +0200 Subject: leds: trigger: netdev: add additional specific link duplex mode Add additional modes for specific link duplex. Use ethtool APIs to get the current link duplex and enable the LED accordingly. Under netdev event handler the rtnl lock is already held and is not needed to be set to access ethtool APIs. This is especially useful for PHY and Switch that supports LEDs hw control for specific link duplex. Add additional modes: - half_duplex: Turn on LED when link is half duplex - full_duplex: Turn on LED when link is full duplex Signed-off-by: Christian Marangi Reviewed-by: Andrew Lunn Acked-by: Lee Jones Signed-off-by: Jakub Kicinski --- include/linux/leds.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/leds.h b/include/linux/leds.h index 126b79019429..3a65ff72bb04 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -558,6 +558,8 @@ enum led_trigger_netdev_modes { TRIGGER_NETDEV_LINK_10, TRIGGER_NETDEV_LINK_100, TRIGGER_NETDEV_LINK_1000, + TRIGGER_NETDEV_HALF_DUPLEX, + TRIGGER_NETDEV_FULL_DUPLEX, TRIGGER_NETDEV_TX, TRIGGER_NETDEV_RX, -- cgit v1.2.3 From 9a5cb79762e0eda17ca15c2a6eaca4622383c21c Mon Sep 17 00:00:00 2001 From: Gilad Sever Date: Wed, 21 Jun 2023 13:42:10 +0300 Subject: bpf: Fix bpf socket lookup from tc/xdp to respect socket VRF bindings When calling bpf_sk_lookup_tcp(), bpf_sk_lookup_udp() or bpf_skc_lookup_tcp() from tc/xdp ingress, VRF socket bindings aren't respoected, i.e. unbound sockets are returned, and bound sockets aren't found. VRF binding is determined by the sdif argument to sk_lookup(), however when called from tc the IP SKB control block isn't initialized and thus inet{,6}_sdif() always returns 0. Fix by calculating sdif for the tc/xdp flows by observing the device's l3 enslaved state. The cg/sk_skb hooking points which are expected to support inet{,6}_sdif() pass sdif=-1 which makes __bpf_skc_lookup() use the existing logic. Fixes: 6acc9b432e67 ("bpf: Add helper to retrieve socket in BPF") Signed-off-by: Gilad Sever Signed-off-by: Daniel Borkmann Reviewed-by: Shmulik Ladkani Reviewed-by: Eyal Birger Acked-by: Stanislav Fomichev Cc: David Ahern Link: https://lore.kernel.org/bpf/20230621104211.301902-4-gilad9366@gmail.com --- include/linux/netdevice.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 08fbd4622ccf..8c95ebbcf203 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -5090,6 +5090,15 @@ static inline bool netif_is_l3_slave(const struct net_device *dev) return dev->priv_flags & IFF_L3MDEV_SLAVE; } +static inline int dev_sdif(const struct net_device *dev) +{ +#ifdef CONFIG_NET_L3_MASTER_DEV + if (netif_is_l3_slave(dev)) + return dev->ifindex; +#endif + return 0; +} + static inline bool netif_is_bridge_master(const struct net_device *dev) { return dev->priv_flags & IFF_EBRIDGE; -- cgit v1.2.3 From 9fde4c557f78ee2f3626e92b4089ce9d54a2573a Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Sun, 11 Jun 2023 11:57:26 +0900 Subject: can: length: fix bitstuffing count The Stuff Bit Count is always coded on 4 bits [1]. Update the Stuff Bit Count size accordingly. In addition, the CRC fields of CAN FD Frames contain stuff bits at fixed positions called fixed stuff bits [2]. The CRC field starts with a fixed stuff bit and then has another fixed stuff bit after each fourth bit [2], which allows us to derive this formula: FSB count = 1 + round_down(len(CRC field)/4) The length of the CRC field is [1]: len(CRC field) = len(Stuff Bit Count) + len(CRC) = 4 + len(CRC) with len(CRC) either 17 or 21 bits depending of the payload length. In conclusion, for CRC17: FSB count = 1 + round_down((4 + 17)/4) = 6 and for CRC 21: FSB count = 1 + round_down((4 + 21)/4) = 7 Add a Fixed Stuff bits (FSB) field with above values and update CANFD_FRAME_OVERHEAD_SFF and CANFD_FRAME_OVERHEAD_EFF accordingly. [1] ISO 11898-1:2015 section 10.4.2.6 "CRC field": The CRC field shall contain the CRC sequence followed by a recessive CRC delimiter. For FD Frames, the CRC field shall also contain the stuff count. Stuff count If FD Frames, the stuff count shall be at the beginning of the CRC field. It shall consist of the stuff bit count modulo 8 in a 3-bit gray code followed by a parity bit [...] [2] ISO 11898-1:2015 paragraph 10.5 "Frame coding": In the CRC field of FD Frames, the stuff bits shall be inserted at fixed positions; they are called fixed stuff bits. There shall be a fixed stuff bit before the first bit of the stuff count, even if the last bits of the preceding field are a sequence of five consecutive bits of identical value, there shall be only the fixed stuff bit, there shall not be two consecutive stuff bits. A further fixed stuff bit shall be inserted after each fourth bit of the CRC field [...] Fixes: 85d99c3e2a13 ("can: length: can_skb_get_frame_len(): introduce function to get data length of frame in data link layer") Suggested-by: Thomas Kopp Signed-off-by: Vincent Mailhol Reviewed-by: Thomas Kopp Link: https://lore.kernel.org/all/20230611025728.450837-2-mailhol.vincent@wanadoo.fr Signed-off-by: Marc Kleine-Budde --- include/linux/can/length.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/can/length.h b/include/linux/can/length.h index 69336549d24f..b8c12c83bc51 100644 --- a/include/linux/can/length.h +++ b/include/linux/can/length.h @@ -72,17 +72,18 @@ * Error Status Indicator (ESI) 1 * Data length code (DLC) 4 * Data field 0...512 - * Stuff Bit Count (SBC) 0...16: 4 20...64:5 + * Stuff Bit Count (SBC) 4 * CRC 0...16: 17 20...64:21 * CRC delimiter (CD) 1 + * Fixed Stuff bits (FSB) 0...16: 6 20...64:7 * ACK slot (AS) 1 * ACK delimiter (AD) 1 * End-of-frame (EOF) 7 * Inter frame spacing 3 * - * assuming CRC21, rounded up and ignoring bitstuffing + * assuming CRC21, rounded up and ignoring dynamic bitstuffing */ -#define CANFD_FRAME_OVERHEAD_SFF DIV_ROUND_UP(61, 8) +#define CANFD_FRAME_OVERHEAD_SFF DIV_ROUND_UP(67, 8) /* * Size of a CAN-FD Extended Frame @@ -101,17 +102,18 @@ * Error Status Indicator (ESI) 1 * Data length code (DLC) 4 * Data field 0...512 - * Stuff Bit Count (SBC) 0...16: 4 20...64:5 + * Stuff Bit Count (SBC) 4 * CRC 0...16: 17 20...64:21 * CRC delimiter (CD) 1 + * Fixed Stuff bits (FSB) 0...16: 6 20...64:7 * ACK slot (AS) 1 * ACK delimiter (AD) 1 * End-of-frame (EOF) 7 * Inter frame spacing 3 * - * assuming CRC21, rounded up and ignoring bitstuffing + * assuming CRC21, rounded up and ignoring dynamic bitstuffing */ -#define CANFD_FRAME_OVERHEAD_EFF DIV_ROUND_UP(80, 8) +#define CANFD_FRAME_OVERHEAD_EFF DIV_ROUND_UP(86, 8) /* * Maximum size of a Classical CAN frame -- cgit v1.2.3 From 10711b11102bfc351240982d46a51d4eecc28c10 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Sun, 11 Jun 2023 11:57:27 +0900 Subject: can: length: fix description of the RRS field The CAN-FD frames only have one reserved bit. The bit corresponding to Classical CAN frame's RTR bit is called the "Remote Request Substitution (RRS)" [1]. N.B. The RRS is not to be confused with the Substitute Remote Request (SRR). Fix the description in the CANFD_FRAME_OVERHEAD_SFF/EFF macros. The total remains unchanged, so this is just a documentation fix. In addition to the above add myself as copyright owner for 2020 (as coauthor of the initial version, c.f. Fixes tag). [1] ISO 11898-1:2015 paragraph 10.4.2.3 "Arbitration field": RSS bit [only in FD Frames] The RRS bit shall be transmitted in FD Frames at the position of the RTR bit in Classical Frames. The RRS bit shall be transmitted dominant, but receivers shall accept recessive and dominant RRS bits. Fixes: 85d99c3e2a13 ("can: length: can_skb_get_frame_len(): introduce function to get data length of frame in data link layer") Signed-off-by: Vincent Mailhol Reviewed-by: Thomas Kopp Link: https://lore.kernel.org/all/20230611025728.450837-3-mailhol.vincent@wanadoo.fr Signed-off-by: Marc Kleine-Budde --- include/linux/can/length.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/can/length.h b/include/linux/can/length.h index b8c12c83bc51..521fdbce2d69 100644 --- a/include/linux/can/length.h +++ b/include/linux/can/length.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2020 Oliver Hartkopp * Copyright (C) 2020 Marc Kleine-Budde + * Copyright (C) 2020 Vincent Mailhol */ #ifndef _CAN_LENGTH_H @@ -64,7 +65,7 @@ * --------------------------------------------------------- * Start-of-frame 1 * Identifier 11 - * Reserved bit (r1) 1 + * Remote Request Substitution (RRS) 1 * Identifier extension bit (IDE) 1 * Flexible data rate format (FDF) 1 * Reserved bit (r0) 1 @@ -95,7 +96,7 @@ * Substitute remote request (SRR) 1 * Identifier extension bit (IDE) 1 * Identifier B 18 - * Reserved bit (r1) 1 + * Remote Request Substitution (RRS) 1 * Flexible data rate format (FDF) 1 * Reserved bit (r0) 1 * Bit Rate Switch (BRS) 1 -- cgit v1.2.3 From 80a2fbce456e3f73b4f49ce12fefa3215a3d0773 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Sun, 11 Jun 2023 11:57:28 +0900 Subject: can: length: refactor frame lengths definition to add size in bits Introduce a method to calculate the exact size in bits of a CAN(-FD) frame with or without dynamic bitstuffing. These are all the possible combinations taken into account: - Classical CAN or CAN-FD - Standard or Extended frame format - CAN-FD CRC17 or CRC21 - Include or not intermission Instead of doing several individual macro definitions, declare the can_frame_bits() function-like macro. To this extent, do a full refactoring of the length definitions. In addition add the can_frame_bytes(). This function-like macro replaces the existing macro: - CAN_FRAME_OVERHEAD_SFF: can_frame_bytes(false, false, 0) - CAN_FRAME_OVERHEAD_EFF: can_frame_bytes(false, true, 0) - CANFD_FRAME_OVERHEAD_SFF: can_frame_bytes(true, false, 0) - CANFD_FRAME_OVERHEAD_EFF: can_frame_bytes(true, true, 0) Function-like macros were chosen over inline functions because they can be used to initialize const struct fields. The different maximum frame lengths (maximum data length, including intermission) are as follow: Frame type bits bytes ------------------------------------------------------- Classic CAN SFF no bitstuffing 111 14 Classic CAN EFF no bitstuffing 131 17 Classic CAN SFF bitstuffing 135 17 Classic CAN EFF bitstuffing 160 20 CAN-FD SFF no bitstuffing 579 73 CAN-FD EFF no bitstuffing 598 75 CAN-FD SFF bitstuffing 712 89 CAN-FD EFF bitstuffing 736 92 The macro CAN_FRAME_LEN_MAX and CANFD_FRAME_LEN_MAX are kept as an alias to, respectively, can_frame_bytes(false, true, CAN_MAX_DLEN) and can_frame_bytes(true, true, CANFD_MAX_DLEN). In addition to the above: - Use ISO 11898-1:2015 definitions for the names of the CAN frame fields. - Include linux/bits.h for use of BITS_PER_BYTE. - Include linux/math.h for use of mult_frac() and DIV_ROUND_UP(). N.B: the use of DIV_ROUND_UP() is not new to this patch, but the include was previously omitted. - Add copyright 2023 for myself. Suggested-by: Thomas Kopp Signed-off-by: Vincent Mailhol Reviewed-by: Thomas Kopp Link: https://lore.kernel.org/all/20230611025728.450837-4-mailhol.vincent@wanadoo.fr Signed-off-by: Marc Kleine-Budde --- include/linux/can/length.h | 302 ++++++++++++++++++++++++++++++++------------- 1 file changed, 214 insertions(+), 88 deletions(-) (limited to 'include/linux') diff --git a/include/linux/can/length.h b/include/linux/can/length.h index 521fdbce2d69..abc978b38f79 100644 --- a/include/linux/can/length.h +++ b/include/linux/can/length.h @@ -1,132 +1,258 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2020 Oliver Hartkopp * Copyright (C) 2020 Marc Kleine-Budde - * Copyright (C) 2020 Vincent Mailhol + * Copyright (C) 2020, 2023 Vincent Mailhol */ #ifndef _CAN_LENGTH_H #define _CAN_LENGTH_H +#include #include #include +#include /* - * Size of a Classical CAN Standard Frame + * Size of a Classical CAN Standard Frame header in bits * - * Name of Field Bits + * Name of Field Bits * --------------------------------------------------------- - * Start-of-frame 1 - * Identifier 11 - * Remote transmission request (RTR) 1 - * Identifier extension bit (IDE) 1 - * Reserved bit (r0) 1 - * Data length code (DLC) 4 - * Data field 0...64 - * CRC 15 - * CRC delimiter 1 - * ACK slot 1 - * ACK delimiter 1 - * End-of-frame (EOF) 7 - * Inter frame spacing 3 + * Start Of Frame (SOF) 1 + * Arbitration field: + * base ID 11 + * Remote Transmission Request (RTR) 1 + * Control field: + * IDentifier Extension bit (IDE) 1 + * FD Format indicator (FDF) 1 + * Data Length Code (DLC) 4 + * + * including all fields preceding the data field, ignoring bitstuffing + */ +#define CAN_FRAME_HEADER_SFF_BITS 19 + +/* + * Size of a Classical CAN Extended Frame header in bits * - * rounded up and ignoring bitstuffing + * Name of Field Bits + * --------------------------------------------------------- + * Start Of Frame (SOF) 1 + * Arbitration field: + * base ID 11 + * Substitute Remote Request (SRR) 1 + * IDentifier Extension bit (IDE) 1 + * ID extension 18 + * Remote Transmission Request (RTR) 1 + * Control field: + * FD Format indicator (FDF) 1 + * Reserved bit (r0) 1 + * Data length code (DLC) 4 + * + * including all fields preceding the data field, ignoring bitstuffing */ -#define CAN_FRAME_OVERHEAD_SFF DIV_ROUND_UP(47, 8) +#define CAN_FRAME_HEADER_EFF_BITS 39 /* - * Size of a Classical CAN Extended Frame + * Size of a CAN-FD Standard Frame in bits + * + * Name of Field Bits + * --------------------------------------------------------- + * Start Of Frame (SOF) 1 + * Arbitration field: + * base ID 11 + * Remote Request Substitution (RRS) 1 + * Control field: + * IDentifier Extension bit (IDE) 1 + * FD Format indicator (FDF) 1 + * Reserved bit (res) 1 + * Bit Rate Switch (BRS) 1 + * Error Status Indicator (ESI) 1 + * Data length code (DLC) 4 + * + * including all fields preceding the data field, ignoring bitstuffing + */ +#define CANFD_FRAME_HEADER_SFF_BITS 22 + +/* + * Size of a CAN-FD Extended Frame in bits + * + * Name of Field Bits + * --------------------------------------------------------- + * Start Of Frame (SOF) 1 + * Arbitration field: + * base ID 11 + * Substitute Remote Request (SRR) 1 + * IDentifier Extension bit (IDE) 1 + * ID extension 18 + * Remote Request Substitution (RRS) 1 + * Control field: + * FD Format indicator (FDF) 1 + * Reserved bit (res) 1 + * Bit Rate Switch (BRS) 1 + * Error Status Indicator (ESI) 1 + * Data length code (DLC) 4 + * + * including all fields preceding the data field, ignoring bitstuffing + */ +#define CANFD_FRAME_HEADER_EFF_BITS 41 + +/* + * Size of a CAN CRC Field in bits * * Name of Field Bits * --------------------------------------------------------- - * Start-of-frame 1 - * Identifier A 11 - * Substitute remote request (SRR) 1 - * Identifier extension bit (IDE) 1 - * Identifier B 18 - * Remote transmission request (RTR) 1 - * Reserved bits (r1, r0) 2 - * Data length code (DLC) 4 - * Data field 0...64 - * CRC 15 - * CRC delimiter 1 - * ACK slot 1 - * ACK delimiter 1 - * End-of-frame (EOF) 7 - * Inter frame spacing 3 + * CRC sequence (CRC15) 15 + * CRC Delimiter 1 + * + * ignoring bitstuffing + */ +#define CAN_FRAME_CRC_FIELD_BITS 16 + +/* + * Size of a CAN-FD CRC17 Field in bits (length: 0..16) * - * rounded up and ignoring bitstuffing + * Name of Field Bits + * --------------------------------------------------------- + * Stuff Count 4 + * CRC Sequence (CRC17) 17 + * CRC Delimiter 1 + * Fixed stuff bits 6 */ -#define CAN_FRAME_OVERHEAD_EFF DIV_ROUND_UP(67, 8) +#define CANFD_FRAME_CRC17_FIELD_BITS 28 /* - * Size of a CAN-FD Standard Frame + * Size of a CAN-FD CRC21 Field in bits (length: 20..64) * * Name of Field Bits * --------------------------------------------------------- - * Start-of-frame 1 - * Identifier 11 - * Remote Request Substitution (RRS) 1 - * Identifier extension bit (IDE) 1 - * Flexible data rate format (FDF) 1 - * Reserved bit (r0) 1 - * Bit Rate Switch (BRS) 1 - * Error Status Indicator (ESI) 1 - * Data length code (DLC) 4 - * Data field 0...512 - * Stuff Bit Count (SBC) 4 - * CRC 0...16: 17 20...64:21 - * CRC delimiter (CD) 1 - * Fixed Stuff bits (FSB) 0...16: 6 20...64:7 - * ACK slot (AS) 1 - * ACK delimiter (AD) 1 - * End-of-frame (EOF) 7 - * Inter frame spacing 3 - * - * assuming CRC21, rounded up and ignoring dynamic bitstuffing - */ -#define CANFD_FRAME_OVERHEAD_SFF DIV_ROUND_UP(67, 8) + * Stuff Count 4 + * CRC sequence (CRC21) 21 + * CRC Delimiter 1 + * Fixed stuff bits 7 + */ +#define CANFD_FRAME_CRC21_FIELD_BITS 33 /* - * Size of a CAN-FD Extended Frame + * Size of a CAN(-FD) Frame footer in bits * * Name of Field Bits * --------------------------------------------------------- - * Start-of-frame 1 - * Identifier A 11 - * Substitute remote request (SRR) 1 - * Identifier extension bit (IDE) 1 - * Identifier B 18 - * Remote Request Substitution (RRS) 1 - * Flexible data rate format (FDF) 1 - * Reserved bit (r0) 1 - * Bit Rate Switch (BRS) 1 - * Error Status Indicator (ESI) 1 - * Data length code (DLC) 4 - * Data field 0...512 - * Stuff Bit Count (SBC) 4 - * CRC 0...16: 17 20...64:21 - * CRC delimiter (CD) 1 - * Fixed Stuff bits (FSB) 0...16: 6 20...64:7 - * ACK slot (AS) 1 - * ACK delimiter (AD) 1 - * End-of-frame (EOF) 7 - * Inter frame spacing 3 - * - * assuming CRC21, rounded up and ignoring dynamic bitstuffing - */ -#define CANFD_FRAME_OVERHEAD_EFF DIV_ROUND_UP(86, 8) + * ACK slot 1 + * ACK delimiter 1 + * End Of Frame (EOF) 7 + * + * including all fields following the CRC field + */ +#define CAN_FRAME_FOOTER_BITS 9 + +/* + * First part of the Inter Frame Space + * (a.k.a. IMF - intermission field) + */ +#define CAN_INTERMISSION_BITS 3 + +/** + * can_bitstuffing_len() - Calculate the maximum length with bitstuffing + * @destuffed_len: length of a destuffed bit stream + * + * The worst bit stuffing case is a sequence in which dominant and + * recessive bits alternate every four bits: + * + * Destuffed: 1 1111 0000 1111 0000 1111 + * Stuffed: 1 1111o 0000i 1111o 0000i 1111o + * + * Nomenclature + * + * - "0": dominant bit + * - "o": dominant stuff bit + * - "1": recessive bit + * - "i": recessive stuff bit + * + * Aside from the first bit, one stuff bit is added every four bits. + * + * Return: length of the stuffed bit stream in the worst case scenario. + */ +#define can_bitstuffing_len(destuffed_len) \ + (destuffed_len + (destuffed_len - 1) / 4) + +#define __can_bitstuffing_len(bitstuffing, destuffed_len) \ + (bitstuffing ? can_bitstuffing_len(destuffed_len) : \ + destuffed_len) + +#define __can_cc_frame_bits(is_eff, bitstuffing, \ + intermission, data_len) \ +( \ + __can_bitstuffing_len(bitstuffing, \ + (is_eff ? CAN_FRAME_HEADER_EFF_BITS : \ + CAN_FRAME_HEADER_SFF_BITS) + \ + (data_len) * BITS_PER_BYTE + \ + CAN_FRAME_CRC_FIELD_BITS) + \ + CAN_FRAME_FOOTER_BITS + \ + (intermission ? CAN_INTERMISSION_BITS : 0) \ +) + +#define __can_fd_frame_bits(is_eff, bitstuffing, \ + intermission, data_len) \ +( \ + __can_bitstuffing_len(bitstuffing, \ + (is_eff ? CANFD_FRAME_HEADER_EFF_BITS : \ + CANFD_FRAME_HEADER_SFF_BITS) + \ + (data_len) * BITS_PER_BYTE) + \ + ((data_len) <= 16 ? \ + CANFD_FRAME_CRC17_FIELD_BITS : \ + CANFD_FRAME_CRC21_FIELD_BITS) + \ + CAN_FRAME_FOOTER_BITS + \ + (intermission ? CAN_INTERMISSION_BITS : 0) \ +) + +/** + * can_frame_bits() - Calculate the number of bits on the wire in a + * CAN frame + * @is_fd: true: CAN-FD frame; false: Classical CAN frame. + * @is_eff: true: Extended frame; false: Standard frame. + * @bitstuffing: true: calculate the bitstuffing worst case; false: + * calculate the bitstuffing best case (no dynamic + * bitstuffing). CAN-FD's fixed stuff bits are always included. + * @intermission: if and only if true, include the inter frame space + * assuming no bus idle (i.e. only the intermission). Strictly + * speaking, the inter frame space is not part of the + * frame. However, it is needed when calculating the delay + * between the Start Of Frame of two consecutive frames. + * @data_len: length of the data field in bytes. Correspond to + * can(fd)_frame->len. Should be zero for remote frames. No + * sanitization is done on @data_len and it shall have no side + * effects. + * + * Return: the numbers of bits on the wire of a CAN frame. + */ +#define can_frame_bits(is_fd, is_eff, bitstuffing, \ + intermission, data_len) \ +( \ + is_fd ? __can_fd_frame_bits(is_eff, bitstuffing, \ + intermission, data_len) : \ + __can_cc_frame_bits(is_eff, bitstuffing, \ + intermission, data_len) \ +) + +/* + * Number of bytes in a CAN frame + * (rounded up, including intermission) + */ +#define can_frame_bytes(is_fd, is_eff, bitstuffing, data_len) \ + DIV_ROUND_UP(can_frame_bits(is_fd, is_eff, bitstuffing, \ + true, data_len), \ + BITS_PER_BYTE) /* * Maximum size of a Classical CAN frame - * (rounded up and ignoring bitstuffing) + * (rounded up, ignoring bitstuffing but including intermission) */ -#define CAN_FRAME_LEN_MAX (CAN_FRAME_OVERHEAD_EFF + CAN_MAX_DLEN) +#define CAN_FRAME_LEN_MAX can_frame_bytes(false, true, false, CAN_MAX_DLEN) /* * Maximum size of a CAN-FD frame - * (rounded up and ignoring bitstuffing) + * (rounded up, ignoring dynamic bitstuffing but including intermission) */ -#define CANFD_FRAME_LEN_MAX (CANFD_FRAME_OVERHEAD_EFF + CANFD_MAX_DLEN) +#define CANFD_FRAME_LEN_MAX can_frame_bytes(true, true, false, CANFD_MAX_DLEN) /* * can_cc_dlc2len(value) - convert a given data length code (dlc) of a -- cgit v1.2.3 From d8e4ebf87018736c0c29e2eb4afe3915156483cd Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 22 Jun 2023 11:06:32 +0200 Subject: spi: Create a helper to derive adaptive timeouts Big transfers might take a bit of time, too constraining timeouts might lead to false positives. In order to simplify the drivers work and with the goal of factorizing code in mind, let's add a helper that can be used by any spi controller driver to derive a relevant per-transfer timeout value. The logic is simple: we know how much time it would take to transfer a byte, we can easily derive the total theoretical amount of time involved for each transfer. We multiply it by two to have a bit of margin and enforce a minimum of 500ms. Suggested-by: Mark Brown Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/r/Message-Id: <20230622090634.3411468-2-miquel.raynal@bootlin.com> Signed-off-by: Mark Brown --- include/linux/spi/spi.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index cfe42f8cd7a4..32c94eae8926 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -1261,6 +1261,23 @@ static inline bool spi_is_bpw_supported(struct spi_device *spi, u32 bpw) return false; } +/** + * spi_controller_xfer_timeout - Compute a suitable timeout value + * @ctlr: SPI device + * @xfer: Transfer descriptor + * + * Compute a relevant timeout value for the given transfer. We derive the time + * that it would take on a single data line and take twice this amount of time + * with a minimum of 500ms to avoid false positives on loaded systems. + * + * Returns: Transfer timeout value in milliseconds. + */ +static inline unsigned int spi_controller_xfer_timeout(struct spi_controller *ctlr, + struct spi_transfer *xfer) +{ + return max(xfer->len * 8 * 2 / (xfer->speed_hz / 1000), 500U); +} + /*---------------------------------------------------------------------------*/ /* SPI transfer replacement methods which make use of spi_res */ -- cgit v1.2.3 From 31b5a547622b3782388eb676081da1eefe5b98d2 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Thu, 22 Jun 2023 19:44:22 +0200 Subject: wifi: ieee80211: fix erroneous NSTR bitmap size checks The complete profile bit together with the NSTR link pair present bit indicate whether or not the NSTR bitmap is, the NSTR bitmap size just indicates how big it is. Fixes: 7b6f08771bf6 ("wifi: ieee80211: Support validating ML station profile length") Fixes: 5c1f97537bfb ("wifi: mac80211: store BSS param change count from assoc response") Reported-by: Dan Carpenter Signed-off-by: Johannes Berg --- include/linux/ieee80211.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 6f1747a9c106..4b998090898e 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -4920,7 +4920,7 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT) info_len += 2; if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE && - control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) { + control & IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT) { if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) info_len += 2; else @@ -4959,7 +4959,7 @@ ieee80211_mle_basic_sta_prof_bss_param_ch_cnt(const struct ieee80211_mle_per_sta if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT) pos += 2; if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE && - control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) { + control & IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT) { if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) pos += 2; else -- cgit v1.2.3 From f99d471afa03f770149f1cc60a288b9a08285903 Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 16 Jun 2023 13:06:22 +0100 Subject: net: phylink: add PCS negotiation mode PCS have to work out whether they should enable PCS negotiation by looking at the "mode" and "interface" arguments, and the Autoneg bit in the advertising mask. This leads to some complex logic, so lets pull that out into phylink and instead pass a "neg_mode" argument to the PCS configuration and link up methods, instead of the "mode" argument. In order to transition drivers, add a "neg_mode" flag to the phylink PCS structure to PCS can indicate whether they want to be passed the neg_mode or the old mode argument. Signed-off-by: Russell King (Oracle) Link: https://lore.kernel.org/r/E1qA8De-00EaFA-Ht@rmk-PC.armlinux.org.uk Signed-off-by: Jakub Kicinski --- include/linux/phylink.h | 104 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 0cf07d7d11b8..2b322d7fa51a 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -21,6 +21,24 @@ enum { MLO_AN_FIXED, /* Fixed-link mode */ MLO_AN_INBAND, /* In-band protocol */ + /* PCS "negotiation" mode. + * PHYLINK_PCS_NEG_NONE - protocol has no inband capability + * PHYLINK_PCS_NEG_OUTBAND - some out of band or fixed link setting + * PHYLINK_PCS_NEG_INBAND_DISABLED - inband mode disabled, e.g. + * 1000base-X with autoneg off + * PHYLINK_PCS_NEG_INBAND_ENABLED - inband mode enabled + * Additionally, this can be tested using bitmasks: + * PHYLINK_PCS_NEG_INBAND - inband mode selected + * PHYLINK_PCS_NEG_ENABLED - negotiation mode enabled + */ + PHYLINK_PCS_NEG_NONE = 0, + PHYLINK_PCS_NEG_ENABLED = BIT(4), + PHYLINK_PCS_NEG_OUTBAND = BIT(5), + PHYLINK_PCS_NEG_INBAND = BIT(6), + PHYLINK_PCS_NEG_INBAND_DISABLED = PHYLINK_PCS_NEG_INBAND, + PHYLINK_PCS_NEG_INBAND_ENABLED = PHYLINK_PCS_NEG_INBAND | + PHYLINK_PCS_NEG_ENABLED, + /* MAC_SYM_PAUSE and MAC_ASYM_PAUSE are used when configuring our * autonegotiation advertisement. They correspond to the PAUSE and * ASM_DIR bits defined by 802.3, respectively. @@ -79,6 +97,70 @@ static inline bool phylink_autoneg_inband(unsigned int mode) return mode == MLO_AN_INBAND; } +/** + * phylink_pcs_neg_mode() - helper to determine PCS inband mode + * @mode: one of %MLO_AN_FIXED, %MLO_AN_PHY, %MLO_AN_INBAND. + * @interface: interface mode to be used + * @advertising: adertisement ethtool link mode mask + * + * Determines the negotiation mode to be used by the PCS, and returns + * one of: + * %PHYLINK_PCS_NEG_NONE: interface mode does not support inband + * %PHYLINK_PCS_NEG_OUTBAND: an out of band mode (e.g. reading the PHY) + * will be used. + * %PHYLINK_PCS_NEG_INBAND_DISABLED: inband mode selected but autoneg disabled + * %PHYLINK_PCS_NEG_INBAND_ENABLED: inband mode selected and autoneg enabled + * + * Note: this is for cases where the PCS itself is involved in negotiation + * (e.g. Clause 37, SGMII and similar) not Clause 73. + */ +static inline unsigned int phylink_pcs_neg_mode(unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising) +{ + unsigned int neg_mode; + + switch (interface) { + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_QUSGMII: + case PHY_INTERFACE_MODE_USXGMII: + /* These protocols are designed for use with a PHY which + * communicates its negotiation result back to the MAC via + * inband communication. Note: there exist PHYs that run + * with SGMII but do not send the inband data. + */ + if (!phylink_autoneg_inband(mode)) + neg_mode = PHYLINK_PCS_NEG_OUTBAND; + else + neg_mode = PHYLINK_PCS_NEG_INBAND_ENABLED; + break; + + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + /* 1000base-X is designed for use media-side for Fibre + * connections, and thus the Autoneg bit needs to be + * taken into account. We also do this for 2500base-X + * as well, but drivers may not support this, so may + * need to override this. + */ + if (!phylink_autoneg_inband(mode)) + neg_mode = PHYLINK_PCS_NEG_OUTBAND; + else if (linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + advertising)) + neg_mode = PHYLINK_PCS_NEG_INBAND_ENABLED; + else + neg_mode = PHYLINK_PCS_NEG_INBAND_DISABLED; + break; + + default: + neg_mode = PHYLINK_PCS_NEG_NONE; + break; + } + + return neg_mode; +} + /** * struct phylink_link_state - link state structure * @advertising: ethtool bitmask containing advertised link modes @@ -436,6 +518,7 @@ struct phylink_pcs_ops; /** * struct phylink_pcs - PHYLINK PCS instance * @ops: a pointer to the &struct phylink_pcs_ops structure + * @neg_mode: provide PCS neg mode via "mode" argument * @poll: poll the PCS for link changes * * This structure is designed to be embedded within the PCS private data, @@ -443,6 +526,7 @@ struct phylink_pcs_ops; */ struct phylink_pcs { const struct phylink_pcs_ops *ops; + bool neg_mode; bool poll; }; @@ -460,12 +544,12 @@ struct phylink_pcs_ops { const struct phylink_link_state *state); void (*pcs_get_state)(struct phylink_pcs *pcs, struct phylink_link_state *state); - int (*pcs_config)(struct phylink_pcs *pcs, unsigned int mode, + int (*pcs_config)(struct phylink_pcs *pcs, unsigned int neg_mode, phy_interface_t interface, const unsigned long *advertising, bool permit_pause_to_mac); void (*pcs_an_restart)(struct phylink_pcs *pcs); - void (*pcs_link_up)(struct phylink_pcs *pcs, unsigned int mode, + void (*pcs_link_up)(struct phylink_pcs *pcs, unsigned int neg_mode, phy_interface_t interface, int speed, int duplex); }; @@ -508,7 +592,7 @@ void pcs_get_state(struct phylink_pcs *pcs, /** * pcs_config() - Configure the PCS mode and advertisement * @pcs: a pointer to a &struct phylink_pcs. - * @mode: one of %MLO_AN_FIXED, %MLO_AN_PHY, %MLO_AN_INBAND. + * @neg_mode: link negotiation mode (see below) * @interface: interface mode to be used * @advertising: adertisement ethtool link mode mask * @permit_pause_to_mac: permit forwarding pause resolution to MAC @@ -526,8 +610,12 @@ void pcs_get_state(struct phylink_pcs *pcs, * For 1000BASE-X, the advertisement should be programmed into the PCS. * * For most 10GBASE-R, there is no advertisement. + * + * The %neg_mode argument should be tested via the phylink_mode_*() family of + * functions, or for PCS that set pcs->neg_mode true, should be tested + * against the %PHYLINK_PCS_NEG_* definitions. */ -int pcs_config(struct phylink_pcs *pcs, unsigned int mode, +int pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode, phy_interface_t interface, const unsigned long *advertising, bool permit_pause_to_mac); @@ -543,7 +631,7 @@ void pcs_an_restart(struct phylink_pcs *pcs); /** * pcs_link_up() - program the PCS for the resolved link configuration * @pcs: a pointer to a &struct phylink_pcs. - * @mode: link autonegotiation mode + * @neg_mode: link negotiation mode (see below) * @interface: link &typedef phy_interface_t mode * @speed: link speed * @duplex: link duplex @@ -552,8 +640,12 @@ void pcs_an_restart(struct phylink_pcs *pcs); * the resolved link parameters. For example, a PCS operating in SGMII * mode without in-band AN needs to be manually configured for the link * and duplex setting. Otherwise, this should be a no-op. + * + * The %mode argument should be tested via the phylink_mode_*() family of + * functions, or for PCS that set pcs->neg_mode true, should be tested + * against the %PHYLINK_PCS_NEG_* definitions. */ -void pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, +void pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, phy_interface_t interface, int speed, int duplex); #endif -- cgit v1.2.3 From febf2aaf05641f3258cc30e072aff65cffc7c82c Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 16 Jun 2023 13:06:32 +0100 Subject: net: phylink: pass neg_mode into phylink_mii_c22_pcs_config() Convert fman_dtsec, xilinx_axienet and pcs-lynx to pass the neg_mode into phylink_mii_c22_pcs_config(). Where appropriate, drivers are updated to have neg_mode passed into their pcs_config() and pcs_link_up() functions. For other drivers, we just hoist the call to phylink_pcs_neg_mode() to their pcs_config() method out of phylink_mii_c22_pcs_config(). Signed-off-by: Russell King (Oracle) Link: https://lore.kernel.org/r/E1qA8Do-00EaFM-Ra@rmk-PC.armlinux.org.uk Signed-off-by: Jakub Kicinski --- include/linux/phylink.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 2b322d7fa51a..516240f1e950 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -743,9 +743,10 @@ void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, struct phylink_link_state *state); int phylink_mii_c22_pcs_encode_advertisement(phy_interface_t interface, const unsigned long *advertising); -int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, +int phylink_mii_c22_pcs_config(struct mdio_device *pcs, phy_interface_t interface, - const unsigned long *advertising); + const unsigned long *advertising, + unsigned int neg_mode); void phylink_mii_c22_pcs_an_restart(struct mdio_device *pcs); void phylink_resolve_c73(struct phylink_link_state *state); -- cgit v1.2.3 From a3a47cfb88fcdf862594eae417611ef533ed8bae Mon Sep 17 00:00:00 2001 From: "Russell King (Oracle)" Date: Fri, 16 Jun 2023 13:06:37 +0100 Subject: net: pcs: xpcs: update PCS driver to use neg_mode Update xpcs to use neg_mode to configure whether inband negotiation should be used. We need to update sja1105 as well as that directly calls into the XPCS driver's config function. Signed-off-by: Russell King (Oracle) Link: https://lore.kernel.org/r/E1qA8Dt-00EaFS-W9@rmk-PC.armlinux.org.uk Signed-off-by: Jakub Kicinski --- include/linux/pcs/pcs-xpcs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pcs/pcs-xpcs.h b/include/linux/pcs/pcs-xpcs.h index ec8175b847cc..ff99cf7a5d0d 100644 --- a/include/linux/pcs/pcs-xpcs.h +++ b/include/linux/pcs/pcs-xpcs.h @@ -29,10 +29,10 @@ struct dw_xpcs { }; int xpcs_get_an_mode(struct dw_xpcs *xpcs, phy_interface_t interface); -void xpcs_link_up(struct phylink_pcs *pcs, unsigned int mode, +void xpcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, phy_interface_t interface, int speed, int duplex); int xpcs_do_config(struct dw_xpcs *xpcs, phy_interface_t interface, - unsigned int mode, const unsigned long *advertising); + const unsigned long *advertising, unsigned int neg_mode); void xpcs_get_interfaces(struct dw_xpcs *xpcs, unsigned long *interfaces); int xpcs_config_eee(struct dw_xpcs *xpcs, int mult_fact_100ns, int enable); -- cgit v1.2.3 From 9ee473c259de393718ae1c6fdd51271d5d087c4b Mon Sep 17 00:00:00 2001 From: Lama Kayal Date: Mon, 12 Jun 2023 16:34:43 +0300 Subject: net/mlx5: Fix reserved at offset in hca_cap register A member of struct mlx5_ifc_cmd_hca_cap_bits has been mistakenly assigned the wrong reserved_at offset value. Correct it to align to the right value, thus avoid future miscalculation. Signed-off-by: Lama Kayal Reviewed-by: Tariq Toukan Reviewed-by: Rahul Rameshbabu Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 354c7e326eab..33344a71c3e3 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1710,9 +1710,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 regexp_params[0x1]; u8 uar_sz[0x6]; u8 port_selection_cap[0x1]; - u8 reserved_at_248[0x1]; + u8 reserved_at_251[0x1]; u8 umem_uid_0[0x1]; - u8 reserved_at_250[0x5]; + u8 reserved_at_253[0x5]; u8 log_pg_sz[0x8]; u8 bf[0x1]; -- cgit v1.2.3 From 61167ad5fecdeaa037f3df1ba354dddd5f66a1ed Mon Sep 17 00:00:00 2001 From: Yajun Deng Date: Mon, 19 Jun 2023 10:34:06 +0800 Subject: mm: pass nid to reserve_bootmem_region() early_pfn_to_nid() is called frequently in init_reserved_page(), it returns the node id of the PFN. These PFN are probably from the same memory region, they have the same node id. It's not necessary to call early_pfn_to_nid() for each PFN. Pass nid to reserve_bootmem_region() and drop the call to early_pfn_to_nid() in init_reserved_page(). Also, set nid on all reserved pages before doing this, as some reserved memory regions may not be set nid. The most beneficial function is memmap_init_reserved_pages() if CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled. The following data was tested on an x86 machine with 190GB of RAM. before: memmap_init_reserved_pages() 67ms after: memmap_init_reserved_pages() 20ms Link: https://lkml.kernel.org/r/20230619023406.424298-1-yajun.deng@linux.dev Signed-off-by: Yajun Deng Reviewed-by: Mike Rapoport (IBM) Signed-off-by: Andrew Morton --- include/linux/mm.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index cf43deb25553..9ecb8b9c07f6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2940,7 +2940,8 @@ extern unsigned long free_reserved_area(void *start, void *end, extern void adjust_managed_page_count(struct page *page, long count); -extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end); +extern void reserve_bootmem_region(phys_addr_t start, + phys_addr_t end, int nid); /* Free the reserved page into the buddy system, so it gets managed. */ static inline void free_reserved_page(struct page *page) -- cgit v1.2.3 From 982a7194afc9a58ec55ed174c61869c2722bb918 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 21 Jun 2023 17:45:46 +0100 Subject: mm: add __folio_batch_release() This performs the same role as __pagevec_release(), ie skipping the check for batch length of 0. Link: https://lkml.kernel.org/r/20230621164557.3510324-3-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagevec.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index f582f7213ea5..42aad53e382e 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -127,9 +127,15 @@ static inline unsigned folio_batch_add(struct folio_batch *fbatch, return fbatch_space(fbatch); } +static inline void __folio_batch_release(struct folio_batch *fbatch) +{ + __pagevec_release((struct pagevec *)fbatch); +} + static inline void folio_batch_release(struct folio_batch *fbatch) { - pagevec_release((struct pagevec *)fbatch); + if (folio_batch_count(fbatch)) + __folio_batch_release(fbatch); } void folio_batch_remove_exceptionals(struct folio_batch *fbatch); -- cgit v1.2.3 From bdadc6d83156016d2b5eed582c1458c881c53a1e Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 21 Jun 2023 17:45:47 +0100 Subject: scatterlist: add sg_set_folio() This wrapper for sg_set_page() lets drivers add folios to a scatterlist more easily. We could, perhaps, do better by using a different page in the folio if offset is larger than UINT_MAX, but let's hope we get a better data structure than this before we need to care about such large folios. Link: https://lkml.kernel.org/r/20230621164557.3510324-4-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/scatterlist.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'include/linux') diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index ec46d8e8e49d..77df3d7b18a6 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -141,6 +141,30 @@ static inline void sg_set_page(struct scatterlist *sg, struct page *page, sg->length = len; } +/** + * sg_set_folio - Set sg entry to point at given folio + * @sg: SG entry + * @folio: The folio + * @len: Length of data + * @offset: Offset into folio + * + * Description: + * Use this function to set an sg entry pointing at a folio, never assign + * the folio directly. We encode sg table information in the lower bits + * of the folio pointer. See sg_page() for looking up the page belonging + * to an sg entry. + * + **/ +static inline void sg_set_folio(struct scatterlist *sg, struct folio *folio, + size_t len, size_t offset) +{ + WARN_ON_ONCE(len > UINT_MAX); + WARN_ON_ONCE(offset > UINT_MAX); + sg_assign_page(sg, &folio->page); + sg->offset = offset; + sg->length = len; +} + static inline struct page *sg_page(struct scatterlist *sg) { #ifdef CONFIG_DEBUG_SG -- cgit v1.2.3 From e0b72c14d8dcc9477e580c261041dae86d4906fe Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 21 Jun 2023 17:45:50 +0100 Subject: mm: remove check_move_unevictable_pages() All callers have now been converted to call check_move_unevictable_folios(). Link: https://lkml.kernel.org/r/20230621164557.3510324-7-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/swap.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index ce7e82cf787f..456546443f1f 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -439,7 +439,6 @@ static inline bool node_reclaim_enabled(void) } void check_move_unevictable_folios(struct folio_batch *fbatch); -void check_move_unevictable_pages(struct pagevec *pvec); extern void __meminit kswapd_run(int nid); extern void __meminit kswapd_stop(int nid); -- cgit v1.2.3 From ce06442812fc584337c5b23a43bd2be7d037041d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 21 Jun 2023 17:45:51 +0100 Subject: pagevec: rename fbatch_count() This should always have been called folio_batch_count(). Link: https://lkml.kernel.org/r/20230621164557.3510324-8-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagevec.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index 42aad53e382e..3a9d29dd28a3 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -105,7 +105,7 @@ static inline unsigned int folio_batch_count(struct folio_batch *fbatch) return fbatch->nr; } -static inline unsigned int fbatch_space(struct folio_batch *fbatch) +static inline unsigned int folio_batch_space(struct folio_batch *fbatch) { return PAGEVEC_SIZE - fbatch->nr; } @@ -124,7 +124,7 @@ static inline unsigned folio_batch_add(struct folio_batch *fbatch, struct folio *folio) { fbatch->folios[fbatch->nr++] = folio; - return fbatch_space(fbatch); + return folio_batch_space(fbatch); } static inline void __folio_batch_release(struct folio_batch *fbatch) -- cgit v1.2.3 From 76fa88429075667fe76d4905f2f471e0ac3d543c Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 21 Jun 2023 17:45:53 +0100 Subject: net: convert sunrpc from pagevec to folio_batch Remove the last usage of pagevecs. There is a slight change here; we now free the folio_batch as soon as it fills up instead of freeing the folio_batch when we try to add a page to a full batch. This should have no effect in practice. Link: https://lkml.kernel.org/r/20230621164557.3510324-10-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Acked-by: Chuck Lever Signed-off-by: Andrew Morton --- include/linux/sunrpc/svc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 762d7231e574..a3a64fb4053c 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -223,7 +223,7 @@ struct svc_rqst { struct page * *rq_next_page; /* next reply page to use */ struct page * *rq_page_end; /* one past the last page */ - struct pagevec rq_pvec; + struct folio_batch rq_fbatch; struct kvec rq_vec[RPCSVC_MAXPAGES]; /* generally useful.. */ struct bio_vec rq_bvec[RPCSVC_MAXPAGES]; -- cgit v1.2.3 From 1e0877d58b1e22517d8939b22b963c043e6c63fd Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 21 Jun 2023 17:45:54 +0100 Subject: mm: remove struct pagevec All users are now converted to use the folio_batch so we can get rid of this data structure. Link: https://lkml.kernel.org/r/20230621164557.3510324-11-willy@infradead.org Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Andrew Morton --- include/linux/pagevec.h | 63 ++++--------------------------------------------- 1 file changed, 4 insertions(+), 59 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pagevec.h b/include/linux/pagevec.h index 3a9d29dd28a3..87cc678adc85 100644 --- a/include/linux/pagevec.h +++ b/include/linux/pagevec.h @@ -3,65 +3,18 @@ * include/linux/pagevec.h * * In many places it is efficient to batch an operation up against multiple - * pages. A pagevec is a multipage container which is used for that. + * folios. A folio_batch is a container which is used for that. */ #ifndef _LINUX_PAGEVEC_H #define _LINUX_PAGEVEC_H -#include +#include -/* 15 pointers + header align the pagevec structure to a power of two */ +/* 15 pointers + header align the folio_batch structure to a power of two */ #define PAGEVEC_SIZE 15 -struct page; struct folio; -struct address_space; - -/* Layout must match folio_batch */ -struct pagevec { - unsigned char nr; - bool percpu_pvec_drained; - struct page *pages[PAGEVEC_SIZE]; -}; - -void __pagevec_release(struct pagevec *pvec); - -static inline void pagevec_init(struct pagevec *pvec) -{ - pvec->nr = 0; - pvec->percpu_pvec_drained = false; -} - -static inline void pagevec_reinit(struct pagevec *pvec) -{ - pvec->nr = 0; -} - -static inline unsigned pagevec_count(struct pagevec *pvec) -{ - return pvec->nr; -} - -static inline unsigned pagevec_space(struct pagevec *pvec) -{ - return PAGEVEC_SIZE - pvec->nr; -} - -/* - * Add a page to a pagevec. Returns the number of slots still available. - */ -static inline unsigned pagevec_add(struct pagevec *pvec, struct page *page) -{ - pvec->pages[pvec->nr++] = page; - return pagevec_space(pvec); -} - -static inline void pagevec_release(struct pagevec *pvec) -{ - if (pagevec_count(pvec)) - __pagevec_release(pvec); -} /** * struct folio_batch - A collection of folios. @@ -78,11 +31,6 @@ struct folio_batch { struct folio *folios[PAGEVEC_SIZE]; }; -/* Layout must match pagevec */ -static_assert(sizeof(struct pagevec) == sizeof(struct folio_batch)); -static_assert(offsetof(struct pagevec, pages) == - offsetof(struct folio_batch, folios)); - /** * folio_batch_init() - Initialise a batch of folios * @fbatch: The folio batch. @@ -127,10 +75,7 @@ static inline unsigned folio_batch_add(struct folio_batch *fbatch, return folio_batch_space(fbatch); } -static inline void __folio_batch_release(struct folio_batch *fbatch) -{ - __pagevec_release((struct pagevec *)fbatch); -} +void __folio_batch_release(struct folio_batch *pvec); static inline void folio_batch_release(struct folio_batch *fbatch) { -- cgit v1.2.3 From 7302338a14f97eb44cd13f34aab0dc6596f1632c Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Mon, 19 Jun 2023 19:07:18 +0800 Subject: mm: kill [add|del]_page_to_lru_list() Now no one call [add|del]_page_to_lru_list(), let's drop unused page interfaces. Link:https://lkml.kernel.org/r/20230619110718.65679-2-wangkefeng.wang@huawei.com Signed-off-by: Kefeng Wang Acked-by: Yu Zhao Reviewed-by: Baolin Wang Cc: James Gowans Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- include/linux/mm_inline.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 08c2bcefcb2b..21d6c72bcc71 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -323,12 +323,6 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) list_add(&folio->lru, &lruvec->lists[lru]); } -static __always_inline void add_page_to_lru_list(struct page *page, - struct lruvec *lruvec) -{ - lruvec_add_folio(lruvec, page_folio(page)); -} - static __always_inline void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) { @@ -357,12 +351,6 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) -folio_nr_pages(folio)); } -static __always_inline void del_page_from_lru_list(struct page *page, - struct lruvec *lruvec) -{ - lruvec_del_folio(lruvec, page_folio(page)); -} - #ifdef CONFIG_ANON_VMA_NAME /* * mmap_lock should be read-locked when calling anon_vma_name(). Caller should -- cgit v1.2.3 From 1bc545bff45ce9eefc176ccf663074462a209cb6 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Wed, 21 Jun 2023 02:31:01 +0000 Subject: mm/vmscan: fix root proactive reclaim unthrottling unbalanced node When memory.reclaim was introduced, it became the first case where cgroup_reclaim() is true for the root cgroup. Johannes concluded [1] that for most cases this is okay, except for one case. Historically, kswapd would throttle reclaim on a node if a lot of pages marked for reclaim are under writeback (aka the node is congested). This occurred by setting LRUVEC_CONGESTED bit in lruvec->flags. The bit would be cleared when the node is balanced. Similarly, cgroup reclaim would set the same bit when an lruvec is congested, and clear it on the way out of reclaim (to throttle local reclaimers). Before the introduction of memory.reclaim, the root memcg was the only target of kswapd reclaim, and non-root memcgs were the only targets of cgroup reclaim, so they would never interfere. Using the same bit for both was fine. After memory.reclaim, it is possible for cgroup reclaim on the root cgroup to clear the bit set by kswapd. This would result in reclaim on the node to be unthrottled before the node is balanced. Fix this by introducing separate bits for cgroup-level and node-level congestion. kswapd can unthrottle an lruvec that is marked as congested by cgroup reclaim (as the entire node should no longer be congested), but not vice versa (to prevent premature unthrottling before the entire node is balanced). [1]https://lore.kernel.org/lkml/20230405200150.GA35884@cmpxchg.org/ Link: https://lkml.kernel.org/r/20230621023101.432780-1-yosryahmed@google.com Signed-off-by: Yosry Ahmed Reported-by: Johannes Weiner Closes: https://lore.kernel.org/lkml/20230405200150.GA35884@cmpxchg.org/ Cc: Michal Hocko Cc: Roman Gushchin Cc: Shakeel Butt Cc: Muchun Song Cc: Yu Zhao Signed-off-by: Andrew Morton --- include/linux/mmzone.h | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 3e822335f214..d863698a84e0 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -293,9 +293,21 @@ static inline bool is_active_lru(enum lru_list lru) #define ANON_AND_FILE 2 enum lruvec_flags { - LRUVEC_CONGESTED, /* lruvec has many dirty pages - * backed by a congested BDI - */ + /* + * An lruvec has many dirty pages backed by a congested BDI: + * 1. LRUVEC_CGROUP_CONGESTED is set by cgroup-level reclaim. + * It can be cleared by cgroup reclaim or kswapd. + * 2. LRUVEC_NODE_CONGESTED is set by kswapd node-level reclaim. + * It can only be cleared by kswapd. + * + * Essentially, kswapd can unthrottle an lruvec throttled by cgroup + * reclaim, but not vice versa. This only applies to the root cgroup. + * The goal is to prevent cgroup reclaim on the root cgroup (e.g. + * memory.reclaim) to unthrottle an unbalanced node (that was throttled + * by kswapd). + */ + LRUVEC_CGROUP_CONGESTED, + LRUVEC_NODE_CONGESTED, }; #endif /* !__GENERATING_BOUNDS_H */ -- cgit v1.2.3 From acc72d59c7509540c27c49625cb4b5a8db1f1a84 Mon Sep 17 00:00:00 2001 From: Sidhartha Kumar Date: Thu, 22 Jun 2023 22:49:48 -0700 Subject: mm/hugetlb: remove hugetlb_set_page_subpool() All users have been converted to hugetlb_set_folio_subpool() so we can safely remove this function. Link: https://lkml.kernel.org/r/20230623054948.280627-1-sidhartha.kumar@oracle.com Signed-off-by: Sidhartha Kumar Cc: Mike Kravetz Cc: Muchun Song Cc: Tarun Sahu Signed-off-by: Andrew Morton --- include/linux/hugetlb.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index beb7c63d2871..ca3c8e10f24a 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -761,12 +761,6 @@ static inline void hugetlb_set_folio_subpool(struct folio *folio, folio->_hugetlb_subpool = subpool; } -static inline void hugetlb_set_page_subpool(struct page *hpage, - struct hugepage_subpool *subpool) -{ - hugetlb_set_folio_subpool(page_folio(hpage), subpool); -} - static inline struct hstate *hstate_file(struct file *f) { return hstate_inode(file_inode(f)); -- cgit v1.2.3 From c2508ec5a58db67093f4fb8bf89a9a7c53a109e9 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 15 Jun 2023 15:17:36 -0700 Subject: mm: introduce new 'lock_mm_and_find_vma()' page fault helper .. and make x86 use it. This basically extracts the existing x86 "find and expand faulting vma" code, but extends it to also take the mmap lock for writing in case we actually do need to expand the vma. We've historically short-circuited that case, and have some rather ugly special logic to serialize the stack segment expansion (since we only hold the mmap lock for reading) that doesn't match the normal VM locking. That slight violation of locking worked well, right up until it didn't: the maple tree code really does want proper locking even for simple extension of an existing vma. So extract the code for "look up the vma of the fault" from x86, fix it up to do the necessary write locking, and make it available as a helper function for other architectures that can use the common helper. Note: I say "common helper", but it really only handles the normal stack-grows-down case. Which is all architectures except for PA-RISC and IA64. So some rare architectures can't use the helper, but if they care they'll just need to open-code this logic. It's also worth pointing out that this code really would like to have an optimistic "mmap_upgrade_trylock()" to make it quicker to go from a read-lock (for the common case) to taking the write lock (for having to extend the vma) in the normal single-threaded situation where there is no other locking activity. But that _is_ all the very uncommon special case, so while it would be nice to have such an operation, it probably doesn't matter in reality. I did put in the skeleton code for such a possible future expansion, even if it only acts as pseudo-documentation for what we're doing. Signed-off-by: Linus Torvalds --- include/linux/mm.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 27ce77080c79..570cf906fbcc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2325,6 +2325,8 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t nr, bool even_cows); void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows); +struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, + unsigned long address, struct pt_regs *regs); #else static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, unsigned int flags, -- cgit v1.2.3 From f440fa1ac955e2898893f9301568435eb5cdfc4b Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Fri, 16 Jun 2023 15:58:54 -0700 Subject: mm: make find_extend_vma() fail if write lock not held Make calls to extend_vma() and find_extend_vma() fail if the write lock is required. To avoid making this a flag-day event, this still allows the old read-locking case for the trivial situations, and passes in a flag to say "is it write-locked". That way write-lockers can say "yes, I'm being careful", and legacy users will continue to work in all the common cases until they have been fully converted to the new world order. Co-Developed-by: Matthew Wilcox (Oracle) Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Liam R. Howlett Signed-off-by: Linus Torvalds --- include/linux/mm.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 570cf906fbcc..01a016521b60 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3192,11 +3192,13 @@ extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); extern unsigned long stack_guard_gap; /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ -extern int expand_stack(struct vm_area_struct *vma, unsigned long address); +int expand_stack_locked(struct vm_area_struct *vma, unsigned long address, + bool write_locked); +#define expand_stack(vma,addr) expand_stack_locked(vma,addr,false) /* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */ -extern int expand_downwards(struct vm_area_struct *vma, - unsigned long address); +int expand_downwards(struct vm_area_struct *vma, unsigned long address, + bool write_locked); #if VM_GROWSUP extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); #else @@ -3297,6 +3299,8 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, #endif struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); +struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, + unsigned long addr, bool write_locked); int remap_pfn_range(struct vm_area_struct *, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t); int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, -- cgit v1.2.3 From dc97391e661009eab46783030d2404c9b6e6f2e7 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 23 Jun 2023 23:55:12 +0100 Subject: sock: Remove ->sendpage*() in favour of sendmsg(MSG_SPLICE_PAGES) Remove ->sendpage() and ->sendpage_locked(). sendmsg() with MSG_SPLICE_PAGES should be used instead. This allows multiple pages and multipage folios to be passed through. Signed-off-by: David Howells Acked-by: Marc Kleine-Budde # for net/can cc: Jens Axboe cc: Matthew Wilcox cc: linux-afs@lists.infradead.org cc: mptcp@lists.linux.dev cc: rds-devel@oss.oracle.com cc: tipc-discussion@lists.sourceforge.net cc: virtualization@lists.linux-foundation.org Link: https://lore.kernel.org/r/20230623225513.2732256-16-dhowells@redhat.com Signed-off-by: Jakub Kicinski --- include/linux/net.h | 8 -------- 1 file changed, 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/net.h b/include/linux/net.h index 23324e9a2b3d..41c608c1b02c 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -207,8 +207,6 @@ struct proto_ops { size_t total_len, int flags); int (*mmap) (struct file *file, struct socket *sock, struct vm_area_struct * vma); - ssize_t (*sendpage) (struct socket *sock, struct page *page, - int offset, size_t size, int flags); ssize_t (*splice_read)(struct socket *sock, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); void (*splice_eof)(struct socket *sock); @@ -222,8 +220,6 @@ struct proto_ops { sk_read_actor_t recv_actor); /* This is different from read_sock(), it reads an entire skb at a time. */ int (*read_skb)(struct sock *sk, skb_read_actor_t recv_actor); - int (*sendpage_locked)(struct sock *sk, struct page *page, - int offset, size_t size, int flags); int (*sendmsg_locked)(struct sock *sk, struct msghdr *msg, size_t size); int (*set_rcvlowat)(struct sock *sk, int val); @@ -341,10 +337,6 @@ int kernel_connect(struct socket *sock, struct sockaddr *addr, int addrlen, int flags); int kernel_getsockname(struct socket *sock, struct sockaddr *addr); int kernel_getpeername(struct socket *sock, struct sockaddr *addr); -int kernel_sendpage(struct socket *sock, struct page *page, int offset, - size_t size, int flags); -int kernel_sendpage_locked(struct sock *sk, struct page *page, int offset, - size_t size, int flags); int kernel_sock_shutdown(struct socket *sock, enum sock_shutdown_cmd how); /* Routine returns the IP overhead imposed by a (caller-protected) socket. */ -- cgit v1.2.3 From b848b26c6672c9b977890ba85f5a155e5eb221f0 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 23 Jun 2023 23:55:13 +0100 Subject: net: Kill MSG_SENDPAGE_NOTLAST Now that ->sendpage() has been removed, MSG_SENDPAGE_NOTLAST can be cleaned up. Things were converted to use MSG_MORE instead, but the protocol sendpage stubs still convert MSG_SENDPAGE_NOTLAST to MSG_MORE, which is now unnecessary. Signed-off-by: David Howells cc: Jens Axboe cc: Matthew Wilcox cc: linux-afs@lists.infradead.org cc: mptcp@lists.linux.dev cc: rds-devel@oss.oracle.com cc: tipc-discussion@lists.sourceforge.net cc: virtualization@lists.linux-foundation.org Link: https://lore.kernel.org/r/20230623225513.2732256-17-dhowells@redhat.com Signed-off-by: Jakub Kicinski --- include/linux/socket.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/socket.h b/include/linux/socket.h index 58204700018a..39b74d83c7c4 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -319,7 +319,6 @@ struct ucred { #define MSG_MORE 0x8000 /* Sender will send more */ #define MSG_WAITFORONE 0x10000 /* recvmmsg(): block until 1+ packets avail */ #define MSG_SENDPAGE_NOPOLICY 0x10000 /* sendpage() internal : do no apply policy */ -#define MSG_SENDPAGE_NOTLAST 0x20000 /* sendpage() internal : not the last page */ #define MSG_BATCH 0x40000 /* sendmmsg(): more messages coming */ #define MSG_EOF MSG_FIN #define MSG_NO_SHARED_FRAGS 0x80000 /* sendpage() internal : page frags are not shared */ @@ -341,8 +340,7 @@ struct ucred { /* Flags to be cleared on entry by sendmsg and sendmmsg syscalls */ #define MSG_INTERNAL_SENDMSG_FLAGS \ - (MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_NOTLAST | \ - MSG_SENDPAGE_DECRYPTED) + (MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_DECRYPTED) /* Setsockoptions(2) level. Thanks to BSD these must match IPPROTO_xxx */ #define SOL_IP 0 -- cgit v1.2.3 From 9338c2233b97f97aa68bc42f53b06e90def129d7 Mon Sep 17 00:00:00 2001 From: Ross Lagerwall Date: Mon, 5 Jun 2023 11:28:40 +0100 Subject: iscsi_ibft: Fix finding the iBFT under Xen Dom 0 To facilitate diskless iSCSI boot, the firmware can place a table of configuration details in memory called the iBFT. The presence of this table is not specified, nor is the precise location (and it's not in the E820) so the kernel has to search for a magic marker to find it. When running under Xen, Dom 0 does not have access to the entire host's memory, only certain regions which are identity-mapped which means that the pseudo-physical address in Dom0 == real host physical address. Add the iBFT search bounds as a reserved region which causes it to be identity-mapped in xen_set_identity_and_remap_chunk() which allows Dom0 access to the specific physical memory to correctly search for the iBFT magic marker (and later access the full table). This necessitates moving the call to reserve_ibft_region() somewhat later so that it is called after e820__memory_setup() which is when the Xen identity mapping adjustments are applied. The precise location of the call is not too important so I've put it alongside dmi_setup() which does similar scanning of memory for configuration tables. Finally in the iBFT find code, instead of using isa_bus_to_virt() which doesn't do the right thing under Xen, use early_memremap() like the dmi_setup() code does. The result of these changes is that it is possible to boot a diskless Xen + Dom0 running off an iSCSI disk whereas previously it would fail to find the iBFT and consequently, the iSCSI root disk. Signed-off-by: Ross Lagerwall Reviewed-by: Juergen Gross Acked-by: Konrad Rzeszutek Wilk Acked-by: Dave Hansen # for x86 Link: https://lore.kernel.org/r/20230605102840.1521549-1-ross.lagerwall@citrix.com Signed-off-by: Juergen Gross --- include/linux/iscsi_ibft.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iscsi_ibft.h b/include/linux/iscsi_ibft.h index 790e7fcfc1a6..e2742748104d 100644 --- a/include/linux/iscsi_ibft.h +++ b/include/linux/iscsi_ibft.h @@ -21,12 +21,20 @@ */ extern phys_addr_t ibft_phys_addr; +#ifdef CONFIG_ISCSI_IBFT_FIND + /* * Routine used to find and reserve the iSCSI Boot Format Table. The * physical address is set in the ibft_phys_addr variable. */ -#ifdef CONFIG_ISCSI_IBFT_FIND void reserve_ibft_region(void); + +/* + * Physical bounds to search for the iSCSI Boot Format Table. + */ +#define IBFT_START 0x80000 /* 512kB */ +#define IBFT_END 0x100000 /* 1MB */ + #else static inline void reserve_ibft_region(void) {} #endif -- cgit v1.2.3 From f18e7122cc73d9218930156fa38f050a2e37de57 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 22 Jun 2023 23:11:01 -0700 Subject: linux/netfilter.h: fix kernel-doc warnings kernel-doc does not support DECLARE_PER_CPU(), so don't mark it with kernel-doc notation. One comment block is not kernel-doc notation, so just use "/*" to begin the comment. Quietens these warnings: netfilter.h:493: warning: Function parameter or member 'bool' not described in 'DECLARE_PER_CPU' netfilter.h:493: warning: Function parameter or member 'nf_skb_duplicated' not described in 'DECLARE_PER_CPU' netfilter.h:493: warning: expecting prototype for nf_skb_duplicated(). Prototype was for DECLARE_PER_CPU() instead netfilter.h:496: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Contains bitmask of ctnetlink event subscribers, if any. Fixes: e7c8899f3e6f ("netfilter: move tee_active to core") Fixes: fdf6491193e4 ("netfilter: ctnetlink: make event listener tracking global") Signed-off-by: Randy Dunlap Reviewed-by: Simon Horman Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index 0762444e3767..d4fed4c508ca 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -481,7 +481,7 @@ struct nfnl_ct_hook { }; extern const struct nfnl_ct_hook __rcu *nfnl_ct_hook; -/** +/* * nf_skb_duplicated - TEE target has sent a packet * * When a xtables target sends a packet, the OUTPUT and POSTROUTING @@ -492,7 +492,7 @@ extern const struct nfnl_ct_hook __rcu *nfnl_ct_hook; */ DECLARE_PER_CPU(bool, nf_skb_duplicated); -/** +/* * Contains bitmask of ctnetlink event subscribers, if any. * Can't be pernet due to NETLINK_LISTEN_ALL_NSID setsockopt flag. */ -- cgit v1.2.3 From 5c5bd1fef3ec913f9c597c6f61a9b903096415bf Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Wed, 15 Mar 2023 09:31:25 +0800 Subject: jbd2: remove unused feature macros JBD2_HAS_[IN|RO_]COMPAT_FEATURE macros are no longer used, just remove them. Signed-off-by: Zhang Yi Signed-off-by: Zhihao Cheng Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230315013128.3911115-4-chengzhihao1@huawei.com Signed-off-by: Theodore Ts'o --- include/linux/jbd2.h | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index f619bae1dcc5..a91cf9c7a94b 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -274,17 +274,6 @@ typedef struct journal_superblock_s /* 0x0400 */ } journal_superblock_t; -/* Use the jbd2_{has,set,clear}_feature_* helpers; these will be removed */ -#define JBD2_HAS_COMPAT_FEATURE(j,mask) \ - ((j)->j_format_version >= 2 && \ - ((j)->j_superblock->s_feature_compat & cpu_to_be32((mask)))) -#define JBD2_HAS_RO_COMPAT_FEATURE(j,mask) \ - ((j)->j_format_version >= 2 && \ - ((j)->j_superblock->s_feature_ro_compat & cpu_to_be32((mask)))) -#define JBD2_HAS_INCOMPAT_FEATURE(j,mask) \ - ((j)->j_format_version >= 2 && \ - ((j)->j_superblock->s_feature_incompat & cpu_to_be32((mask)))) - #define JBD2_FEATURE_COMPAT_CHECKSUM 0x00000001 #define JBD2_FEATURE_INCOMPAT_REVOKE 0x00000001 -- cgit v1.2.3 From 5cf036d4f1489d7ba04b948e415f662521902c30 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Wed, 15 Mar 2023 09:31:26 +0800 Subject: jbd2: switch to check format version in superblock directly We should only check and set extented features if journal format version is 2, and now we check the in memory copy of the superblock 'journal->j_format_version', which relys on the parameter initialization sequence, switch to use the h_blocktype in superblock cloud be more clear. Signed-off-by: Zhang Yi Signed-off-by: Zhihao Cheng Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230315013128.3911115-5-chengzhihao1@huawei.com Signed-off-by: Theodore Ts'o --- include/linux/jbd2.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index a91cf9c7a94b..1ffcea5c024e 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -1313,11 +1313,22 @@ struct journal_s rwsem_release(&j->j_trans_commit_map, _THIS_IP_); \ } while (0) +/* + * We can support any known requested features iff the + * superblock is not in version 1. Otherwise we fail to support any + * extended sb features. + */ +static inline bool jbd2_format_support_feature(journal_t *j) +{ + return j->j_superblock->s_header.h_blocktype != + cpu_to_be32(JBD2_SUPERBLOCK_V1); +} + /* journal feature predicate functions */ #define JBD2_FEATURE_COMPAT_FUNCS(name, flagname) \ static inline bool jbd2_has_feature_##name(journal_t *j) \ { \ - return ((j)->j_format_version >= 2 && \ + return (jbd2_format_support_feature(j) && \ ((j)->j_superblock->s_feature_compat & \ cpu_to_be32(JBD2_FEATURE_COMPAT_##flagname)) != 0); \ } \ @@ -1335,7 +1346,7 @@ static inline void jbd2_clear_feature_##name(journal_t *j) \ #define JBD2_FEATURE_RO_COMPAT_FUNCS(name, flagname) \ static inline bool jbd2_has_feature_##name(journal_t *j) \ { \ - return ((j)->j_format_version >= 2 && \ + return (jbd2_format_support_feature(j) && \ ((j)->j_superblock->s_feature_ro_compat & \ cpu_to_be32(JBD2_FEATURE_RO_COMPAT_##flagname)) != 0); \ } \ @@ -1353,7 +1364,7 @@ static inline void jbd2_clear_feature_##name(journal_t *j) \ #define JBD2_FEATURE_INCOMPAT_FUNCS(name, flagname) \ static inline bool jbd2_has_feature_##name(journal_t *j) \ { \ - return ((j)->j_format_version >= 2 && \ + return (jbd2_format_support_feature(j) && \ ((j)->j_superblock->s_feature_incompat & \ cpu_to_be32(JBD2_FEATURE_INCOMPAT_##flagname)) != 0); \ } \ -- cgit v1.2.3 From 04c2e98179658d223665661f12c5043224e8f8d3 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Wed, 15 Mar 2023 09:31:28 +0800 Subject: jbd2: remove j_format_version journal->j_format_version is no longer used, remove it. Signed-off-by: Zhang Yi Signed-off-by: Zhihao Cheng Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230315013128.3911115-7-chengzhihao1@huawei.com Signed-off-by: Theodore Ts'o --- include/linux/jbd2.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 1ffcea5c024e..6990fc891612 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -792,11 +792,6 @@ struct journal_s */ journal_superblock_t *j_superblock; - /** - * @j_format_version: Version of the superblock format. - */ - int j_format_version; - /** * @j_state_lock: Protect the various scalars in the journal. */ -- cgit v1.2.3 From c7fc60555864c0e67f5e5754a9053986f8fb8491 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Wed, 22 Mar 2023 09:33:51 +0800 Subject: jbd2: continue to record log between each mount For a newly mounted file system, the journal committing thread always record new transactions from the start of the journal area, no matter whether the journal was clean or just has been recovered. So the logdump code in debugfs cannot dump continuous logs between each mount, it is disadvantageous to analysis corrupted file system image and locate the file system inconsistency bugs. If we get a corrupted file system in the running products and want to find out what has happened, besides lookup the system log, one effective way is to backtrack the journal log. But we may not always run e2fsck before each mount and the default fsck -a mode also cannot always checkout all inconsistencies, so it could left over some inconsistencies into the next mount until we detect it. Finally, transactions in the journal may probably discontinuous and some relatively new transactions has been covered, it becomes hard to analyse. If we could record transactions continuously between each mount, we could acquire more useful info from the journal. Like this: |Previous mount checkpointed/recovered logs|Current mount logs | |{------}{---}{--------} ... {------}| ... |{======}{========}...000000| And yes the journal area is limited and cannot record everything, the problematic transaction may also be covered even if we do this, but this is still useful for fuzzy tests and short-running products. This patch save the head blocknr in the superblock after flushing the journal or unmounting the file system, let the next mount could continue to record new transaction behind it. This change is backward compatible because the old kernel does not care about the head blocknr of the journal. It is also fine if we mount a clean old image without valid head blocknr, we fail back to set it to s_first just like before. Finally, for the case of mount an unclean file system, we could also get the journal head easily after scanning/replaying the journal, it will continue to record new transaction after the recovered transactions. Signed-off-by: Zhang Yi Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20230322013353.1843306-2-yi.zhang@huaweicloud.com Signed-off-by: Theodore Ts'o --- include/linux/jbd2.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 6990fc891612..d860499e15e4 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -265,8 +265,10 @@ typedef struct journal_superblock_s __u8 s_padding2[3]; /* 0x0054 */ __be32 s_num_fc_blks; /* Number of fast commit blocks */ -/* 0x0058 */ - __u32 s_padding[41]; + __be32 s_head; /* blocknr of head of log, only uptodate + * while the filesystem is clean */ +/* 0x005C */ + __u32 s_padding[40]; __be32 s_checksum; /* crc32c(superblock) */ /* 0x0100 */ @@ -1395,6 +1397,9 @@ JBD2_FEATURE_INCOMPAT_FUNCS(fast_commit, FAST_COMMIT) #define JBD2_ABORT_ON_SYNCDATA_ERR 0x040 /* Abort the journal on file * data write error in ordered * mode */ +#define JBD2_CYCLE_RECORD 0x080 /* Journal cycled record log on + * clean and empty filesystem + * logging area */ #define JBD2_FAST_COMMIT_ONGOING 0x100 /* Fast commit is ongoing */ #define JBD2_FULL_COMMIT_ONGOING 0x200 /* Full commit is ongoing */ #define JBD2_JOURNAL_FLUSH_DISCARD 0x0001 -- cgit v1.2.3 From 1a3f6fc430ed220889c7fb1a63bc2a30267ebc2a Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 26 Jun 2023 14:46:40 -0700 Subject: phylink: ReST-ify the phylink_pcs_neg_mode() kdoc Stephen reports warnings when rendering phylink kdocs as HTML: include/linux/phylink.h:110: ERROR: Unexpected indentation. include/linux/phylink.h:111: WARNING: Block quote ends without a blank line; unexpected unindent. include/linux/phylink.h:614: WARNING: Inline literal start-string without end-string. include/linux/phylink.h:644: WARNING: Inline literal start-string without end-string. Make phylink_pcs_neg_mode() use a proper list format to fix the first two warnings. The last two warnings, AFAICT, come from the use of shorthand like phylink_mode_*(). Perhaps those should be special-cased at the Sphinx level. Reported-by: Stephen Rothwell Link: https://lore.kernel.org/all/20230626162908.2f149f98@canb.auug.org.au/ Link: https://lore.kernel.org/r/20230626214640.3142252-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- include/linux/phylink.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 516240f1e950..1817940a3418 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -105,11 +105,13 @@ static inline bool phylink_autoneg_inband(unsigned int mode) * * Determines the negotiation mode to be used by the PCS, and returns * one of: - * %PHYLINK_PCS_NEG_NONE: interface mode does not support inband - * %PHYLINK_PCS_NEG_OUTBAND: an out of band mode (e.g. reading the PHY) + * + * - %PHYLINK_PCS_NEG_NONE: interface mode does not support inband + * - %PHYLINK_PCS_NEG_OUTBAND: an out of band mode (e.g. reading the PHY) * will be used. - * %PHYLINK_PCS_NEG_INBAND_DISABLED: inband mode selected but autoneg disabled - * %PHYLINK_PCS_NEG_INBAND_ENABLED: inband mode selected and autoneg enabled + * - %PHYLINK_PCS_NEG_INBAND_DISABLED: inband mode selected but autoneg + * disabled + * - %PHYLINK_PCS_NEG_INBAND_ENABLED: inband mode selected and autoneg enabled * * Note: this is for cases where the PCS itself is involved in negotiation * (e.g. Clause 37, SGMII and similar) not Clause 73. -- cgit v1.2.3 From 8d7071af890768438c14db6172cc8f9f4d04e184 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sat, 24 Jun 2023 13:45:51 -0700 Subject: mm: always expand the stack with the mmap write lock held This finishes the job of always holding the mmap write lock when extending the user stack vma, and removes the 'write_locked' argument from the vm helper functions again. For some cases, we just avoid expanding the stack at all: drivers and page pinning really shouldn't be extending any stacks. Let's see if any strange users really wanted that. It's worth noting that architectures that weren't converted to the new lock_mm_and_find_vma() helper function are left using the legacy "expand_stack()" function, but it has been changed to drop the mmap_lock and take it for writing while expanding the vma. This makes it fairly straightforward to convert the remaining architectures. As a result of dropping and re-taking the lock, the calling conventions for this function have also changed, since the old vma may no longer be valid. So it will now return the new vma if successful, and NULL - and the lock dropped - if the area could not be extended. Tested-by: Vegard Nossum Tested-by: John Paul Adrian Glaubitz # ia64 Tested-by: Frank Scheiner # ia64 Signed-off-by: Linus Torvalds --- include/linux/mm.h | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 01a016521b60..4a9533efbd5d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3192,18 +3192,11 @@ extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); extern unsigned long stack_guard_gap; /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */ -int expand_stack_locked(struct vm_area_struct *vma, unsigned long address, - bool write_locked); -#define expand_stack(vma,addr) expand_stack_locked(vma,addr,false) +int expand_stack_locked(struct vm_area_struct *vma, unsigned long address); +struct vm_area_struct *expand_stack(struct mm_struct * mm, unsigned long addr); /* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */ -int expand_downwards(struct vm_area_struct *vma, unsigned long address, - bool write_locked); -#if VM_GROWSUP -extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); -#else - #define expand_upwards(vma, address) (0) -#endif +int expand_downwards(struct vm_area_struct *vma, unsigned long address); /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); @@ -3298,9 +3291,8 @@ unsigned long change_prot_numa(struct vm_area_struct *vma, unsigned long start, unsigned long end); #endif -struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, - unsigned long addr, bool write_locked); + unsigned long addr); int remap_pfn_range(struct vm_area_struct *, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t); int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, -- cgit v1.2.3 From 1b2c92a1cb2469d8c0079dbf496ab86e22e1cb7c Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Wed, 28 Jun 2023 12:47:30 -0700 Subject: x86/mem_encrypt: Remove stale mem_encrypt_init() declaration The memory encryption initialization logic was moved from init/main.c into arch_cpu_finalize_init() in commit 439e17576eb4 ("init, x86: Move mem_encrypt_init() into arch_cpu_finalize_init()"), but a stale declaration for the init function was left in . And didn't cause any problems if you had X86_MEM_ENCRYPT enabled, which apparently everybody involved did have. See also commit 0a9567ac5e6a ("x86/mem_encrypt: Unbreak the AMD_MEM_ENCRYPT=n build") in this whole sad saga of conflicting declarations for different situations. Reported-by: Matthew Wilcox Fixes: 439e17576eb4 init, x86: Move mem_encrypt_init() into arch_cpu_finalize_init() Cc: Thomas Gleixner Signed-off-by: Linus Torvalds --- include/linux/init.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/init.h b/include/linux/init.h index 1200fa99e848..266c3e1640d4 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -155,7 +155,6 @@ void __init init_rootfs(void); void init_IRQ(void); void time_init(void); -void mem_encrypt_init(void); void poking_init(void); void pgtable_cache_init(void); -- cgit v1.2.3