diff options
Diffstat (limited to 'lib')
110 files changed, 6315 insertions, 1784 deletions
diff --git a/lib/Kconfig b/lib/Kconfig index 55748b68714e..4e6b34c3346d 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -159,12 +159,6 @@ config AUDIT_COMPAT_GENERIC depends on AUDIT_GENERIC && AUDIT_ARCH_COMPAT_GENERIC && COMPAT default y -config RANDOM32_SELFTEST - bool "PRNG perform self test on init" - help - This option enables the 32 bit PRNG library functions to perform a - self test on initialization. - # # compression support is select'ed if needed # diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1244dcac2294..134b15a44625 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -613,21 +613,6 @@ config BUILTIN_MODULE_RANGES It also records an anchor symbol to determine the load address of the section. -config DEBUG_FORCE_WEAK_PER_CPU - bool "Force weak per-cpu definitions" - depends on DEBUG_KERNEL - help - s390 and alpha require percpu variables in modules to be - defined weak to work around addressing range issue which - puts the following two restrictions on percpu variable - definitions. - - 1. percpu symbols must be unique whether static or not - 2. percpu variables can't be defined inside a function - - To ensure that generic code follows the above rules, this - option forces all percpu variables to be defined as weak. - config WARN_CONTEXT_ANALYSIS bool "Compiler context-analysis warnings" depends on CC_IS_CLANG && CLANG_VERSION >= 230000 @@ -1048,34 +1033,6 @@ config CODE_TAGGING bool select KALLSYMS -config MEM_ALLOC_PROFILING - bool "Enable memory allocation profiling" - default n - depends on MMU - depends on PROC_FS - depends on !DEBUG_FORCE_WEAK_PER_CPU - select CODE_TAGGING - select PAGE_EXTENSION - select SLAB_OBJ_EXT - help - Track allocation source code and record total allocation size - initiated at that code location. The mechanism can be used to track - memory leaks with a low performance and memory impact. - -config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - bool "Enable memory allocation profiling by default" - default y - depends on MEM_ALLOC_PROFILING - -config MEM_ALLOC_PROFILING_DEBUG - bool "Memory allocation profiler debugging" - default n - depends on MEM_ALLOC_PROFILING - select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - help - Adds warnings with helpful error messages for memory allocation - profiling. - source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" source "lib/Kconfig.kmsan" @@ -1320,10 +1277,10 @@ config DETECT_HUNG_TASK_BLOCKER depends on !PREEMPT_RT default y help - Say Y here to show the blocker task's stacktrace who acquires - the mutex lock which "hung tasks" are waiting. - This will add overhead a bit but shows suspicious tasks and - call trace if it comes from waiting a mutex. + Say Y here to show a possible blocker task's stacktrace when + a hung task is waiting on a mutex, semaphore, or rwsem. + This will add a bit of overhead but shows suspicious tasks and + call traces for waits on one of these locks. config WQ_WATCHDOG bool "Detect Workqueue Stalls" @@ -2247,10 +2204,11 @@ config KCOV_INSTRUMENT_ALL config KCOV_IRQ_AREA_SIZE hex "Size of interrupt coverage collection area in words" depends on KCOV + range 0x80 0x1000000 default 0x40000 help - KCOV uses preallocated per-cpu areas to collect coverage from - soft interrupts. This specifies the size of those areas in the + KCOV uses preallocated areas to collect coverage from soft + interrupts. This specifies the size of those areas in the number of unsigned long words. config KCOV_SELFTEST @@ -2683,6 +2641,19 @@ config FIND_BIT_BENCHMARK If unsure, say N. +config REGION_ALLOC_BENCHMARK + tristate "Benchmark bitmap, IDA and Maple Tree region allocation" + help + This builds a microbenchmark comparing variable-sized region + allocation using bitmaps, IDA and Maple Tree. The benchmark + runs at initialization time. + + Usage: + insmod region_alloc_benchmark.ko + insmod region_alloc_benchmark.ko capacities=1024,2048,4096,65536 + + If unsure, say N. + config FIND_BIT_BENCHMARK_RUST tristate "Test find_bit functions in Rust" depends on RUST @@ -2710,7 +2681,7 @@ config TEST_FIRMWARE config TEST_SYSCTL tristate "sysctl test driver" - depends on PROC_SYSCTL + depends on SYSCTL help This builds the "test_sysctl" module. This driver enables to test the proc sysctl interfaces available to drivers safely without affecting @@ -3504,6 +3475,23 @@ config GCD_KUNIT_TEST If unsure, say N +config POLYNOMIAL_KUNIT_TEST + tristate "Polynomial calculation (polynomial_calc) test" if !KUNIT_ALL_TESTS + depends on KUNIT + select POLYNOMIAL + default KUNIT_ALL_TESTS + help + This option enables the KUnit test suite for the polynomial_calc() + function, which evaluates integer polynomials using factor + redistribution to avoid overflow. + + The test suite verifies correctness for constant, linear, and + quadratic polynomials, negative coefficients, per-step dividers, + divider_leftover, total_divider scaling, and a real sensor + N-to-temperature conversion polynomial. + + If unsure, say N + config PRIME_NUMBERS_KUNIT_TEST tristate "Prime number generator test" if !KUNIT_ALL_TESTS depends on KUNIT @@ -3531,6 +3519,15 @@ config GLOB_KUNIT_TEST If unsure, say N +config PRANDOM_KUNIT_TEST + tristate "KUnit test for prandom" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Enable this option to test the prandom functions at runtime. + + If unsure, say N + endif # RUNTIME_TESTING_MENU config ARCH_USE_MEMTEST diff --git a/lib/Makefile b/lib/Makefile index 7f75cc6edf94..dfab958327c5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -64,6 +64,7 @@ obj-y += hexdump.o obj-$(CONFIG_TEST_HEXDUMP) += test_hexdump.o obj-y += kstrtox.o obj-$(CONFIG_FIND_BIT_BENCHMARK) += find_bit_benchmark.o +obj-$(CONFIG_REGION_ALLOC_BENCHMARK) += region_alloc_benchmark.o obj-$(CONFIG_FIND_BIT_BENCHMARK_RUST) += find_bit_benchmark_rust.o obj-$(CONFIG_TEST_BPF) += test_bpf.o test_dhry-objs := dhry_1.o dhry_2.o dhry_run.o @@ -196,7 +197,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_CODE_TAGGING) += codetag.o -obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o lib-$(CONFIG_GENERIC_BUG) += bug.o @@ -273,6 +273,22 @@ filechk_defbconf = cat $(or $(real-prereqs), /dev/null) $(obj)/default.bconf: $(CONFIG_BOOT_CONFIG_EMBED_FILE) FORCE $(call filechk,defbconf) +obj-$(CONFIG_CMDLINE_FROM_BOOTCONFIG) += embedded-cmdline.o +$(obj)/embedded-cmdline.o: $(obj)/embedded_cmdline.bin + +# Render the bootconfig "kernel" subtree to a flat cmdline string using +# the userspace tools/bootconfig parser (-C mode). The runtime prepend +# helper enforces COMMAND_LINE_SIZE at boot, so no build-time size +# check is performed here (COMMAND_LINE_SIZE is an arch header +# constant, not a Kconfig value). +quiet_cmd_render_cmdline = BCONF2C $@ + cmd_render_cmdline = \ + $(objtree)/tools/bootconfig/bootconfig -C $< > $@ + +targets += embedded_cmdline.bin +$(obj)/embedded_cmdline.bin: $(obj)/default.bconf $(objtree)/tools/bootconfig/bootconfig FORCE + $(call if_changed,render_cmdline) + obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index d9be1cf5187d..e5b218176c5a 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -26,6 +26,15 @@ static bool mem_profiling_support = true; static bool mem_profiling_support; #endif +/* + * Memory allocation profiling is permanently disabled and cannot be enabled. + * Must be called after setup_early_mem_profiling(). + */ +bool mem_alloc_profiling_permanently_disabled(void) +{ + return !mem_profiling_support; +} + static struct codetag_type *alloc_tag_cttype; #ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU diff --git a/lib/assoc_array.c b/lib/assoc_array.c index bcc6e0a013eb..b6c9723e12ce 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -255,7 +255,8 @@ follow_shortcut: sc_segments = shortcut->index_key[sc_level >> ASSOC_ARRAY_KEY_CHUNK_SHIFT]; dissimilarity = segments ^ sc_segments; - if (round_up(sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE) > shortcut->skip_to_level) { + if (shortcut->skip_to_level < round_down(sc_level, + ASSOC_ARRAY_KEY_CHUNK_SIZE) + ASSOC_ARRAY_KEY_CHUNK_SIZE) { /* Trim segments that are beyond the shortcut */ int shift = shortcut->skip_to_level & ASSOC_ARRAY_KEY_CHUNK_MASK; dissimilarity &= ~(ULONG_MAX << shift); diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c index 26d36c938c6a..dd9aa0635fa5 100644 --- a/lib/bitmap-str.c +++ b/lib/bitmap-str.c @@ -75,8 +75,7 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * @off: in the string from which we are copying, We copy to @buf * @count: the maximum number of bytes to print * - * The sprintf("%*pb[l]") is used indirectly via its cpumap wrapper - * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal + * The sprintf("%*pb[l]") format is used by drivers to export hexadecimal * bitmask and decimal list to userspace by sysfs ABI. * Drivers might be using a normal attribute for this kind of ABIs. A * normal attribute typically has show entry as below:: @@ -115,9 +114,9 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * parameters such as off, count from bin_attribute show entry to this API. * * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() - * is similar with cpumap_print_to_pagebuf(), the difference is that - * scnprintf("%*pb[l]") mainly serves sysfs attribute with the assumption - * the destination buffer is exactly one page and won't be more than one page. + * is similar to direct sysfs_emit("%*pb[l]") formatting, but the latter + * assumes the destination buffer is exactly one page and won't be more than + * one page. * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other * hand, mainly serves bin_attribute which doesn't work with exact one page, * and it can break the size limit of converted decimal list and hexadecimal diff --git a/lib/bitmap.c b/lib/bitmap.c index b9bfa157e095..ed685127a107 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -424,6 +424,9 @@ EXPORT_SYMBOL(__bitmap_clear); * The @align_mask should be one less than a power of 2; the effect is that * the bit offset of all zero areas this function finds plus @align_offset * is multiple of that power of 2. + * + * Return: The bit offset of the found area or a value greater than or equal + * to @size if no area is found. */ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, unsigned long size, @@ -432,22 +435,23 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, unsigned long align_mask, unsigned long align_offset) { - unsigned long index, end, i; -again: - index = find_next_zero_bit(map, size, start); + unsigned long end, i, off; + + for_each_clear_bit_from(start, map, size) { + start = __ALIGN_MASK(start + align_offset, align_mask) - align_offset; + end = start + nr; + if (end > size) + break; - /* Align allocation */ - index = __ALIGN_MASK(index + align_offset, align_mask) - align_offset; + off = round_down(start, BITS_PER_LONG); + i = find_last_bit(map + start / BITS_PER_LONG, end - off) + off; + if (i >= end || i < start) + return start; - end = index + nr; - if (end > size) - return end; - i = find_next_bit(map, end, index); - if (i < end) { - start = i + 1; - goto again; + start = i; } - return index; + + return size; } EXPORT_SYMBOL(bitmap_find_next_zero_area_off); diff --git a/lib/bootconfig.c b/lib/bootconfig.c index f445b7703fdd..89c88e359179 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -19,9 +19,13 @@ #include <linux/errno.h> #include <linux/cache.h> #include <linux/compiler.h> +#include <linux/init.h> +#include <linux/moduleparam.h> +#include <linux/printk.h> #include <linux/sprintf.h> #include <linux/memblock.h> #include <linux/string.h> +#include <asm/setup.h> /* COMMAND_LINE_SIZE */ #ifdef CONFIG_BOOT_CONFIG_EMBED /* embedded_bootconfig_data is defined in bootconfig-data.S */ @@ -34,7 +38,129 @@ const char * __init xbc_get_embedded_bootconfig(size_t *size) return (*size) ? embedded_bootconfig_data : NULL; } #endif -#endif + +#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG +/* embedded_kernel_cmdline is defined in embedded-cmdline.S */ +extern __visible const char embedded_kernel_cmdline[]; +extern __visible const char embedded_kernel_cmdline_end[]; + +/* Set once the embedded cmdline has actually been prepended. */ +static bool xbc_cmdline_applied __initdata; + +/* + * str_prepend() - Prepend @src in front of the string in @dst, in place + * @dst: NUL-terminated destination buffer, currently @dst_len bytes long + * @dst_len: length of the current @dst string (excluding its NUL) + * @src: bytes to prepend (not NUL-terminated) + * @src_len: number of bytes from @src to prepend + * + * The caller must guarantee @dst has room for src_len + dst_len + 1 bytes. + * Moving dst_len + 1 bytes carries @dst's NUL terminator too, so an empty + * @dst needs no special case. + */ +static void __init str_prepend(char *dst, size_t dst_len, + const char *src, size_t src_len) +{ + memmove(dst + src_len, dst, dst_len + 1); + memcpy(dst, src, src_len); +} + +/** + * xbc_prepend_embedded_cmdline() - Prepend embedded bootconfig cmdline + * @dst: cmdline buffer to prepend into (must already contain a NUL byte) + * @size: total capacity of @dst in bytes + * + * Prepend the build-time-rendered "kernel" subtree of the embedded + * bootconfig to @dst. The rendered string already ends with a single + * space (the xbc_snprint_cmdline() invariant), which serves as the + * separator between the embedded keys and any existing content of @dst. + * On overflow, log an error and leave @dst untouched rather than + * silently truncating: booting without the embedded values is better + * than refusing to boot, and the error message tells the user why + * their embedded keys are missing. + * + * Intended to be called from setup_arch() before parse_early_param() so + * that early_param() handlers see the embedded values. + */ +void __init xbc_prepend_embedded_cmdline(char *dst, size_t size) +{ + size_t embed_len = embedded_kernel_cmdline_end - embedded_kernel_cmdline; + size_t dst_len; + + if (!size || embed_len <= 1) /* trailing NUL only */ + return; + embed_len--; /* exclude trailing NUL byte */ + + dst_len = strnlen(dst, size); + if (embed_len + dst_len + 1 > size) { + pr_err("embedded bootconfig cmdline (%zu bytes) does not fit in COMMAND_LINE_SIZE with %zu bytes already used; ignoring embedded values\n", + embed_len, dst_len); + return; + } + + str_prepend(dst, dst_len, embedded_kernel_cmdline, embed_len); + xbc_cmdline_applied = true; +} + +/** + * xbc_embedded_cmdline_applied() - Did the embedded cmdline get prepended? + * + * Return true if xbc_prepend_embedded_cmdline() actually prepended the + * embedded "kernel" subtree. setup_boot_config() uses this to avoid + * rendering the same keys a second time. + */ +bool __init xbc_embedded_cmdline_applied(void) +{ + return xbc_cmdline_applied; +} +#endif /* CONFIG_CMDLINE_FROM_BOOTCONFIG */ + +/* parse_args() callback: flag when the "bootconfig" parameter is present. */ +static int __init bootconfig_optin(char *param, char *val, + const char *unused, void *arg) +{ + if (!strcmp(param, "bootconfig")) + *(bool *)arg = true; + return 0; +} + +/** + * bootconfig_cmdline_requested() - Was "bootconfig" passed on the cmdline? + * @boot_cmdline: kernel command line to inspect (not modified) + * @end_offset: if non-NULL, set to the offset of the init arguments that + * follow a "--" separator, or 0 when there is none + * + * Parse a private copy of @boot_cmdline (parse_args() is destructive) and + * report whether "bootconfig" is present before the "--" separator. + * setup_arch() uses this to gate prepending the build-time embedded cmdline; + * setup_boot_config() uses it for the runtime opt-in and to locate the init + * arguments via @end_offset. Sharing one parser keeps the early and late + * paths agreeing on what counts as opt-in. CONFIG_BOOT_CONFIG_FORCE is not + * folded in here; callers apply it where they need it. + */ +bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset) +{ + static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata; + bool found = false; + char *err; + + if (end_offset) + *end_offset = 0; + + strscpy(tmp_cmdline, boot_cmdline, COMMAND_LINE_SIZE); + err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, + &found, bootconfig_optin); + if (IS_ERR(err)) + return false; + + /* parse_args() stops at "--" and returns the address of the rest. */ + if (end_offset && err) + *end_offset = err - tmp_cmdline; + + return found; +} + +#endif /* __KERNEL__ */ /* * Extra Boot Config (XBC) is given as tree-structured ascii text of @@ -427,11 +553,30 @@ static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) { struct xbc_node *knode, *vnode; - char *end = buf + size; const char *val, *q; + size_t len = 0; int ret; + /* + * Track the running written length rather than advancing @buf, so we + * never form "buf + size" or "buf += ret" while @buf is NULL (the + * size-probe call passes buf=NULL, size=0). NULL pointer arithmetic + * is undefined behavior and trips host UBSan / FORTIFY_SOURCE when + * this renderer runs at kernel build time. snprintf(NULL, 0, ...) + * itself is well defined and returns the would-be length. + */ xbc_node_for_each_key_value(root, knode, val) { + /* + * An empty or value-only @root (e.g. "kernel {}" or + * "kernel = x", possibly alongside "kernel.foo = bar") + * yields @root itself here. Skip it: composing a key for it + * would fail with -EINVAL, yet any real descendant keys must + * still be rendered. An entirely empty subtree then renders + * nothing and returns 0 rather than an error. + */ + if (knode == root) + continue; + ret = xbc_node_compose_key_after(root, knode, xbc_namebuf, XBC_KEYLEN_MAX); if (ret < 0) @@ -439,10 +584,11 @@ int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) vnode = xbc_node_get_child(knode); if (!vnode) { - ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf); + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s ", xbc_namebuf); if (ret < 0) return ret; - buf += ret; + len += ret; continue; } xbc_array_for_each_value(vnode, val) { @@ -452,15 +598,15 @@ int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) * whitespace. */ q = strpbrk(val, " \t\r\n") ? "\"" : ""; - ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ", - xbc_namebuf, q, val, q); + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s=%s%s%s ", xbc_namebuf, q, val, q); if (ret < 0) return ret; - buf += ret; + len += ret; } } - return buf - (end - size); + return len; } #undef rest diff --git a/lib/bug.c b/lib/bug.c index 292420f45811..7c1c2c27f58e 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -219,14 +219,12 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga no_cut = bug->flags & BUGFLAG_NO_CUT_HERE; has_args = bug->flags & BUGFLAG_ARGS; -#ifdef CONFIG_KUNIT /* * Before the once logic so suppressed warnings do not consume * the single-fire budget of WARN_ON_ONCE(). */ if (warning && kunit_is_suppressed_warning(true)) return BUG_TRAP_TYPE_WARN; -#endif disable_trace_on_warning(); diff --git a/lib/codetag.c b/lib/codetag.c index 4001a7ea6675..a9cda4c962a3 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -19,6 +19,8 @@ struct codetag_type { struct codetag_type_desc desc; /* generates unique sequence number for module load */ unsigned long next_mod_seq; + /* bumped on every module load and unload */ + unsigned long content_id; }; struct codetag_range { @@ -50,6 +52,20 @@ void codetag_unlock_module_list(struct codetag_type *cttype) up_read(&cttype->mod_lock); } +unsigned long codetag_get_content_id(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->content_id; +} + +unsigned int codetag_get_count(struct codetag_type *cttype) +{ + lockdep_assert_held(&cttype->mod_lock); + + return cttype->count; +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { @@ -204,6 +220,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod) down_write(&cttype->mod_lock); cmod->mod_seq = ++cttype->next_mod_seq; + ++cttype->content_id; mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL); if (mod_id >= 0) { if (cttype->desc.module_load) { @@ -368,6 +385,7 @@ void codetag_unload_module(struct module *mod) cttype->count -= range_size(cttype, &cmod->range); idr_remove(&cttype->mod_idr, mod_id); kfree(cmod); + ++cttype->content_id; } up_write(&cttype->mod_lock); if (found && cttype->desc.free_section_mem) diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 3efc854a2c08..60e0a77f9889 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -3,6 +3,8 @@ CONFIG_KUNIT=y CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_AES_CCM_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_AES_GCM_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 591c1c2a7fb3..56f308dd0273 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -9,7 +9,8 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate # Select dependencies of modes that are part of libaes. - select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_GF128HASH if CRYPTO_LIB_AES_GCM != n + select CRYPTO_LIB_UTILS config CRYPTO_LIB_AES_ARCH bool @@ -28,6 +29,12 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS +config CRYPTO_LIB_AES_CBC + tristate + select CRYPTO_LIB_AES + help + The AES-CBC and AES-CBC-CTS library functions. + config CRYPTO_LIB_AES_CBC_MACS tristate select CRYPTO_LIB_AES @@ -36,11 +43,38 @@ config CRYPTO_LIB_AES_CBC_MACS this if your module uses any of the functions from <crypto/aes-cbc-macs.h>. -config CRYPTO_LIB_AESGCM +config CRYPTO_LIB_AES_CCM tristate select CRYPTO_LIB_AES - select CRYPTO_LIB_GF128HASH - select CRYPTO_LIB_UTILS + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CTR + help + The AES-CCM library functions. + +config CRYPTO_LIB_AES_CTR + tristate + select CRYPTO_LIB_AES + help + The AES-CTR and AES-XCTR library functions. + +config CRYPTO_LIB_AES_ECB + tristate + select CRYPTO_LIB_AES + help + The AES-ECB library functions. + +config CRYPTO_LIB_AES_GCM + tristate + select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CTR + help + The AES-GCM library functions. + +config CRYPTO_LIB_AES_XTS + tristate + select CRYPTO_LIB_AES + help + The AES-XTS library functions. config CRYPTO_LIB_ARC4 tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index f1e9bf89785f..ca068df1f71f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -63,9 +63,6 @@ clean-files += powerpc/aesp8-ppc.S obj-$(CONFIG_CRYPTO_LIB_AESCFB) += libaescfb.o libaescfb-y := aescfb.o -obj-$(CONFIG_CRYPTO_LIB_AESGCM) += libaesgcm.o -libaesgcm-y := aesgcm.o - obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o libarc4-y := arc4.o diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index ca733f15b2a8..f1549839b3de 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -5,14 +5,21 @@ */ #include <crypto/aes-cbc-macs.h> +#include <crypto/aes-cbc.h> +#include <crypto/aes-ccm.h> +#include <crypto/aes-ctr.h> +#include <crypto/aes-ecb.h> +#include <crypto/aes-gcm.h> +#include <crypto/aes-xts.h> #include <crypto/aes.h> +#include <crypto/gf128mul.h> #include <crypto/utils.h> #include <linux/cache.h> #include <linux/crypto.h> #include <linux/export.h> #include <linux/module.h> #include <linux/unaligned.h> -#include "fips.h" +#include "fips-aes.h" static const u8 ____cacheline_aligned aes_sbox[] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, @@ -515,6 +522,26 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], } EXPORT_SYMBOL(aes_decrypt); +/* FIPS cryptographic algorithm self-test for "bare" AES */ +static void __init aes_fips_test(void) +{ + struct aes_key key; + u8 data[AES_BLOCK_SIZE]; + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: FIPS self-test failed (preparekey)\n"); + + aes_encrypt(&key, data, fips_test_data); + if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) + panic("aes: FIPS self-test failed (wrong ciphertext)\n"); + + aes_decrypt(&key, data, data); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} + #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) #ifndef aes_cbcmac_blocks_arch @@ -710,17 +737,10 @@ void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) } EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); -/* - * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 - * Implementation Guidance, a cryptographic algorithm self-test for at least one - * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes - * is implemented. This fulfills that requirement via AES-CMAC. - * - * This is just for FIPS. The full tests are in the KUnit test suite. - */ +/* FIPS cryptographic algorithm self-test for AES-CMAC */ static void __init aes_cmac_fips_test(void) { - struct aes_cmac_key key; + struct aes_cmac_key key __cleanup(aes_cmac_zeroize_key); u8 mac[AES_BLOCK_SIZE]; if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != @@ -729,7 +749,6 @@ static void __init aes_cmac_fips_test(void) aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); - memzero_explicit(&key, sizeof(key)); } #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ static inline void aes_cmac_fips_test(void) @@ -737,13 +756,1383 @@ static inline void aes_cmac_fips_test(void) } #endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB) +/* + * Hooks for optimized AES-ECB implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_ecb_encrypt_arch +static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_ecb_decrypt_arch +static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_ecb_encrypt_arch(dst, src, len, key.enc_key))) + return; + + for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) + aes_encrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_encrypt); + +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key) +{ + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_ecb_decrypt_arch(dst, src, len, key))) + return; + + for (size_t i = 0; i < len; i += AES_BLOCK_SIZE) + aes_decrypt(key, &dst[i], &src[i]); +} +EXPORT_SYMBOL_GPL(aes_ecb_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-ECB */ +static void __init aes_ecb_fips_test(void) +{ + struct aes_key key; + u8 data[sizeof(fips_test_data)]; + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: ECB FIPS self-test failed (preparekey)\n"); + + aes_ecb_encrypt(data, fips_test_data, sizeof(data), &key); + if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0) + panic("aes: ECB FIPS self-test failed (wrong ciphertext)\n"); + + aes_ecb_decrypt(data, data, sizeof(data), &key); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: ECB FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_ECB */ +static inline void aes_ecb_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_ECB */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC) +/* + * Hooks for optimized AES-CBC implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. Returning false + * causes the fallback implementation to be used instead. + */ +#ifndef aes_cbc_encrypt_arch +static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_decrypt_arch +static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + const u8 *prev = iv; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_cbc_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + do { + crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); + aes_encrypt(key, dst, dst); + prev = dst; + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } while (len); + memcpy(iv, prev, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_encrypt); + +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + u8 next_iv[AES_BLOCK_SIZE]; + + if (WARN_ON_ONCE(len % AES_BLOCK_SIZE)) + len = round_down(len, AES_BLOCK_SIZE); + + if (unlikely(len == 0)) + return; + + if (likely(aes_cbc_decrypt_arch(dst, src, len, iv, key))) + return; + + len -= AES_BLOCK_SIZE; + dst += len; + src += len; + memcpy(next_iv, src, AES_BLOCK_SIZE); + for (;;) { + aes_decrypt(key, dst, src); + if (len == 0) + break; + src -= AES_BLOCK_SIZE; + crypto_xor(dst, src, AES_BLOCK_SIZE); + dst -= AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } + crypto_xor(dst, iv, AES_BLOCK_SIZE); + memcpy(iv, next_iv, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_cbc_decrypt); + +/* + * Hooks for optimized AES-CBC-CTS implementations, overridable by the + * architecture. They are called with len > AES_BLOCK_SIZE. Returning false + * causes the fallback implementation to be used instead. The fallback + * implementation still uses the arch-optimized AES-CBC code if available, but + * direct implementation of AES-CBC-CTS is helpful on short messages. + */ +#ifndef aes_cbc_cts_encrypt_arch +static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_cbc_cts_decrypt_arch +static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], + const struct aes_key *key) +{ + return false; +} +#endif + +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ + size_t pn_len = len - pn_offset; + u8 tmp[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len == AES_BLOCK_SIZE) { + aes_cbc_encrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_encrypt_arch(dst, src, len, iv, key.enc_key))) + return; + + /* CBC-encrypt all blocks except the last. */ + aes_cbc_encrypt(dst, src, pn_offset, iv, key); + + /* + * Compute C[n] and C[n - 1]. + * + * Careful: src may equal dst (i.e., the encryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + */ + pad = &dst[pn_offset - AES_BLOCK_SIZE]; + memcpy(tmp, pad, AES_BLOCK_SIZE); + crypto_xor(tmp, &src[pn_offset], pn_len); + memcpy(&dst[pn_offset], pad, pn_len); /* C[n] */ + aes_encrypt(key, pad, tmp); /* C[n - 1] */ + + memzero_explicit(tmp, sizeof(tmp)); +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_encrypt); + +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[AES_BLOCK_SIZE], const struct aes_key *key) +{ + /* Offset to P[n] and C[n] (last plaintext and ciphertext block) */ + size_t pn_offset = round_down(len - 1, AES_BLOCK_SIZE); + /* Length of P[n] and C[n], 1 <= pn_len <= AES_BLOCK_SIZE */ + size_t pn_len = len - pn_offset; + u8 *pad; + + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (len == AES_BLOCK_SIZE) { + aes_cbc_decrypt(dst, src, len, iv, key); + return; + } + if (likely(aes_cbc_cts_decrypt_arch(dst, src, len, iv, key))) + return; + + /* Compute P[0]..P[n - 2]. */ + aes_cbc_decrypt(dst, src, pn_offset - AES_BLOCK_SIZE, iv, key); + + /* + * Compute P[n] and P[n - 1]. + * + * Careful: src may equal dst (i.e., the decryption can be in-place), so + * src[pn_offset..] can't be read after dst[pn_offset..] is written. + * + * To avoid needing a temporary buffer, do a "redundant" XOR to recover + * src[pn_offset..] from dst[pn_offset..] after the latter is written. + */ + pad = &dst[pn_offset - AES_BLOCK_SIZE]; + aes_decrypt(key, pad, &src[pn_offset - AES_BLOCK_SIZE]); + crypto_xor_cpy(&dst[pn_offset], &src[pn_offset], pad, + pn_len); /* P[n] */ + crypto_xor(pad, &dst[pn_offset], pn_len); + aes_decrypt(key, pad, pad); + crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */ +} +EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-CBC */ +static void __init aes_cbc_fips_test(void) +{ + struct aes_key key; + u8 iv[AES_BLOCK_SIZE]; + u8 data[sizeof(fips_test_data)]; + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: CBC FIPS self-test failed (preparekey)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_encrypt(data, fips_test_data, sizeof(data), iv, &key); + if (memcmp(fips_test_aes_cbc_ctext, data, sizeof(data)) != 0) + panic("aes: CBC FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_decrypt(data, data, sizeof(data), iv, &key); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: CBC FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} + +/* FIPS cryptographic algorithm self-test for AES-CBC-CTS */ +static void __init aes_cbc_cts_fips_test(void) +{ + struct aes_key key; + u8 iv[AES_BLOCK_SIZE]; + const size_t data_len = 2 * AES_BLOCK_SIZE; + u8 ptext[2 * AES_BLOCK_SIZE]; + u8 data[2 * AES_BLOCK_SIZE]; + + /* ptext = fips_test_data || fips_test_data */ + memcpy(ptext, fips_test_data, AES_BLOCK_SIZE); + memcpy(&ptext[AES_BLOCK_SIZE], ptext, AES_BLOCK_SIZE); + + if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: CBC-CTS FIPS self-test failed (preparekey)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_cts_encrypt(data, ptext, data_len, iv, &key); + if (memcmp(fips_test_aes_cbc_cts_ctext, data, data_len) != 0) + panic("aes: CBC-CTS FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(iv, fips_test_iv, sizeof(iv)); + aes_cbc_cts_decrypt(data, data, data_len, iv, &key); + if (memcmp(ptext, data, data_len) != 0) + panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CBC */ +static inline void aes_cbc_fips_test(void) +{ +} +static inline void aes_cbc_cts_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CBC */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) +/* + * Hooks for optimized AES-CTR and AES-XCTR implementations, overridable by the + * architecture. They are called with any len >= 0. Returning false causes the + * fallback implementation to be used instead. + */ +#ifndef aes_ctr_arch +static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len, + u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key) +{ + return false; +} +#endif +#ifndef aes_xctr_arch +static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], + const struct aes_enckey *key) +{ + return false; +} +#endif + +static __always_inline void inc_be128_ctr(u8 ctr[AES_BLOCK_SIZE]) +{ + /* + * 255 times out of 256 the first iteration is enough, so unroll the + * first iteration as a micro-optimization. + */ + if ((++ctr[AES_BLOCK_SIZE - 1]) != 0) + return; + for (int i = AES_BLOCK_SIZE - 2; i >= 0; i--) { + if (++ctr[i] != 0) + break; + } +} + +void aes_ctr(u8 *dst, const u8 *src, size_t len, u8 ctr[AES_BLOCK_SIZE], + aes_encrypt_arg key) +{ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_ctr_arch(dst, src, len, ctr, key.enc_key))) + return; + + /* Handle the full blocks. */ + for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + inc_be128_ctr(ctr); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + aes_encrypt(key, keystream, ctr); + crypto_xor_cpy(dst, src, keystream, len); + /* Counter is incremented even with just a partial block. */ + inc_be128_ctr(ctr); + } + memzero_explicit(keystream, sizeof(keystream)); +} +EXPORT_SYMBOL_GPL(aes_ctr); + +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[AES_BLOCK_SIZE], aes_encrypt_arg key) +{ + const __le64 iv0 = get_unaligned((const __le64 *)&iv[0]); + __le64 aes_input[2]; + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + if (likely(aes_xctr_arch(dst, src, len, ctr, iv, key.enc_key))) + return; + + aes_input[1] = get_unaligned((const __le64 *)&iv[8]); + /* Handle the full blocks. */ + for (; len >= AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE) { + aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, AES_BLOCK_SIZE); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } + /* Handle any partial block at the end. */ + if (len) { + /* Counter is incremented even with just a partial block. */ + aes_input[0] = iv0 ^ cpu_to_le64((*ctr)++); + aes_encrypt(key, keystream, (const u8 *)aes_input); + crypto_xor_cpy(dst, src, keystream, len); + } + memzero_explicit(keystream, sizeof(keystream)); + memzero_explicit(aes_input, sizeof(aes_input)); +} +EXPORT_SYMBOL_GPL(aes_xctr); + +/* FIPS cryptographic algorithm self-test for AES-CTR */ +static void __init aes_ctr_fips_test(void) +{ + struct aes_enckey key; + u8 ctr[AES_BLOCK_SIZE]; + u8 data[sizeof(fips_test_data)]; + + if (aes_prepareenckey(&key, fips_test_key, sizeof(fips_test_key)) != 0) + panic("aes: CTR FIPS self-test failed (preparekey)\n"); + + memcpy(ctr, fips_test_iv, sizeof(ctr)); + aes_ctr(data, fips_test_data, sizeof(data), ctr, &key); + if (memcmp(fips_test_aes_ctr_ctext, data, sizeof(data)) != 0) + panic("aes: CTR FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(ctr, fips_test_iv, sizeof(ctr)); + aes_ctr(data, data, sizeof(data), ctr, &key); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: CTR FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CTR */ +static inline void aes_ctr_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CTR */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS) +int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, + size_t key_len, int flags) +{ + int err; + + err = __xts_verify_key(in_key, key_len, flags); + if (unlikely(err)) + goto out_zeroize; + /* First half of XTS key is the main key */ + err = aes_preparekey(&key->main_key, in_key, key_len / 2); + if (unlikely(err)) + goto out_zeroize; + /* Second half of XTS key is the tweak key */ + err = aes_prepareenckey(&key->tweak_key, &in_key[key_len / 2], + key_len / 2); + if (unlikely(err)) + goto out_zeroize; + return 0; + +out_zeroize: + memzero_explicit(key, sizeof(*key)); + return err; +} +EXPORT_SYMBOL_GPL(aes_xts_preparekey); + +/* + * Hooks for optimized AES-XTS implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. In other words, + * they aren't expected to handle ciphertext stealing or empty inputs. + * Returning false causes the fallback implementation to be used instead. + * + * (Currently, all users of AES-XTS in the kernel seem to en/decrypt whole + * numbers of blocks anyway, with len >= 512. So there's no need to heavily + * optimize ciphertext stealing for short messages.) + */ +#ifndef aes_xts_encrypt_arch +static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif +#ifndef aes_xts_decrypt_arch +static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont) +{ + return false; +} +#endif + +static noinline void aes_xts_crypt_nocts_blockbyblock( + u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont, bool enc) +{ + le128 t; + + if (cont) + memcpy(&t, tweak, sizeof(t)); + else + aes_encrypt(&key->tweak_key, (u8 *)&t, tweak); + do { + crypto_xor_cpy(dst, src, (const u8 *)&t, AES_BLOCK_SIZE); + if (enc) + aes_encrypt(&key->main_key, dst, dst); + else + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, (const u8 *)&t, AES_BLOCK_SIZE); + gf128mul_x_ble(&t, &t); + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + len -= AES_BLOCK_SIZE; + } while (len); + memcpy(tweak, &t, sizeof(t)); + memzero_explicit(&t, sizeof(t)); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ +static __always_inline void aes_xts_encrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_encrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* + * For the fallback, just go block-by-block. It could be implemented on + * top of AES-ECB, which could be significantly faster than this if the + * arch has optimized AES-ECB code but not AES-XTS. However, AES-XTS + * performance is important enough that it needs to be (and has been) + * implemented directly by every non-obsolete arch anyway. + */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc= */ true); +} + +/* Requires len > 0 && len % AES_BLOCK_SIZE == 0 */ +static __always_inline void aes_xts_decrypt_nocts(u8 *dst, const u8 *src, + size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + if (likely(aes_xts_decrypt_arch(dst, src, len, tweak, key, cont))) + return; + + /* Just go block-by-block. See comment in aes_xts_encrypt_nocts(). */ + aes_xts_crypt_nocts_blockbyblock(dst, src, len, tweak, key, cont, + /* enc= */ false); +} + +static noinline void aes_xts_encrypt_cts(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len = round_down(len, AES_BLOCK_SIZE); + u8 tmp_block[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + + /* Encrypt all full blocks. */ + aes_xts_encrypt_nocts(dst, src, nocts_len, tweak, key, cont); + dst += nocts_len - AES_BLOCK_SIZE; + src += nocts_len - AES_BLOCK_SIZE; + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * encrypted last full block. Note that a temporary buffer is needed to + * support in-place encryption. + */ + memcpy(tmp_block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp_block, partial_len); + + /* Encrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_encrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(tmp_block, sizeof(tmp_block)); +} + +static noinline void aes_xts_decrypt_cts(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], + const struct aes_xts_key *key, + bool cont) +{ + size_t partial_len = len % AES_BLOCK_SIZE; /* Length of partial block */ + size_t nocts_len = round_down(len, AES_BLOCK_SIZE) - AES_BLOCK_SIZE; + union { + u8 block[AES_BLOCK_SIZE]; + le128 tweak; + } tmp __aligned(__alignof__(long)); + + /* + * Decrypt all blocks except the last full block and the partial block. + * The last full block has to be handled specially because decryption + * ciphertext stealing uses the last two tweaks in reverse order. + * + * nocts_len == 0 is possible here, which aes_xts_decrypt_nocts() + * doesn't handle (so that the length doesn't get checked redundantly in + * the fast path). So handle that case specially as well. + */ + if (nocts_len) + aes_xts_decrypt_nocts(dst, src, nocts_len, tweak, key, cont); + else if (!cont) + aes_encrypt(&key->tweak_key, tweak, tweak); + dst += nocts_len; + src += nocts_len; + + /* Copy the tweak, advance it again, then decrypt last full block. */ + memcpy(&tmp.tweak, tweak, AES_BLOCK_SIZE); + gf128mul_x_ble(&tmp.tweak, &tmp.tweak); + crypto_xor_cpy(dst, src, tmp.block, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tmp.block, AES_BLOCK_SIZE); + + /* + * Swap the partial block with the first 'partial_len' bytes of the + * decrypted last full block. Note that a temporary buffer is needed to + * support in-place decryption. + */ + memcpy(tmp.block, src + AES_BLOCK_SIZE, partial_len); + memcpy(dst + AES_BLOCK_SIZE, dst, partial_len); + memcpy(dst, tmp.block, partial_len); + + /* Decrypt the last full block again. */ + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + aes_decrypt(&key->main_key, dst, dst); + crypto_xor(dst, tweak, AES_BLOCK_SIZE); + memzero_explicit(&tmp, sizeof(tmp)); +} + +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_encrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_encrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_encrypt); + +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[AES_BLOCK_SIZE], const struct aes_xts_key *key, + bool cont) +{ + if (WARN_ON_ONCE(len < AES_BLOCK_SIZE)) + return; + + if (unlikely(len % AES_BLOCK_SIZE)) { + aes_xts_decrypt_cts(dst, src, len, tweak, key, cont); + return; + } + + aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont); +} +EXPORT_SYMBOL_GPL(aes_xts_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-XTS */ +static void __init aes_xts_fips_test(void) +{ + struct aes_xts_key *key __free(kfree_sensitive) = kmalloc_obj(*key); + u8 tweak[AES_BLOCK_SIZE]; + u8 data[sizeof(fips_test_data)]; + + if (key == NULL) + panic("aes: XTS FIPS self-test failed (kmalloc)\n"); + + if (aes_xts_preparekey(key, fips_test_xts_key, + sizeof(fips_test_xts_key), 0) != 0) + panic("aes: XTS FIPS self-test failed (preparekey)\n"); + + memcpy(tweak, fips_test_iv, sizeof(tweak)); + aes_xts_encrypt(data, fips_test_data, sizeof(data), tweak, key, false); + if (memcmp(fips_test_aes_xts_ctext, data, sizeof(data)) != 0) + panic("aes: XTS FIPS self-test failed (wrong ciphertext)\n"); + + memcpy(tweak, fips_test_iv, sizeof(tweak)); + aes_xts_decrypt(data, data, sizeof(data), tweak, key, false); + if (memcmp(fips_test_data, data, sizeof(data)) != 0) + panic("aes: XTS FIPS self-test failed (wrong plaintext)\n"); +} +#else /* CONFIG_CRYPTO_LIB_AES_XTS */ +static inline void aes_xts_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_XTS */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM) +/* + * Hooks for optimized AES-GCM implementations, overridable by the architecture. + * They are called with len > 0 && len % AES_BLOCK_SIZE == 0. I.e. they aren't + * expected to handle empty inputs or partial blocks, as those cases are handled + * by non-arch-specific code instead. + * + * The GHASH accumulator is provided in POLYVAL format. The counter is provided + * in big endian format, and it's read-only, as the caller handles updating it. + * + * Returning false causes the fallback implementation to be used instead. + * + * These hooks are used only for en/decrypted data. For the associated data the + * GHASH functions are called instead, so those should be implemented too. + */ +#ifndef aes_gcm_encrypt_update_arch +static bool aes_gcm_encrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif +#ifndef aes_gcm_decrypt_update_arch +static bool aes_gcm_decrypt_update_arch(u8 *dst, const u8 *src, size_t len, + struct polyval_elem *ghash_acc, + const __be32 ctr32[4], + const struct aes_enckey *aes_key, + const struct ghash_key *ghash_key) +{ + return false; +} +#endif + +int aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + u8 h[AES_BLOCK_SIZE] = { 0 }; + int err; + + err = crypto_gcm_check_authsize(authtag_len); + if (unlikely(err)) + return err; + + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + aes_encrypt(&key->aes, h, h); + ghash_preparekey(&key->ghash, h); + + key->authtag_len = authtag_len; + + memzero_explicit(h, sizeof(h)); + return 0; +} +EXPORT_SYMBOL_GPL(aes_gcm_preparekey); + +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + ctx->key = key; + ctx->ad_len = 0; + ctx->data_len = 0; + ghash_init(&ctx->ghash, &key->ghash); + memset(ctx->keystream, 0, sizeof(ctx->keystream)); + + memcpy(ctx->ctr32, nonce, 12); + ctx->ctr32[3] = cpu_to_be32(1); + + aes_encrypt(&key->aes, ctx->j0_enc, ctx->ctr); + ctx->ctr32[3] = cpu_to_be32(2); +} +EXPORT_SYMBOL_GPL(aes_gcm_init); + +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len) +{ + WARN_ON_ONCE(ctx->data_len != 0); + if (len) { + ghash_update(&ctx->ghash, ad, len); + ctx->ad_len += len; + } +} +EXPORT_SYMBOL_GPL(aes_gcm_auth_update); + +static const u8 gcm_zeroes[AES_BLOCK_SIZE]; + +static __always_inline void ghash_pad(struct ghash_ctx *ghash, u64 len) +{ + if (len % AES_BLOCK_SIZE) + ghash_update(ghash, gcm_zeroes, -len % AES_BLOCK_SIZE); +} + +static __always_inline void aes_gcm_crypt_update(struct aes_gcm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len, n; + + if (unlikely(len == 0)) + return; + + partial_len = ctx->data_len % AES_BLOCK_SIZE; + if (ctx->data_len == 0) + ghash_pad(&ctx->ghash, ctx->ad_len); + ctx->data_len += len; + + if (unlikely(partial_len != 0)) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n = min(len, AES_BLOCK_SIZE - partial_len); + if (enc) { + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + ghash_update(&ctx->ghash, dst, n); + } else { + ghash_update(&ctx->ghash, src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], + n); + } + dst += n; + src += n; + len -= n; + } + + if (len >= AES_BLOCK_SIZE) { + n = round_down(len, AES_BLOCK_SIZE); + if (enc) { + if (likely(aes_gcm_encrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + ghash_update(&ctx->ghash, dst, n); + } + } else { + if (likely(aes_gcm_decrypt_update_arch( + dst, src, n, &ctx->ghash.acc, ctx->ctr32, + &ctx->key->aes, &ctx->key->ghash))) { + be32_add_cpu(&ctx->ctr32[3], + n / AES_BLOCK_SIZE); + } else { + ghash_update(&ctx->ghash, src, n); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + } + } + dst += n; + src += n; + len -= n; + } + + if (len != 0) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + be32_add_cpu(&ctx->ctr32[3], 1); + if (enc) { + crypto_xor_cpy(dst, src, ctx->keystream, len); + ghash_update(&ctx->ghash, dst, len); + } else { + ghash_update(&ctx->ghash, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + } + } +} + +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ true); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_update); + +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_gcm_crypt_update(ctx, dst, src, len, /* enc= */ false); +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_update); + +/* Maximum AES-GCM associated data length in bytes */ +#define AES_GCM_MAX_AD_LEN ((1ULL << 61) - 1) +/* Maximum AES-GCM en/decrypted data length in bytes */ +#define AES_GCM_MAX_DATA_LEN ((1ULL << 36) - 32) + +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag) +{ + __be64 tail[2]; + + WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN); + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN); + + ghash_pad(&ctx->ghash, + ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); + + tail[0] = cpu_to_be64(ctx->ad_len * 8); + tail[1] = cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + + crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final); + +int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag) +{ + __be64 tail[2]; + int err; + + if (WARN_ON_ONCE(ctx->ad_len > AES_GCM_MAX_AD_LEN) || + WARN_ON_ONCE(ctx->data_len > AES_GCM_MAX_DATA_LEN)) { + err = -EBADMSG; + goto out; + } + + ghash_pad(&ctx->ghash, + ctx->data_len == 0 ? ctx->ad_len : ctx->data_len); + + tail[0] = cpu_to_be64(ctx->ad_len * 8); + tail[1] = cpu_to_be64(ctx->data_len * 8); + ghash_update(&ctx->ghash, (const u8 *)tail, 16); + ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */ + crypto_xor(ctx->ctr, ctx->j0_enc, ctx->key->authtag_len); + err = crypto_memneq(ctx->ctr, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final); + +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_encrypt_update(&ctx, dst, src, data_len); + aes_gcm_encrypt_final(&ctx, authtag); +} +EXPORT_SYMBOL_GPL(aes_gcm_encrypt); + +int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[12], + const struct aes_gcm_key *key) +{ + struct aes_gcm_ctx ctx; + int err; + + aes_gcm_init(&ctx, nonce, key); + aes_gcm_auth_update(&ctx, ad, ad_len); + aes_gcm_decrypt_update(&ctx, dst, src, data_len); + err = aes_gcm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_gcm_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-GCM */ +static void __init aes_gcm_fips_test(void) +{ + const size_t data_len = sizeof(fips_test_data); + u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; + struct aes_gcm_key key; + int err; + + if (aes_gcm_preparekey(&key, fips_test_key, sizeof(fips_test_key), + AES_BLOCK_SIZE) != 0) + panic("aes: GCM FIPS self-test failed (preparekey)\n"); + + aes_gcm_encrypt(buf, fips_test_data, data_len, &buf[data_len], + fips_test_ad, sizeof(fips_test_ad), fips_test_iv, &key); + if (memcmp(fips_test_aes_gcm_ctext_and_tag, buf, sizeof(buf)) != 0) + panic("aes: GCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); + + err = aes_gcm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, + sizeof(fips_test_ad), fips_test_iv, &key); + if (err != 0) + panic("aes: GCM FIPS self-test failed (decryption failed)\n"); + if (memcmp(fips_test_data, buf, data_len) != 0) + panic("aes: GCM FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_GCM */ +static inline void aes_gcm_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_GCM */ + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM) +int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len) +{ + int err; + + if (unlikely(authtag_len < 4 || authtag_len > 16 || authtag_len % 2)) + return -EINVAL; + + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (unlikely(err)) + return err; + + key->authtag_len = authtag_len; + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_preparekey); + +int aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key) +{ + /* + * This is the value L defined in the CCM specification. It determines + * the maximum allowed message length, and it is itself determined by + * the nonce length. They are inversely related, i.e. the longer the + * nonce the smaller the maximum message length is. + */ + unsigned int l = 15 - nonce_len; + + if (unlikely(nonce_len < 7 || nonce_len > 13)) + return -EINVAL; + /* Thus 2 <= l <= 8. */ + + /* Check whether data_len can be represented in 'l' bytes. */ + if (unlikely(data_len > U64_MAX >> (64 - 8 * l))) + return -EOVERFLOW; + + ctx->key = key; + ctx->ad_remaining = ad_len; + ctx->data_remaining = data_len; + ctx->ad_padded = false; + + /* + * Initialize the zero-th counter block to: + * + * L - 1 || nonce || 0 + * + * ... and the zero-th CBC-MAC block to: + * + * Flags || nonce || data_len + */ + *(__be64 *)&ctx->ctr[8] = 0; + *(__be64 *)&ctx->mac[8] = cpu_to_be64(data_len); + ctx->ctr[0] = l - 1; + ctx->mac[0] = (ad_len ? 0x40 : 0) | + (((key->authtag_len - 2) / 2) << 3) | (l - 1); + memcpy(&ctx->ctr[1], nonce, nonce_len); /* Overlapping store */ + memcpy(&ctx->mac[1], nonce, nonce_len); /* Overlapping store */ + + /* + * Generate S_0 by encrypting the counter (this is used to encrypt the + * auth tag later), and encrypt the zero-th CBC-MAC block. + */ + aes_encrypt(&key->aes, ctx->s0, ctx->ctr); + aes_encrypt(&key->aes, ctx->mac, ctx->mac); + + /* Increment the counter from 0 to 1. */ + ctx->ctr[15] = 1; + + if (ad_len) { + /* + * Update CBC-MAC with the associated data length, represented + * using either 2, 6, or 10 bytes depending on the length. + */ + if (likely(ad_len < 0xff00)) { + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(ad_len); + ctx->partial_len = 2; + } else if (ad_len <= U32_MAX) { + __be32 *p = (__be32 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xfffe); + put_unaligned(get_unaligned(p) ^ cpu_to_be32(ad_len), + p); + ctx->partial_len = 6; + } else { + __be64 *p = (__be64 *)&ctx->mac[2]; + + *(__be16 *)&ctx->mac[0] ^= cpu_to_be16(0xffff); + put_unaligned(get_unaligned(p) ^ cpu_to_be64(ad_len), + p); + ctx->partial_len = 10; + } + } else { + ctx->partial_len = 0; + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_init); + +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len) +{ + size_t partial_len = ctx->partial_len; + bool enc_before = false; + size_t nblocks; + + WARN_ON_ONCE(ctx->ad_padded); + + /* + * We could warn on len > ad_remaining here, but underflow will be + * caught by the != 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->ad_remaining -= len; + + if (partial_len) { + size_t n = min(len, AES_BLOCK_SIZE - partial_len); + + crypto_xor(&ctx->mac[partial_len], ad, n); + ad += n; + len -= n; + partial_len += n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len = partial_len; + return; + } + enc_before = true; + } + + nblocks = len / AES_BLOCK_SIZE; + len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else { + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, ad, nblocks, + enc_before, /* enc_after= */ true); + ad += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->mac, ad, len); + ctx->partial_len = len; +} +EXPORT_SYMBOL_GPL(aes_ccm_auth_update); + +static __always_inline void aes_ccm_crypt_update(struct aes_ccm_ctx *ctx, + u8 *dst, const u8 *src, + size_t len, bool enc) +{ + size_t partial_len = ctx->partial_len; + size_t n, nblocks; + + if (unlikely(len == 0)) + return; + + WARN_ON_ONCE(ctx->ad_remaining != 0); + + /* + * We could warn on len > data_remaining here, but underflow will be + * caught by the != 0 check at the end anyway. (It's a u64, so it isn't + * going to underflow all the way back to 0.) + */ + ctx->data_remaining -= len; + + if (!ctx->ad_padded) { + ctx->ad_padded = true; + if (partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } else if (partial_len) { + /* + * The previous call ended on a non-block-aligned data_len, so + * continue using a previously-generated keystream block. + */ + n = min(len, AES_BLOCK_SIZE - partial_len); + if (enc) + crypto_xor(&ctx->mac[partial_len], src, n); + crypto_xor_cpy(dst, src, &ctx->keystream[partial_len], n); + if (!enc) + crypto_xor(&ctx->mac[partial_len], dst, n); + dst += n; + src += n; + len -= n; + partial_len += n; + if (partial_len < AES_BLOCK_SIZE) { + ctx->partial_len = partial_len; + return; + } + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + } + + if (len >= AES_BLOCK_SIZE) { + n = round_down(len, AES_BLOCK_SIZE); + nblocks = len / AES_BLOCK_SIZE; + if (enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, src, + nblocks, /* enc_before= */ false, + /* enc_after= */ true); + aes_ctr(dst, src, n, ctx->ctr, &ctx->key->aes); + if (!enc) + aes_cbcmac_blocks(ctx->mac, &ctx->key->aes, dst, + nblocks, /* enc_before= */ false, + /* enc_after= */ true); + dst += n; + src += n; + len -= n; + } + + if (len) { + /* + * Ending on a non-block aligned data_len. Generate the next + * keystream block, use the needed portion of it, and leave it + * cached in ctx->keystream in case this isn't the final call. + */ + aes_encrypt(&ctx->key->aes, ctx->keystream, ctx->ctr); + inc_be128_ctr(ctx->ctr); + if (enc) + crypto_xor(ctx->mac, src, len); + crypto_xor_cpy(dst, src, ctx->keystream, len); + if (!enc) + crypto_xor(ctx->mac, dst, len); + } + ctx->partial_len = len; +} + +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ true); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_update); + +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len) +{ + aes_ccm_crypt_update(ctx, dst, src, len, /* enc= */ false); +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_update); + +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag) +{ + WARN_ON_ONCE(ctx->ad_remaining != 0); + WARN_ON_ONCE(ctx->data_remaining != 0); + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final); + +int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag) +{ + int err; + + if (WARN_ON_ONCE(ctx->ad_remaining != 0) || + WARN_ON_ONCE(ctx->data_remaining != 0)) { + err = -EBADMSG; + goto out; + } + + if (ctx->partial_len) + aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac); + crypto_xor(ctx->mac, ctx->s0, ctx->key->authtag_len); + err = crypto_memneq(ctx->mac, authtag, ctx->key->authtag_len) ? + -EBADMSG : + 0; +out: + memzero_explicit(ctx, sizeof(*ctx)); + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final); + +int aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_encrypt_update(&ctx, dst, src, data_len); + aes_ccm_encrypt_final(&ctx, authtag); + return 0; +} +EXPORT_SYMBOL_GPL(aes_ccm_encrypt); + +int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct aes_ccm_key *key) +{ + struct aes_ccm_ctx ctx; + int err; + + err = aes_ccm_init(&ctx, data_len, ad_len, nonce, nonce_len, key); + if (unlikely(err)) + return err; + aes_ccm_auth_update(&ctx, ad, ad_len); + aes_ccm_decrypt_update(&ctx, dst, src, data_len); + err = aes_ccm_decrypt_final(&ctx, authtag); + if (unlikely(err) && data_len) { + /* + * Clear the inauthentic decrypted data so that callers won't + * receive it even if they fail to correctly handle errors. + */ + memset(dst, 0, data_len); + } + return err; +} +EXPORT_SYMBOL_GPL(aes_ccm_decrypt); + +/* FIPS cryptographic algorithm self-test for AES-CCM */ +static void __init aes_ccm_fips_test(void) +{ + const size_t data_len = sizeof(fips_test_data); + const size_t nonce_len = 13; + u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE]; + struct aes_ccm_key key; + int err; + + if (aes_ccm_preparekey(&key, fips_test_key, sizeof(fips_test_key), + AES_BLOCK_SIZE) != 0) + panic("aes: CCM FIPS self-test failed (preparekey)\n"); + + err = aes_ccm_encrypt(buf, fips_test_data, data_len, &buf[data_len], + fips_test_ad, sizeof(fips_test_ad), fips_test_iv, + nonce_len, &key); + if (err != 0) + panic("aes: CCM FIPS self-test failed (encryption failed)\n"); + if (memcmp(fips_test_aes_ccm_ctext_and_tag, buf, sizeof(buf)) != 0) + panic("aes: CCM FIPS self-test failed (wrong ciphertext and/or tag)\n"); + + err = aes_ccm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad, + sizeof(fips_test_ad), fips_test_iv, nonce_len, + &key); + if (err != 0) + panic("aes: CCM FIPS self-test failed (decryption failed)\n"); + if (memcmp(fips_test_data, buf, data_len) != 0) + panic("aes: CCM FIPS self-test failed (wrong plaintext)\n"); + + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CCM */ +static inline void aes_ccm_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CCM */ + static int __init aes_mod_init(void) { #ifdef aes_mod_init_arch aes_mod_init_arch(); #endif - if (fips_enabled) + if (fips_enabled) { + aes_fips_test(); aes_cmac_fips_test(); + aes_ecb_fips_test(); + aes_cbc_fips_test(); + aes_cbc_cts_fips_test(); + aes_ctr_fips_test(); + aes_xts_fips_test(); + aes_gcm_fips_test(); + aes_ccm_fips_test(); + } return 0; } subsys_initcall(aes_mod_init); diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c deleted file mode 100644 index 1da31e1f747d..000000000000 --- a/lib/crypto/aesgcm.c +++ /dev/null @@ -1,721 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Minimal library implementation of GCM - * - * Copyright 2022 Google LLC - */ - -#include <crypto/gcm.h> -#include <crypto/utils.h> -#include <linux/export.h> -#include <linux/module.h> - -/** - * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key - * schedule - * - * @ctx: The data structure that will hold the AES-GCM key schedule - * @key: The AES encryption input key - * @keysize: The length in bytes of the input key - * @authsize: The size in bytes of the GCM authentication tag - * - * Returns: 0 on success, or -EINVAL if @keysize or @authsize contain values - * that are not permitted by the GCM specification. - */ -int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, - unsigned int keysize, unsigned int authsize) -{ - u8 h[AES_BLOCK_SIZE] = {}; - int ret; - - ret = crypto_gcm_check_authsize(authsize) ?: - aes_prepareenckey(&ctx->aes_key, key, keysize); - if (ret) - return ret; - - ctx->authsize = authsize; - aes_encrypt(&ctx->aes_key, h, h); - ghash_preparekey(&ctx->ghash_key, h); - memzero_explicit(h, sizeof(h)); - return 0; -} -EXPORT_SYMBOL(aesgcm_expandkey); - -/** - * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. - * @ctx: The data structure that will hold the AES-GCM key schedule - * @src: The input source data. - * @src_len: Length of the source data. - * @assoc: Points to the associated data. - * @assoc_len: Length of the associated data values. - * @ctr: Points to the counter value. - * @authtag: The output buffer for the authentication tag. - * - * It takes in the AES-GCM context, source data, associated data, counter value, - * and an output buffer for the authentication tag. - */ -static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, - const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) -{ - static const u8 zeroes[GHASH_BLOCK_SIZE]; - __be64 tail[2] = { - cpu_to_be64((u64)assoc_len * 8), - cpu_to_be64((u64)src_len * 8), - }; - struct ghash_ctx ghash; - u8 ghash_out[AES_BLOCK_SIZE]; - u8 enc_ctr[AES_BLOCK_SIZE]; - - ghash_init(&ghash, &ctx->ghash_key); - - ghash_update(&ghash, assoc, assoc_len); - ghash_update(&ghash, zeroes, -assoc_len & (GHASH_BLOCK_SIZE - 1)); - - ghash_update(&ghash, src, src_len); - ghash_update(&ghash, zeroes, -src_len & (GHASH_BLOCK_SIZE - 1)); - - ghash_update(&ghash, (const u8 *)&tail, sizeof(tail)); - - ghash_final(&ghash, ghash_out); - - ctr[3] = cpu_to_be32(1); - aes_encrypt(&ctx->aes_key, enc_ctr, (const u8 *)ctr); - crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize); - - memzero_explicit(ghash_out, sizeof(ghash_out)); - memzero_explicit(enc_ctr, sizeof(enc_ctr)); -} - -static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int len, __be32 *ctr) -{ - u8 buf[AES_BLOCK_SIZE]; - unsigned int n = 2; - - while (len > 0) { - /* - * The counter increment below must not result in overflow or - * carry into the next 32-bit word, as this could result in - * inadvertent IV reuse, which must be avoided at all cost for - * stream ciphers such as AES-CTR. Given the range of 'int - * len', this cannot happen, so no explicit test is necessary. - */ - ctr[3] = cpu_to_be32(n++); - aes_encrypt(&ctx->aes_key, buf, (const u8 *)ctr); - crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE)); - - dst += AES_BLOCK_SIZE; - src += AES_BLOCK_SIZE; - len -= AES_BLOCK_SIZE; - } - memzero_explicit(buf, sizeof(buf)); -} - -/** - * aesgcm_encrypt - Perform AES-GCM encryption on a block of data - * - * @ctx: The AES-GCM key schedule - * @dst: Pointer to the ciphertext output buffer - * @src: Pointer the plaintext (may equal @dst for encryption in place) - * @crypt_len: The size in bytes of the plaintext and ciphertext. - * @assoc: Pointer to the associated data, - * @assoc_len: The size in bytes of the associated data - * @iv: The initialization vector (IV) to use for this block of data - * (must be 12 bytes in size as per the GCM spec recommendation) - * @authtag: The address of the buffer in memory where the authentication - * tag should be stored. The buffer is assumed to have space for - * @ctx->authsize bytes. - */ -void aesgcm_encrypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int crypt_len, const u8 *assoc, int assoc_len, - const u8 iv[GCM_AES_IV_SIZE], u8 *authtag) -{ - __be32 ctr[4]; - - memcpy(ctr, iv, GCM_AES_IV_SIZE); - - aesgcm_crypt(ctx, dst, src, crypt_len, ctr); - aesgcm_mac(ctx, dst, crypt_len, assoc, assoc_len, ctr, authtag); -} -EXPORT_SYMBOL(aesgcm_encrypt); - -/** - * aesgcm_decrypt - Perform AES-GCM decryption on a block of data - * - * @ctx: The AES-GCM key schedule - * @dst: Pointer to the plaintext output buffer - * @src: Pointer the ciphertext (may equal @dst for decryption in place) - * @crypt_len: The size in bytes of the plaintext and ciphertext. - * @assoc: Pointer to the associated data, - * @assoc_len: The size in bytes of the associated data - * @iv: The initialization vector (IV) to use for this block of data - * (must be 12 bytes in size as per the GCM spec recommendation) - * @authtag: The address of the buffer in memory where the authentication - * tag is stored. - * - * Returns: true on success, or false if the ciphertext failed authentication. - * On failure, no plaintext will be returned. - */ -bool __must_check aesgcm_decrypt(const struct aesgcm_ctx *ctx, u8 *dst, - const u8 *src, int crypt_len, const u8 *assoc, - int assoc_len, const u8 iv[GCM_AES_IV_SIZE], - const u8 *authtag) -{ - u8 tagbuf[AES_BLOCK_SIZE]; - __be32 ctr[4]; - - memcpy(ctr, iv, GCM_AES_IV_SIZE); - - aesgcm_mac(ctx, src, crypt_len, assoc, assoc_len, ctr, tagbuf); - if (crypto_memneq(authtag, tagbuf, ctx->authsize)) { - memzero_explicit(tagbuf, sizeof(tagbuf)); - return false; - } - aesgcm_crypt(ctx, dst, src, crypt_len, ctr); - return true; -} -EXPORT_SYMBOL(aesgcm_decrypt); - -MODULE_DESCRIPTION("Generic AES-GCM library"); -MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>"); -MODULE_LICENSE("GPL"); - -#ifdef CONFIG_CRYPTO_SELFTESTS - -/* - * Test code below. Vectors taken from crypto/testmgr.h - */ - -static const u8 __initconst ctext0[16] __nonstring = - "\x58\xe2\xfc\xce\xfa\x7e\x30\x61" - "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a"; - -static const u8 __initconst ptext1[16]; - -static const u8 __initconst ctext1[32] __nonstring = - "\x03\x88\xda\xce\x60\xb6\xa3\x92" - "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78" - "\xab\x6e\x47\xd4\x2c\xec\x13\xbd" - "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf"; - -static const u8 __initconst ptext2[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext2[80] __nonstring = - "\x42\x83\x1e\xc2\x21\x77\x74\x24" - "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" - "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" - "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" - "\x21\xd5\x14\xb2\x54\x66\x93\x1c" - "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" - "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" - "\x3d\x58\xe0\x91\x47\x3f\x59\x85" - "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6" - "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4"; - -static const u8 __initconst ptext3[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext3[76] __nonstring = - "\x42\x83\x1e\xc2\x21\x77\x74\x24" - "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" - "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" - "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" - "\x21\xd5\x14\xb2\x54\x66\x93\x1c" - "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" - "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" - "\x3d\x58\xe0\x91" - "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb" - "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47"; - -static const u8 __initconst ctext4[16] __nonstring = - "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b" - "\xa0\x0e\xd1\xf3\x12\x57\x24\x35"; - -static const u8 __initconst ctext5[32] __nonstring = - "\x98\xe7\x24\x7c\x07\xf0\xfe\x41" - "\x1c\x26\x7e\x43\x84\xb0\xf6\x00" - "\x2f\xf5\x8d\x80\x03\x39\x27\xab" - "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb"; - -static const u8 __initconst ptext6[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext6[80] __nonstring = - "\x39\x80\xca\x0b\x3c\x00\xe8\x41" - "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" - "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" - "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" - "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" - "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" - "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" - "\xcc\xda\x27\x10\xac\xad\xe2\x56" - "\x99\x24\xa7\xc8\x58\x73\x36\xbf" - "\xb1\x18\x02\x4d\xb8\x67\x4a\x14"; - -static const u8 __initconst ctext7[16] __nonstring = - "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9" - "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b"; - -static const u8 __initconst ctext8[32] __nonstring = - "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e" - "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18" - "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0" - "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19"; - -static const u8 __initconst ptext9[64] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39\x1a\xaf\xd2\x55"; - -static const u8 __initconst ctext9[80] __nonstring = - "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" - "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" - "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" - "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" - "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" - "\xa7\xb0\x8b\x10\x56\x82\x88\x38" - "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" - "\xbc\xc9\xf6\x62\x89\x80\x15\xad" - "\xb0\x94\xda\xc5\xd9\x34\x71\xbd" - "\xec\x1a\x50\x22\x70\xe3\xcc\x6c"; - -static const u8 __initconst ptext10[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext10[76] __nonstring = - "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" - "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" - "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" - "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" - "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" - "\xa7\xb0\x8b\x10\x56\x82\x88\x38" - "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" - "\xbc\xc9\xf6\x62" - "\x76\xfc\x6e\xce\x0f\x4e\x17\x68" - "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b"; - -static const u8 __initconst ptext11[60] __nonstring = - "\xd9\x31\x32\x25\xf8\x84\x06\xe5" - "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" - "\x86\xa7\xa9\x53\x15\x34\xf7\xda" - "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" - "\x1c\x3c\x0c\x95\x95\x68\x09\x53" - "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" - "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" - "\xba\x63\x7b\x39"; - -static const u8 __initconst ctext11[76] __nonstring = - "\x39\x80\xca\x0b\x3c\x00\xe8\x41" - "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" - "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" - "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" - "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" - "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" - "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" - "\xcc\xda\x27\x10" - "\x25\x19\x49\x8e\x80\xf1\x47\x8f" - "\x37\xba\x55\xbd\x6d\x27\x61\x8c"; - -static const u8 __initconst ptext12[719] __nonstring = - "\x42\xc1\xcc\x08\x48\x6f\x41\x3f" - "\x2f\x11\x66\x8b\x2a\x16\xf0\xe0" - "\x58\x83\xf0\xc3\x70\x14\xc0\x5b" - "\x3f\xec\x1d\x25\x3c\x51\xd2\x03" - "\xcf\x59\x74\x1f\xb2\x85\xb4\x07" - "\xc6\x6a\x63\x39\x8a\x5b\xde\xcb" - "\xaf\x08\x44\xbd\x6f\x91\x15\xe1" - "\xf5\x7a\x6e\x18\xbd\xdd\x61\x50" - "\x59\xa9\x97\xab\xbb\x0e\x74\x5c" - "\x00\xa4\x43\x54\x04\x54\x9b\x3b" - "\x77\xec\xfd\x5c\xa6\xe8\x7b\x08" - "\xae\xe6\x10\x3f\x32\x65\xd1\xfc" - "\xa4\x1d\x2c\x31\xfb\x33\x7a\xb3" - "\x35\x23\xf4\x20\x41\xd4\xad\x82" - "\x8b\xa4\xad\x96\x1c\x20\x53\xbe" - "\x0e\xa6\xf4\xdc\x78\x49\x3e\x72" - "\xb1\xa9\xb5\x83\xcb\x08\x54\xb7" - "\xad\x49\x3a\xae\x98\xce\xa6\x66" - "\x10\x30\x90\x8c\x55\x83\xd7\x7c" - "\x8b\xe6\x53\xde\xd2\x6e\x18\x21" - "\x01\x52\xd1\x9f\x9d\xbb\x9c\x73" - "\x57\xcc\x89\x09\x75\x9b\x78\x70" - "\xed\x26\x97\x4d\xb4\xe4\x0c\xa5" - "\xfa\x70\x04\x70\xc6\x96\x1c\x7d" - "\x54\x41\x77\xa8\xe3\xb0\x7e\x96" - "\x82\xd9\xec\xa2\x87\x68\x55\xf9" - "\x8f\x9e\x73\x43\x47\x6a\x08\x36" - "\x93\x67\xa8\x2d\xde\xac\x41\xa9" - "\x5c\x4d\x73\x97\x0f\x70\x68\xfa" - "\x56\x4d\x00\xc2\x3b\x1f\xc8\xb9" - "\x78\x1f\x51\x07\xe3\x9a\x13\x4e" - "\xed\x2b\x2e\xa3\xf7\x44\xb2\xe7" - "\xab\x19\x37\xd9\xba\x76\x5e\xd2" - "\xf2\x53\x15\x17\x4c\x6b\x16\x9f" - "\x02\x66\x49\xca\x7c\x91\x05\xf2" - "\x45\x36\x1e\xf5\x77\xad\x1f\x46" - "\xa8\x13\xfb\x63\xb6\x08\x99\x63" - "\x82\xa2\xed\xb3\xac\xdf\x43\x19" - "\x45\xea\x78\x73\xd9\xb7\x39\x11" - "\xa3\x13\x7c\xf8\x3f\xf7\xad\x81" - "\x48\x2f\xa9\x5c\x5f\xa0\xf0\x79" - "\xa4\x47\x7d\x80\x20\x26\xfd\x63" - "\x0a\xc7\x7e\x6d\x75\x47\xff\x76" - "\x66\x2e\x8a\x6c\x81\x35\xaf\x0b" - "\x2e\x6a\x49\x60\xc1\x10\xe1\xe1" - "\x54\x03\xa4\x09\x0c\x37\x7a\x15" - "\x23\x27\x5b\x8b\x4b\xa5\x64\x97" - "\xae\x4a\x50\x73\x1f\x66\x1c\x5c" - "\x03\x25\x3c\x8d\x48\x58\x71\x34" - "\x0e\xec\x4e\x55\x1a\x03\x6a\xe5" - "\xb6\x19\x2b\x84\x2a\x20\xd1\xea" - "\x80\x6f\x96\x0e\x05\x62\xc7\x78" - "\x87\x79\x60\x38\x46\xb4\x25\x57" - "\x6e\x16\x63\xf8\xad\x6e\xd7\x42" - "\x69\xe1\x88\xef\x6e\xd5\xb4\x9a" - "\x3c\x78\x6c\x3b\xe5\xa0\x1d\x22" - "\x86\x5c\x74\x3a\xeb\x24\x26\xc7" - "\x09\xfc\x91\x96\x47\x87\x4f\x1a" - "\xd6\x6b\x2c\x18\x47\xc0\xb8\x24" - "\xa8\x5a\x4a\x9e\xcb\x03\xe7\x2a" - "\x09\xe6\x4d\x9c\x6d\x86\x60\xf5" - "\x2f\x48\x69\x37\x9f\xf2\xd2\xcb" - "\x0e\x5a\xdd\x6e\x8a\xfb\x6a\xfe" - "\x0b\x63\xde\x87\x42\x79\x8a\x68" - "\x51\x28\x9b\x7a\xeb\xaf\xb8\x2f" - "\x9d\xd1\xc7\x45\x90\x08\xc9\x83" - "\xe9\x83\x84\xcb\x28\x69\x09\x69" - "\xce\x99\x46\x00\x54\xcb\xd8\x38" - "\xf9\x53\x4a\xbf\x31\xce\x57\x15" - "\x33\xfa\x96\x04\x33\x42\xe3\xc0" - "\xb7\x54\x4a\x65\x7a\x7c\x02\xe6" - "\x19\x95\xd0\x0e\x82\x07\x63\xf9" - "\xe1\x2b\x2a\xfc\x55\x92\x52\xc9" - "\xb5\x9f\x23\x28\x60\xe7\x20\x51" - "\x10\xd3\xed\x6d\x9b\xab\xb8\xe2" - "\x5d\x9a\x34\xb3\xbe\x9c\x64\xcb" - "\x78\xc6\x91\x22\x40\x91\x80\xbe" - "\xd7\x78\x5c\x0e\x0a\xdc\x08\xe9" - "\x67\x10\xa4\x83\x98\x79\x23\xe7" - "\x92\xda\xa9\x22\x16\xb1\xe7\x78" - "\xa3\x1c\x6c\x8f\x35\x7c\x4d\x37" - "\x2f\x6e\x0b\x50\x5c\x34\xb9\xf9" - "\xe6\x3d\x91\x0d\x32\x95\xaa\x3d" - "\x48\x11\x06\xbb\x2d\xf2\x63\x88" - "\x3f\x73\x09\xe2\x45\x56\x31\x51" - "\xfa\x5e\x4e\x62\xf7\x90\xf9\xa9" - "\x7d\x7b\x1b\xb1\xc8\x26\x6e\x66" - "\xf6\x90\x9a\x7f\xf2\x57\xcc\x23" - "\x59\xfa\xfa\xaa\x44\x04\x01\xa7" - "\xa4\x78\xdb\x74\x3d\x8b\xb5"; - -static const u8 __initconst ctext12[735] __nonstring = - "\x84\x0b\xdb\xd5\xb7\xa8\xfe\x20" - "\xbb\xb1\x12\x7f\x41\xea\xb3\xc0" - "\xa2\xb4\x37\x19\x11\x58\xb6\x0b" - "\x4c\x1d\x38\x05\x54\xd1\x16\x73" - "\x8e\x1c\x20\x90\xa2\x9a\xb7\x74" - "\x47\xe6\xd8\xfc\x18\x3a\xb4\xea" - "\xd5\x16\x5a\x2c\x53\x01\x46\xb3" - "\x18\x33\x74\x6c\x50\xf2\xe8\xc0" - "\x73\xda\x60\x22\xeb\xe3\xe5\x9b" - "\x20\x93\x6c\x4b\x37\x99\xb8\x23" - "\x3b\x4e\xac\xe8\x5b\xe8\x0f\xb7" - "\xc3\x8f\xfb\x4a\x37\xd9\x39\x95" - "\x34\xf1\xdb\x8f\x71\xd9\xc7\x0b" - "\x02\xf1\x63\xfc\x9b\xfc\xc5\xab" - "\xb9\x14\x13\x21\xdf\xce\xaa\x88" - "\x44\x30\x1e\xce\x26\x01\x92\xf8" - "\x9f\x00\x4b\x0c\x4b\xf7\x5f\xe0" - "\x89\xca\x94\x66\x11\x21\x97\xca" - "\x3e\x83\x74\x2d\xdb\x4d\x11\xeb" - "\x97\xc2\x14\xff\x9e\x1e\xa0\x6b" - "\x08\xb4\x31\x2b\x85\xc6\x85\x6c" - "\x90\xec\x39\xc0\xec\xb3\xb5\x4e" - "\xf3\x9c\xe7\x83\x3a\x77\x0a\xf4" - "\x56\xfe\xce\x18\x33\x6d\x0b\x2d" - "\x33\xda\xc8\x05\x5c\xb4\x09\x2a" - "\xde\x6b\x52\x98\x01\xef\x36\x3d" - "\xbd\xf9\x8f\xa8\x3e\xaa\xcd\xd1" - "\x01\x2d\x42\x49\xc3\xb6\x84\xbb" - "\x48\x96\xe0\x90\x93\x6c\x48\x64" - "\xd4\xfa\x7f\x93\x2c\xa6\x21\xc8" - "\x7a\x23\x7b\xaa\x20\x56\x12\xae" - "\x16\x9d\x94\x0f\x54\xa1\xec\xca" - "\x51\x4e\xf2\x39\xf4\xf8\x5f\x04" - "\x5a\x0d\xbf\xf5\x83\xa1\x15\xe1" - "\xf5\x3c\xd8\x62\xa3\xed\x47\x89" - "\x85\x4c\xe5\xdb\xac\x9e\x17\x1d" - "\x0c\x09\xe3\x3e\x39\x5b\x4d\x74" - "\x0e\xf5\x34\xee\x70\x11\x4c\xfd" - "\xdb\x34\xb1\xb5\x10\x3f\x73\xb7" - "\xf5\xfa\xed\xb0\x1f\xa5\xcd\x3c" - "\x8d\x35\x83\xd4\x11\x44\x6e\x6c" - "\x5b\xe0\x0e\x69\xa5\x39\xe5\xbb" - "\xa9\x57\x24\x37\xe6\x1f\xdd\xcf" - "\x16\x2a\x13\xf9\x6a\x2d\x90\xa0" - "\x03\x60\x7a\xed\x69\xd5\x00\x8b" - "\x7e\x4f\xcb\xb9\xfa\x91\xb9\x37" - "\xc1\x26\xce\x90\x97\x22\x64\x64" - "\xc1\x72\x43\x1b\xf6\xac\xc1\x54" - "\x8a\x10\x9c\xdd\x8d\xd5\x8e\xb2" - "\xe4\x85\xda\xe0\x20\x5f\xf4\xb4" - "\x15\xb5\xa0\x8d\x12\x74\x49\x23" - "\x3a\xdf\x4a\xd3\xf0\x3b\x89\xeb" - "\xf8\xcc\x62\x7b\xfb\x93\x07\x41" - "\x61\x26\x94\x58\x70\xa6\x3c\xe4" - "\xff\x58\xc4\x13\x3d\xcb\x36\x6b" - "\x32\xe5\xb2\x6d\x03\x74\x6f\x76" - "\x93\x77\xde\x48\xc4\xfa\x30\x4a" - "\xda\x49\x80\x77\x0f\x1c\xbe\x11" - "\xc8\x48\xb1\xe5\xbb\xf2\x8a\xe1" - "\x96\x2f\x9f\xd1\x8e\x8a\x5c\xe2" - "\xf7\xd7\xd8\x54\xf3\x3f\xc4\x91" - "\xb8\xfb\x86\xdc\x46\x24\x91\x60" - "\x6c\x2f\xc9\x41\x37\x51\x49\x54" - "\x09\x81\x21\xf3\x03\x9f\x2b\xe3" - "\x1f\x39\x63\xaf\xf4\xd7\x53\x60" - "\xa7\xc7\x54\xf9\xee\xb1\xb1\x7d" - "\x75\x54\x65\x93\xfe\xb1\x68\x6b" - "\x57\x02\xf9\xbb\x0e\xf9\xf8\xbf" - "\x01\x12\x27\xb4\xfe\xe4\x79\x7a" - "\x40\x5b\x51\x4b\xdf\x38\xec\xb1" - "\x6a\x56\xff\x35\x4d\x42\x33\xaa" - "\x6f\x1b\xe4\xdc\xe0\xdb\x85\x35" - "\x62\x10\xd4\xec\xeb\xc5\x7e\x45" - "\x1c\x6f\x17\xca\x3b\x8e\x2d\x66" - "\x4f\x4b\x36\x56\xcd\x1b\x59\xaa" - "\xd2\x9b\x17\xb9\x58\xdf\x7b\x64" - "\x8a\xff\x3b\x9c\xa6\xb5\x48\x9e" - "\xaa\xe2\x5d\x09\x71\x32\x5f\xb6" - "\x29\xbe\xe7\xc7\x52\x7e\x91\x82" - "\x6b\x6d\x33\xe1\x34\x06\x36\x21" - "\x5e\xbe\x1e\x2f\x3e\xc1\xfb\xea" - "\x49\x2c\xb5\xca\xf7\xb0\x37\xea" - "\x1f\xed\x10\x04\xd9\x48\x0d\x1a" - "\x1c\xfb\xe7\x84\x0e\x83\x53\x74" - "\xc7\x65\xe2\x5c\xe5\xba\x73\x4c" - "\x0e\xe1\xb5\x11\x45\x61\x43\x46" - "\xaa\x25\x8f\xbd\x85\x08\xfa\x4c" - "\x15\xc1\xc0\xd8\xf5\xdc\x16\xbb" - "\x7b\x1d\xe3\x87\x57\xa7\x2a\x1d" - "\x38\x58\x9e\x8a\x43\xdc\x57" - "\xd1\x81\x7d\x2b\xe9\xff\x99\x3a" - "\x4b\x24\x52\x58\x55\xe1\x49\x14"; - -static struct { - const u8 *ptext; - const u8 *ctext; - - u8 key[AES_MAX_KEY_SIZE] __nonstring; - u8 iv[GCM_AES_IV_SIZE] __nonstring; - u8 assoc[20] __nonstring; - - int klen; - int clen; - int plen; - int alen; -} const aesgcm_tv[] __initconst = { - { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ - .klen = 16, - .ctext = ctext0, - .clen = sizeof(ctext0), - }, { - .klen = 16, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext1, - .clen = sizeof(ctext1), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 16, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext2, - .plen = sizeof(ptext2), - .ctext = ctext2, - .clen = sizeof(ctext2), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 16, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext3, - .plen = sizeof(ptext3), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext3, - .clen = sizeof(ctext3), - }, { - .klen = 24, - .ctext = ctext4, - .clen = sizeof(ctext4), - }, { - .klen = 24, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext5, - .clen = sizeof(ctext5), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c", - .klen = 24, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext6, - .plen = sizeof(ptext6), - .ctext = ctext6, - .clen = sizeof(ctext6), - }, { - .klen = 32, - .ctext = ctext7, - .clen = sizeof(ctext7), - }, { - .klen = 32, - .ptext = ptext1, - .plen = sizeof(ptext1), - .ctext = ctext8, - .clen = sizeof(ctext8), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 32, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext9, - .plen = sizeof(ptext9), - .ctext = ctext9, - .clen = sizeof(ctext9), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08", - .klen = 32, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext10, - .plen = sizeof(ptext10), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext10, - .clen = sizeof(ctext10), - }, { - .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" - "\x6d\x6a\x8f\x94\x67\x30\x83\x08" - "\xfe\xff\xe9\x92\x86\x65\x73\x1c", - .klen = 24, - .iv = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" - "\xde\xca\xf8\x88", - .ptext = ptext11, - .plen = sizeof(ptext11), - .assoc = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xfe\xed\xfa\xce\xde\xad\xbe\xef" - "\xab\xad\xda\xd2", - .alen = 20, - .ctext = ctext11, - .clen = sizeof(ctext11), - }, { - .key = "\x62\x35\xf8\x95\xfc\xa5\xeb\xf6" - "\x0e\x92\x12\x04\xd3\xa1\x3f\x2e" - "\x8b\x32\xcf\xe7\x44\xed\x13\x59" - "\x04\x38\x77\xb0\xb9\xad\xb4\x38", - .klen = 32, - .iv = "\x00\xff\xff\xff\xff\x00\x00\xff" - "\xff\xff\x00\xff", - .ptext = ptext12, - .plen = sizeof(ptext12), - .ctext = ctext12, - .clen = sizeof(ctext12), - } -}; - -static int __init libaesgcm_init(void) -{ - for (int i = 0; i < ARRAY_SIZE(aesgcm_tv); i++) { - u8 tagbuf[AES_BLOCK_SIZE]; - int plen = aesgcm_tv[i].plen; - struct aesgcm_ctx ctx; - static u8 buf[sizeof(ptext12)]; - - if (aesgcm_expandkey(&ctx, aesgcm_tv[i].key, aesgcm_tv[i].klen, - aesgcm_tv[i].clen - plen)) { - pr_err("aesgcm_expandkey() failed on vector %d\n", i); - return -ENODEV; - } - - if (!aesgcm_decrypt(&ctx, buf, aesgcm_tv[i].ctext, plen, - aesgcm_tv[i].assoc, aesgcm_tv[i].alen, - aesgcm_tv[i].iv, aesgcm_tv[i].ctext + plen) - || memcmp(buf, aesgcm_tv[i].ptext, plen)) { - pr_err("aesgcm_decrypt() #1 failed on vector %d\n", i); - return -ENODEV; - } - - /* encrypt in place */ - aesgcm_encrypt(&ctx, buf, buf, plen, aesgcm_tv[i].assoc, - aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf); - if (memcmp(buf, aesgcm_tv[i].ctext, plen)) { - pr_err("aesgcm_encrypt() failed on vector %d\n", i); - return -ENODEV; - } - - /* decrypt in place */ - if (!aesgcm_decrypt(&ctx, buf, buf, plen, aesgcm_tv[i].assoc, - aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf) - || memcmp(buf, aesgcm_tv[i].ptext, plen)) { - pr_err("aesgcm_decrypt() #2 failed on vector %d\n", i); - return -ENODEV; - } - } - return 0; -} -module_init(libaesgcm_init); - -static void __exit libaesgcm_exit(void) -{ -} -module_exit(libaesgcm_exit); -#endif diff --git a/lib/crypto/fips-aes.h b/lib/crypto/fips-aes.h new file mode 100644 index 000000000000..2a1746606533 --- /dev/null +++ b/lib/crypto/fips-aes.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: gen-fips-testvecs.py */ +/* clang-format off */ + +#include <linux/fips.h> + +static const u8 fips_test_data[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, +}; + +static const u8 fips_test_ad[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x61, 0x64, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_iv[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x69, 0x76, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_key[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x6b, 0x65, 0x79, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_xts_key[] __initconst __maybe_unused = { + 0x6b, 0x65, 0x79, 0x31, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = { + 0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6, + 0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18, +}; + +static const u8 fips_test_aes_ecb_ctext[] __initconst __maybe_unused = { + 0x47, 0x76, 0x48, 0xaf, 0x1b, 0xd8, 0x4c, 0xe6, + 0xb5, 0xa7, 0x20, 0x8d, 0x64, 0x88, 0xbc, 0x3f, +}; + +static const u8 fips_test_aes_cbc_ctext[] __initconst __maybe_unused = { + 0xc8, 0x7d, 0x7c, 0x25, 0xba, 0x15, 0xf7, 0xe1, + 0x08, 0xa0, 0xd0, 0x7a, 0x20, 0x37, 0xaf, 0x5e, +}; + +static const u8 fips_test_aes_cbc_cts_ctext[] __initconst __maybe_unused = { + 0x36, 0x8e, 0x37, 0xb4, 0x78, 0xe2, 0x88, 0x59, + 0xd5, 0xe8, 0x17, 0x65, 0x5c, 0xa1, 0x25, 0xe6, + 0xc8, 0x7d, 0x7c, 0x25, 0xba, 0x15, 0xf7, 0xe1, + 0x08, 0xa0, 0xd0, 0x7a, 0x20, 0x37, 0xaf, 0x5e, +}; + +static const u8 fips_test_aes_ctr_ctext[] __initconst __maybe_unused = { + 0x95, 0xf4, 0xf4, 0x7a, 0xc8, 0xa2, 0x53, 0x73, + 0x53, 0x8f, 0x95, 0xfc, 0x18, 0xfe, 0x58, 0x2f, +}; + +static const u8 fips_test_aes_xts_ctext[] __initconst __maybe_unused = { + 0xd4, 0x51, 0x7f, 0x01, 0x14, 0x91, 0x16, 0x29, + 0x26, 0xbe, 0xec, 0x9b, 0x90, 0xed, 0x59, 0x30, +}; + +static const u8 fips_test_aes_gcm_ctext_and_tag[] __initconst __maybe_unused = { + 0x12, 0x0c, 0x5d, 0x03, 0x32, 0x93, 0x13, 0x44, + 0x06, 0x35, 0x26, 0x9d, 0xe0, 0xea, 0xbc, 0xe2, + 0x30, 0xa9, 0xa4, 0x15, 0xc5, 0x3d, 0xb3, 0xf9, + 0x30, 0x82, 0xdf, 0x9c, 0xd8, 0xc4, 0x3f, 0x2f, +}; + +static const u8 fips_test_aes_ccm_ctext_and_tag[] __initconst __maybe_unused = { + 0x11, 0x8e, 0x01, 0xcb, 0xb5, 0x22, 0x6d, 0xb4, + 0x66, 0x98, 0x97, 0x1d, 0x35, 0x53, 0x78, 0xdd, + 0xd1, 0xc5, 0xff, 0xb6, 0x90, 0xcf, 0xb1, 0xf2, + 0x87, 0x99, 0xd6, 0x1e, 0xd5, 0xd1, 0xed, 0x63, +}; diff --git a/lib/crypto/fips.h b/lib/crypto/fips-sha.h index 9fc49747db64..68af7e14e09c 100644 --- a/lib/crypto/fips.h +++ b/lib/crypto/fips-sha.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* This file was generated by: gen-fips-testvecs.py */ +/* clang-format off */ #include <linux/fips.h> @@ -43,8 +44,3 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = { 0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00, 0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82, }; - -static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = { - 0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6, - 0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18, -}; diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c index 6bf130cfbbf9..3d2b017a0525 100644 --- a/lib/crypto/md5.c +++ b/lib/crypto/md5.c @@ -298,19 +298,5 @@ void hmac_md5_usingrawkey(const u8 *raw_key, size_t raw_key_len, } EXPORT_SYMBOL_GPL(hmac_md5_usingrawkey); -#ifdef md5_mod_init_arch -static int __init md5_mod_init(void) -{ - md5_mod_init_arch(); - return 0; -} -subsys_initcall(md5_mod_init); - -static void __exit md5_mod_exit(void) -{ -} -module_exit(md5_mod_exit); -#endif - MODULE_DESCRIPTION("MD5 and HMAC-MD5 library functions"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index daf18c862fdf..b687b89d97cb 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -12,7 +12,7 @@ #include <linux/string.h> #include <linux/unaligned.h> #include <linux/wordpart.h> -#include "fips.h" +#include "fips-sha.h" static const struct sha1_block_state sha1_iv = { .h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 5d6b77e7e141..e8c346f563c5 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -17,7 +17,7 @@ #include <linux/string.h> #include <linux/unaligned.h> #include <linux/wordpart.h> -#include "fips.h" +#include "fips-sha.h" static const struct sha256_block_state sha224_iv = { .h = { diff --git a/lib/crypto/sha3.c b/lib/crypto/sha3.c index 32b7074de792..286a2373c156 100644 --- a/lib/crypto/sha3.c +++ b/lib/crypto/sha3.c @@ -17,7 +17,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/unaligned.h> -#include "fips.h" +#include "fips-sha.h" /* * On some 32-bit architectures, such as h8300, GCC ends up using over 1 KB of diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c index 605eab51aabd..0dd6fb4ca15b 100644 --- a/lib/crypto/sha512.c +++ b/lib/crypto/sha512.c @@ -17,7 +17,7 @@ #include <linux/string.h> #include <linux/unaligned.h> #include <linux/wordpart.h> -#include "fips.h" +#include "fips-sha.h" static const struct sha512_block_state sha384_iv = { .h = { diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 9409c1a935c3..e121114624da 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -9,6 +9,24 @@ config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message authentication codes. +config CRYPTO_LIB_AES_CCM_KUNIT_TEST + tristate "KUnit tests for AES-CCM" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_CCM + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + # This test uses BLAKE2s, but that's always built-in. + help + KUnit tests for the AES-CCM authenticated encryption algorithm. + +config CRYPTO_LIB_AES_GCM_KUNIT_TEST + tristate "KUnit tests for AES-GCM" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_GCM + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + # This test uses BLAKE2s, but that's always built-in. + help + KUnit tests for the AES-GCM authenticated encryption algorithm. + config CRYPTO_LIB_BLAKE2B_KUNIT_TEST tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_BLAKE2B @@ -144,7 +162,13 @@ config CRYPTO_LIB_SM3_KUNIT_TEST config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT tristate "Enable all crypto library code for KUnit tests" depends on KUNIT + select CRYPTO_LIB_AES_CBC select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_AES_CCM + select CRYPTO_LIB_AES_CTR + select CRYPTO_LIB_AES_ECB + select CRYPTO_LIB_AES_GCM + select CRYPTO_LIB_AES_XTS select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CHACHA20POLY1305 select CRYPTO_LIB_CURVE25519 diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index a739413500b6..c97c1d78784a 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-or-later obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o +obj-$(CONFIG_CRYPTO_LIB_AES_CCM_KUNIT_TEST) += aes_ccm_kunit.o +obj-$(CONFIG_CRYPTO_LIB_AES_GCM_KUNIT_TEST) += aes_gcm_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o diff --git a/lib/crypto/tests/aead-test-template.h b/lib/crypto/tests/aead-test-template.h new file mode 100644 index 000000000000..0c0138d2fa8a --- /dev/null +++ b/lib/crypto/tests/aead-test-template.h @@ -0,0 +1,1039 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Shared KUnit test cases for AEAD algorithms, including a benchmark + * + * Copyright 2026 Google LLC + */ + +/* + * This file implements KUnit test cases shared by the different KUnit test + * suites for Authenticated Encryption with Associated Data (AEAD) algorithms. + * + * Test suites including this file must #define the following: + * + * Data structs: + * - AEAD_KEY: name of key struct + * - AEAD_CTX: name of context for incremental computation + * + * Constants: + * - AEAD_VALID_KEY_LENS: array of all valid key lengths in bytes + * - AEAD_VALID_NONCE_LENS: array of all valid nonce lengths in bytes + * - AEAD_VALID_TAG_LENS: array of all valid authtag lengths in bytes + * - AEAD_MAX_KEY_LEN: max key length in bytes (assumed to fit on stack) + * - AEAD_MAX_NONCE_LEN: max nonce length in bytes (assumed to fit on stack) + * - AEAD_MAX_TAG_LEN: max authtag length in bytes (assumed to fit on stack) + * - AEAD_MONTE_CARLO_CHECKSUM: checksum of a deterministically generated series + * of (ciphertext, authtag) pairs (see test_aead_monte_carlo()) + * + * Functions: + * - AEAD_PREPAREKEY: key preparation + * - AEAD_ENCRYPT and AEAD_DECRYPT: one-shot encryption and decryption + * - AEAD_INIT, AEAD_AUTH_UPDATE, AEAD_ENCRYPT_UPDATE, AEAD_ENCRYPT_FINAL, + * AEAD_DECRYPT_UPDATE, AEAD_DECRYPT_FINAL: functions for incremental + * encryption and decryption + * + * Function prototypes and their behavior must match the AES-CCM API. + */ + +#include <crypto/blake2s.h> +#include <kunit/run-in-irq-context.h> +#include <kunit/test.h> +#include <linux/ktime.h> +#include <linux/preempt.h> +#include "test-utils.h" + +/* + * Allocate a KUnit-managed struct AEAD_KEY and prepare it with a random key, + * using a random key length and random authentication tag length. + */ +static struct AEAD_KEY *aead_alloc_random_key(struct kunit *test, + size_t *tag_len_ret) +{ + size_t key_len = + AEAD_VALID_KEY_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_KEY_LENS)]; + size_t tag_len = + AEAD_VALID_TAG_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_TAG_LENS)]; + u8 raw_key[AEAD_MAX_KEY_LEN]; + struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); + int err; + + rand_bytes(raw_key, key_len); + err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + *tag_len_ret = tag_len; + return key; +} + +/* + * Allocate a KUnit-managed slab buffer of length @len bytes and initialize it + * with random data. + */ +static u8 *aead_alloc_random_data(struct kunit *test, size_t len) +{ + u8 *buf = alloc_buf(test, len); + + rand_bytes(buf, len); + return buf; +} + +/* + * Allocate a KUnit-managed guarded buffer of length @len bytes and initialize + * it with random data. + */ +static u8 *aead_alloc_random_data_guarded(struct kunit *test, size_t len) +{ + u8 *buf = alloc_guarded_buf(test, len); + + rand_bytes(buf, len); + return buf; +} + +/* Process the given associated data using a random incremental strategy. */ +static size_t aead_auth_incrementally(struct AEAD_CTX *ctx, const u8 *ad, + size_t ad_len) +{ + size_t num_parts = 0; + size_t pos = 0; + + while (rand_bool()) { + size_t part_len = rand_length(ad_len - pos); + + AEAD_AUTH_UPDATE(ctx, &ad[pos], part_len); + pos += part_len; + num_parts++; + } + if (pos < ad_len || rand_bool()) { + AEAD_AUTH_UPDATE(ctx, &ad[pos], ad_len - pos); + num_parts++; + } + return num_parts; +} + +/* Process the given en/decrypted data using a random incremental strategy. */ +static size_t aead_crypt_incrementally(struct AEAD_CTX *ctx, u8 *dst, + const u8 *src, size_t data_len, bool enc) +{ + size_t num_parts = 0; + size_t pos = 0; + + while (rand_bool()) { + size_t part_len = rand_length(data_len - pos); + + if (enc) + AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos], + part_len); + else + AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos], + part_len); + pos += part_len; + num_parts++; + } + if (pos < data_len || rand_bool()) { + if (enc) + AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos], + data_len - pos); + else + AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos], + data_len - pos); + num_parts++; + } + return num_parts; +} + +struct aead_incremental_info { + size_t num_data_parts; + size_t num_ad_parts; +}; + +static const char *aead_incr_info_str(struct kunit *test, + const struct aead_incremental_info *info) +{ + const size_t max_str_len = 64; + char *str = alloc_buf(test, max_str_len); + + snprintf(str, max_str_len, "num_data_parts=%zu num_ad_parts=%zu", + info->num_data_parts, info->num_ad_parts); + return str; +} + +/* + * Encrypt data using a random incremental strategy. + * Return information about the incremental strategy used. + */ +static struct aead_incremental_info +aead_encrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst, + const u8 *src, size_t data_len, u8 *tag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct AEAD_KEY *key) +{ + struct aead_incremental_info info; + int err; + + err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len); + info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len, + /* enc= */ true); + AEAD_ENCRYPT_FINAL(ctx, tag); + KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)), + "encrypt_final didn't zeroize context"); + return info; +} + +/* + * Decrypt authentic data using a random incremental strategy. + * Return information about the incremental strategy used. + */ +static struct aead_incremental_info +aead_decrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst, + const u8 *src, size_t data_len, const u8 *tag, + const u8 *ad, size_t ad_len, const u8 *nonce, + size_t nonce_len, const struct AEAD_KEY *key) +{ + struct aead_incremental_info info; + int err; + + err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len); + info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len, + /* enc= */ false); + err = AEAD_DECRYPT_FINAL(ctx, tag); + KUNIT_ASSERT_EQ(test, 0, err); + KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)), + "decrypt_final didn't zeroize context"); + return info; +} + +/* Return true if key_len is declared to be a valid key length. */ +static bool aead_is_key_len_expected_valid(size_t key_len) +{ + for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_KEY_LENS); i++) { + if (AEAD_VALID_KEY_LENS[i] == key_len) + return true; + } + return false; +} + +/* Return true if nonce_len is declared to be a valid nonce length. */ +static bool aead_is_nonce_len_expected_valid(size_t nonce_len) +{ + for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_NONCE_LENS); i++) { + if (AEAD_VALID_NONCE_LENS[i] == nonce_len) + return true; + } + return false; +} + +/* Return true if tag_len is declared to be a valid tag length. */ +static bool aead_is_tag_len_expected_valid(size_t tag_len) +{ + for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_TAG_LENS); i++) { + if (AEAD_VALID_TAG_LENS[i] == tag_len) + return true; + } + return false; +} + +struct aead_basic_validation_test_ctx { + struct AEAD_KEY key; + struct AEAD_CTX ctx; + u8 *raw_key_buf_end; + u8 *nonce_buf_end; + u8 *tag_buf_end; + u8 pt[64]; /* plaintext */ + u8 ct[64]; /* ciphertext */ + u8 decrypted[64]; + u8 ad[16]; /* associated data */ + u8 *unused_buf; + size_t data_len; + size_t ad_len; +}; + +static struct aead_basic_validation_test_ctx * +aead_alloc_basic_validation_test_ctx(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + alloc_buf(test, sizeof(*ctx)); + + memset(ctx, 0, sizeof(*ctx)); + ctx->raw_key_buf_end = + aead_alloc_random_data_guarded(test, AEAD_MAX_KEY_LEN) + + AEAD_MAX_KEY_LEN; + ctx->nonce_buf_end = + aead_alloc_random_data_guarded(test, AEAD_MAX_NONCE_LEN) + + AEAD_MAX_NONCE_LEN; + ctx->tag_buf_end = + aead_alloc_random_data_guarded(test, AEAD_MAX_TAG_LEN) + + AEAD_MAX_TAG_LEN; + + /* + * A pointer to this buffer is passed when passing a length that is + * expected to be invalid. It should never actually be accessed. + */ + ctx->unused_buf = + alloc_buf(test, max3(AEAD_MAX_KEY_LEN, AEAD_MAX_NONCE_LEN, + AEAD_MAX_TAG_LEN)); + + ctx->data_len = sizeof(ctx->pt); + ctx->ad_len = sizeof(ctx->ad); + return ctx; +} + +/* + * Given an expected-valid key_len, nonce_len, and tag_len, verify round-trip + * encryption and decryption with them. Use guarded buffers for each of the raw + * key, nonce, and tag to detect any buffer overruns in them. Also, verify that + * every byte of the tag is actually checked. + */ +static void aead_do_basic_checks(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t key_len, size_t nonce_len, + size_t tag_len) +{ + /* Set up exact-size guarded buffers for (raw_key, nonce, tag). */ + const u8 *raw_key = ctx->raw_key_buf_end - key_len; + const u8 *nonce = ctx->nonce_buf_end - nonce_len; + u8 *tag = ctx->tag_buf_end - tag_len; + int err; + + /* Key preparation should succeed. */ + err = AEAD_PREPAREKEY(&ctx->key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ_MSG(test, 0, err, + "key_len=%zu, tag_len=%zu wasn't accepted", key_len, + tag_len); + + /* Encryption should succeed. */ + err = AEAD_ENCRYPT(ctx->ct, ctx->pt, ctx->data_len, tag, ctx->ad, + ctx->ad_len, nonce, nonce_len, &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Encryption failed with key_len=%zu, nonce_len=%zu, tag_len=%zu", + key_len, nonce_len, tag_len); + + /* Decryption should succeed and give the original data. */ + err = AEAD_DECRYPT(ctx->decrypted, ctx->ct, ctx->data_len, tag, ctx->ad, + ctx->ad_len, nonce, nonce_len, &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Decryption failed with key_len=%zu, nonce_len=%zu, tag_len=%zu", + key_len, nonce_len, tag_len); + KUNIT_ASSERT_MEMEQ_MSG( + test, ctx->pt, ctx->decrypted, ctx->data_len, + "Decryption gave wrong output with key_len=%zu, nonce_len=%zu, tag_len=%zu", + key_len, nonce_len, tag_len); + + /* + * Every byte of the tag should actually be checked. + * And on authentication failure, the dst buffer should be cleared. + */ + for (size_t i = 0; i < tag_len; i++) { + memset(ctx->decrypted, 0xff, ctx->data_len); + tag[i] ^= 1; + err = AEAD_DECRYPT(ctx->decrypted, ctx->ct, ctx->data_len, tag, + ctx->ad, ctx->ad_len, nonce, nonce_len, + &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, -EBADMSG, err, + "Decryption with bad auth tag with key_len=%zu, nonce_len=%zu, tag_len=%zu didn't fail with -EBADMSG", + key_len, nonce_len, tag_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(ctx->decrypted, ctx->data_len), + "dst wasn't cleared on authentication failure"); + tag[i] ^= 1; + } +} + +/* Verify that the given expected-invalid key_len is actually rejected. */ +static void +aead_verify_invalid_key_len(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t key_len) +{ + int err; + + /* + * The preparekey function should reject the key_len. It should do so + * before writing to the key struct. + */ + memset(&ctx->key, 0, sizeof(ctx->key)); + err = AEAD_PREPAREKEY(&ctx->key, ctx->unused_buf, key_len, + AEAD_MAX_TAG_LEN); + KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, + "key_len=%zu wasn't rejected with -EINVAL", + key_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(&ctx->key, sizeof(ctx->key)), + "Key struct was written to before length validation"); +} + +/* + * Test that every valid key length is accepted and basic checks pass with it, + * and test that invalid key lengths are rejected. + */ +static void test_aead_all_key_lens(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + aead_alloc_basic_validation_test_ctx(test); + + for (size_t key_len = 0; key_len <= AEAD_MAX_KEY_LEN; key_len++) { + if (aead_is_key_len_expected_valid(key_len)) + aead_do_basic_checks(test, ctx, key_len, + AEAD_MAX_NONCE_LEN, + AEAD_MAX_TAG_LEN); + else + aead_verify_invalid_key_len(test, ctx, key_len); + } + aead_verify_invalid_key_len(test, ctx, AEAD_MAX_KEY_LEN + 1); + aead_verify_invalid_key_len(test, ctx, AEAD_MAX_KEY_LEN * 2); + aead_verify_invalid_key_len(test, ctx, U32_MAX); + aead_verify_invalid_key_len(test, ctx, SIZE_MAX); +} + +/* Verify that the given expected-invalid nonce_len is actually rejected. */ +static void +aead_verify_invalid_nonce_len(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t nonce_len) +{ + static const u8 raw_key[AEAD_MAX_KEY_LEN]; + int err; + + /* Key preparation should succeed, as nonce_len isn't given yet. */ + err = AEAD_PREPAREKEY(&ctx->key, raw_key, sizeof(raw_key), + AEAD_MAX_TAG_LEN); + KUNIT_ASSERT_EQ(test, 0, err); + + /* The init function should reject the nonce_len. */ + memset(&ctx->ctx, 0, sizeof(ctx->ctx)); + err = AEAD_INIT(&ctx->ctx, ctx->data_len, ctx->ad_len, ctx->unused_buf, + nonce_len, &ctx->key); + KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, + "nonce_len=%zu wasn't rejected with -EINVAL (init)", + nonce_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(&ctx->ctx, sizeof(ctx->ctx)), + "Context struct was written to before length validation"); + + /* The encrypt function should reject the nonce_len. */ + err = AEAD_ENCRYPT(ctx->ct, ctx->pt, ctx->data_len, ctx->unused_buf, + ctx->ad, ctx->ad_len, ctx->unused_buf, nonce_len, + &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, -EINVAL, err, + "nonce_len=%zu wasn't rejected with -EINVAL (encrypt)", + nonce_len); + + /* The decrypt function should reject the nonce_len. */ + err = AEAD_DECRYPT(ctx->pt, ctx->ct, ctx->data_len, ctx->unused_buf, + ctx->ad, ctx->ad_len, ctx->unused_buf, nonce_len, + &ctx->key); + KUNIT_ASSERT_EQ_MSG( + test, -EINVAL, err, + "nonce_len=%zu wasn't rejected with -EINVAL (decrypt)", + nonce_len); +} + +/* + * Test that every valid nonce length is accepted and basic checks pass with it, + * and test that invalid nonce lengths are rejected. + */ +static void test_aead_all_nonce_lens(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + aead_alloc_basic_validation_test_ctx(test); + + for (size_t nonce_len = 0; nonce_len <= AEAD_MAX_NONCE_LEN; + nonce_len++) { + if (aead_is_nonce_len_expected_valid(nonce_len)) + aead_do_basic_checks(test, ctx, AEAD_MAX_KEY_LEN, + nonce_len, AEAD_MAX_TAG_LEN); + else + aead_verify_invalid_nonce_len(test, ctx, nonce_len); + } + aead_verify_invalid_nonce_len(test, ctx, AEAD_MAX_NONCE_LEN + 1); + aead_verify_invalid_nonce_len(test, ctx, AEAD_MAX_NONCE_LEN * 2); + aead_verify_invalid_nonce_len(test, ctx, U32_MAX); + aead_verify_invalid_nonce_len(test, ctx, SIZE_MAX); +} + +/* Verify that the given expected-invalid tag_len is actually rejected. */ +static void +aead_verify_invalid_tag_len(struct kunit *test, + struct aead_basic_validation_test_ctx *ctx, + size_t tag_len) +{ + static const u8 raw_key[AEAD_MAX_KEY_LEN]; + int err; + + /* + * The preparekey function should reject the tag_len. It should do so + * before writing to the key struct. + */ + memset(&ctx->key, 0, sizeof(ctx->key)); + err = AEAD_PREPAREKEY(&ctx->key, raw_key, sizeof(raw_key), tag_len); + KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, + "tag_len=%zu wasn't rejected with -EINVAL", + tag_len); + KUNIT_ASSERT_TRUE_MSG( + test, mem_is_zero(&ctx->key, sizeof(ctx->key)), + "Key struct was written to before length validation"); +} + +/* + * Test that every valid authentication tag length is accepted and basic checks + * pass with it, and test that invalid authentication tag lengths are rejected. + */ +static void test_aead_all_tag_lens(struct kunit *test) +{ + struct aead_basic_validation_test_ctx *ctx = + aead_alloc_basic_validation_test_ctx(test); + + for (size_t tag_len = 0; tag_len <= AEAD_MAX_TAG_LEN; tag_len++) { + if (aead_is_tag_len_expected_valid(tag_len)) + aead_do_basic_checks(test, ctx, AEAD_MAX_KEY_LEN, + AEAD_MAX_NONCE_LEN, tag_len); + else + aead_verify_invalid_tag_len(test, ctx, tag_len); + } + aead_verify_invalid_tag_len(test, ctx, AEAD_MAX_TAG_LEN + 1); + aead_verify_invalid_tag_len(test, ctx, AEAD_MAX_TAG_LEN * 2); + aead_verify_invalid_tag_len(test, ctx, U32_MAX); + aead_verify_invalid_tag_len(test, ctx, SIZE_MAX); +} + +/* + * Test that one-shot encryption and decryption are consistent with each other + * and with incremental encryption and decryption. + */ +static void test_aead_incremental_updates(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 512; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + size_t tag_len; + struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); + struct AEAD_CTX *ctx = alloc_buf(test, sizeof(*ctx)); + u8 *pt = aead_alloc_random_data(test, max_data_len); + u8 *ad = aead_alloc_random_data(test, max_ad_len); + u8 *nonce = aead_alloc_random_data(test, nonce_len); + u8 *ct = alloc_buf(test, max_data_len); + u8 *ct2 = alloc_buf(test, max_data_len); + u8 *decrypted = alloc_buf(test, max_data_len); + u8 *tag = alloc_buf(test, tag_len); + u8 *tag2 = alloc_buf(test, tag_len); + int err; + + for (int i = 0; i < 500; i++) { + /* Select the lengths to test. */ + const size_t data_len = rand_length(max_data_len); + const size_t ad_len = rand_length(max_ad_len); + struct aead_incremental_info incr_info; + + /* Try one-shot encryption and decryption. */ + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", + data_len, ad_len); + + /* Try incremental encryption and decryption. */ + incr_info = aead_encrypt_incrementally(test, ctx, ct2, pt, + data_len, tag2, ad, + ad_len, nonce, nonce_len, + key); + KUNIT_ASSERT_MEMEQ_MSG( + test, ct, ct2, data_len, + "One-shot and incremental encryption gave different ciphertexts; data_len=%zu ad_len=%zu %s", + data_len, ad_len, aead_incr_info_str(test, &incr_info)); + KUNIT_ASSERT_MEMEQ_MSG( + test, tag, tag2, tag_len, + "One-shot and incremental encryption gave different auth tags; data_len=%zu ad_len=%zu %s", + data_len, ad_len, aead_incr_info_str(test, &incr_info)); + incr_info = aead_decrypt_incrementally(test, ctx, decrypted, + ct2, data_len, tag2, ad, + ad_len, nonce, nonce_len, + key); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "One-shot and incremental decryption gave different plaintexts; data_len=%zu ad_len=%zu %s", + data_len, ad_len, aead_incr_info_str(test, &incr_info)); + } +} + +/* + * Test using guarded buffers for the plaintext, ciphertext, and associated + * data. This detects out-of-bounds accesses, even in assembly code. + * + * Note: other test cases cover overrun of raw_key, nonce, and tag. + */ +static void test_aead_data_buffer_overruns(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 512; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + size_t tag_len; + struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); + const u8 *nonce = aead_alloc_random_data(test, nonce_len); + const u8 *pt_end = aead_alloc_random_data_guarded(test, max_data_len) + + max_data_len; + const u8 *ad_end = + aead_alloc_random_data_guarded(test, max_ad_len) + max_ad_len; + u8 *ct_end = alloc_guarded_buf(test, max_data_len) + max_data_len; + u8 *decrypted_end = + alloc_guarded_buf(test, max_data_len) + max_data_len; + u8 *tag = alloc_buf(test, tag_len); + + for (int i = 0; i < 200; i++) { + /* Select the lengths to test. */ + const size_t data_len = rand_length(max_data_len); + const size_t ad_len = rand_length(max_ad_len); + /* Set up exact-size guarded buffers. */ + const u8 *pt = pt_end - data_len; + const u8 *ad = ad_end - ad_len; + u8 *ct = ct_end - data_len; + u8 *decrypted = decrypted_end - data_len; + int err; + + /* Encrypt and decrypt. */ + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", + data_len, ad_len); + } +} + +/* + * Test that encryption and decryption produce the same results regardless of + * how the buffers are aligned in memory. + */ +static void test_aead_alignment_consistency(struct kunit *test) +{ + const size_t max_data_len = 4096; + const size_t max_ad_len = 4096; + const size_t max_offset = 128; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + const size_t key_len = AEAD_MAX_KEY_LEN; + const size_t tag_len = AEAD_MAX_TAG_LEN; + u8 *raw_key1_buf = alloc_buf(test, key_len + max_offset); + u8 *raw_key2_buf = alloc_buf(test, key_len + max_offset); + u8 *nonce1_buf = alloc_buf(test, nonce_len + max_offset); + u8 *nonce2_buf = alloc_buf(test, nonce_len + max_offset); + u8 *pt1_buf = alloc_buf(test, max_data_len); + u8 *pt2_buf = alloc_buf(test, max_data_len); + u8 *ct1_buf = alloc_buf(test, max_data_len); + u8 *ct2_buf = alloc_buf(test, max_data_len); + u8 *ad1_buf = alloc_buf(test, max_ad_len); + u8 *ad2_buf = alloc_buf(test, max_ad_len); + u8 *tag1_buf = alloc_buf(test, tag_len + max_offset); + u8 *tag2_buf = alloc_buf(test, tag_len + max_offset); + struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); + int err; + + for (int i = 0; i < 100; i++) { + /* Generate lengths. */ + size_t data_len = rand_length(max_data_len); + size_t ad_len = rand_length(max_ad_len); + + /* Generate two sets of alignments. */ + u8 *raw_key1 = raw_key1_buf + rand_offset(max_offset); + u8 *raw_key2 = raw_key2_buf + rand_offset(max_offset); + u8 *nonce1 = nonce1_buf + rand_offset(max_offset); + u8 *nonce2 = nonce2_buf + rand_offset(max_offset); + u8 *pt1 = pt1_buf + rand_offset(max_data_len - data_len); + u8 *pt2 = pt2_buf + rand_offset(max_data_len - data_len); + u8 *ct1 = ct1_buf + rand_offset(max_data_len - data_len); + u8 *ct2 = ct2_buf + rand_offset(max_data_len - data_len); + u8 *ad1 = ad1_buf + rand_offset(max_ad_len - ad_len); + u8 *ad2 = ad2_buf + rand_offset(max_ad_len - ad_len); + u8 *tag1 = tag1_buf + rand_offset(max_offset); + u8 *tag2 = tag2_buf + rand_offset(max_offset); + + /* + * Generate inputs in the first set of buffers using the first + * set of alignments. + */ + rand_bytes(raw_key1, key_len); + rand_bytes(nonce1, nonce_len); + rand_bytes(pt1, data_len); + rand_bytes(ad1, ad_len); + + /* + * Copy the inputs to the second set of buffers using the second + * set of alignments. + */ + memcpy(raw_key2, raw_key1, key_len); + memcpy(nonce2, nonce1, nonce_len); + memcpy(pt2, pt1, data_len); + memcpy(ad2, ad1, ad_len); + + /* Verify encryption consistency. */ + + err = AEAD_PREPAREKEY(key, raw_key1, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_ENCRYPT(ct1, pt1, data_len, tag1, ad1, ad_len, + nonce1, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + err = AEAD_PREPAREKEY(key, raw_key2, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_ENCRYPT(ct2, pt2, data_len, tag2, ad2, ad_len, + nonce2, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + KUNIT_ASSERT_MEMEQ(test, ct1, ct2, data_len); + KUNIT_ASSERT_MEMEQ(test, tag1, tag2, tag_len); + + /* Verify decryption consistency. */ + + err = AEAD_PREPAREKEY(key, raw_key1, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(pt1, ct1, data_len, tag1, ad1, ad_len, + nonce1, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + err = AEAD_PREPAREKEY(key, raw_key2, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(pt2, ct2, data_len, tag2, ad2, ad_len, + nonce2, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + KUNIT_ASSERT_MEMEQ(test, pt1, pt2, data_len); + } +} + +static void test_aead_inplace(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 512; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + size_t tag_len; + struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); + u8 *data = aead_alloc_random_data(test, max_data_len + tag_len); + u8 *data2 = alloc_buf(test, max_data_len + tag_len); + u8 *ad = aead_alloc_random_data(test, max_ad_len); + const u8 *nonce = aead_alloc_random_data(test, nonce_len); + + for (int i = 0; i < 100; i++) { + size_t data_len = rand_length(max_data_len); + size_t ad_len = rand_length(max_ad_len); + int err; + + /* Encrypt out-of-place. */ + err = AEAD_ENCRYPT(data2, data, data_len, data2 + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Encrypt in-place. */ + err = AEAD_ENCRYPT(data, data, data_len, data + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Compare the results. */ + KUNIT_ASSERT_MEMEQ(test, data2, data, data_len + tag_len); + + /* Decrypt out-of-place. */ + err = AEAD_DECRYPT(data2, data, data_len, data + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Decrypt in-place. */ + err = AEAD_DECRYPT(data, data, data_len, data + data_len, ad, + ad_len, nonce, nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Compare the results. */ + KUNIT_ASSERT_MEMEQ(test, data2, data, data_len); + } +} + +/* + * Monte-Carlo test for AEAD algorithms. This deterministically generates + * random AEAD inputs, encrypts them, verifies decryption, and computes and + * verifies the checksum of all computed (ciphertext, tag) pairs. + */ +static void test_aead_monte_carlo(struct kunit *test) +{ + const size_t max_data_len = 1024; + const size_t max_ad_len = 293; + u8 raw_key[AEAD_MAX_KEY_LEN]; + u8 nonce[AEAD_MAX_NONCE_LEN]; + u8 tag[AEAD_MAX_TAG_LEN]; + u8 *pt = alloc_buf(test, max_data_len); + u8 *ct = alloc_buf(test, max_data_len); + u8 *decrypted = alloc_buf(test, max_data_len); + u8 *ad = alloc_buf(test, max_ad_len); + struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); + struct blake2s_ctx checksum_ctx; + u8 actual_checksum[BLAKE2S_HASH_SIZE]; + int err; + + blake2s_init(&checksum_ctx, BLAKE2S_HASH_SIZE); + + for (size_t data_len = 0; data_len <= max_data_len; data_len++) { + size_t ad_len = data_len % max_ad_len; + size_t key_len = + AEAD_VALID_KEY_LENS[data_len % + ARRAY_SIZE(AEAD_VALID_KEY_LENS)]; + size_t nonce_len = + AEAD_VALID_NONCE_LENS[data_len % + ARRAY_SIZE(AEAD_VALID_NONCE_LENS)]; + size_t tag_len = + AEAD_VALID_TAG_LENS[data_len % + ARRAY_SIZE(AEAD_VALID_TAG_LENS)]; + + rand_bytes_seeded_from_len(pt, data_len); + rand_bytes_seeded_from_len(ad, ad_len); + rand_bytes_seeded_from_len(raw_key, key_len); + rand_bytes_seeded_from_len(nonce, nonce_len); + + err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Encryption failed with data_len=%zu, ad_len=%zu", + data_len, ad_len); + err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "Decryption failed with data_len=%zu, ad_len=%zu", + data_len, ad_len); + KUNIT_ASSERT_MEMEQ_MSG( + test, pt, decrypted, data_len, + "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", + data_len, ad_len); + + blake2s_update(&checksum_ctx, ct, data_len); + blake2s_update(&checksum_ctx, tag, tag_len); + } + + blake2s_final(&checksum_ctx, actual_checksum); + KUNIT_EXPECT_MEMEQ_MSG(test, actual_checksum, AEAD_MONTE_CARLO_CHECKSUM, + BLAKE2S_HASH_SIZE, + "Monte-Carlo checksum mismatch"); +} + +#define IRQ_TEST_DATA_LEN 256 +#define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */ + +struct aead_irq_test_slot { + /* Fields written only at test case initialization time */ + u8 raw_key[AEAD_MAX_KEY_LEN]; + u8 nonce[AEAD_MAX_NONCE_LEN]; + u8 pt[IRQ_TEST_DATA_LEN]; + u8 ct[IRQ_TEST_DATA_LEN + AEAD_MAX_TAG_LEN]; + u8 ad[IRQ_TEST_DATA_LEN]; + + /* Fields written throughout the test case */ + struct AEAD_KEY key; + u8 scratch_buf[IRQ_TEST_DATA_LEN + AEAD_MAX_TAG_LEN]; + int phase; + atomic_t in_use; +}; + +struct aead_irq_test_state { + struct aead_irq_test_slot slots[IRQ_TEST_NUM_BUFFERS]; +}; + +static bool aead_irq_test_func(void *state_) +{ + struct aead_irq_test_state *state = state_; + struct aead_irq_test_slot *slot; + size_t data_len; + bool ok = true; + + /* + * Find a free slot. This should always succeed, since the number of + * slots is equal to the max concurrency level of kunit_run_irq_test(). + */ + for (slot = &state->slots[0]; + slot < &state->slots[ARRAY_SIZE(state->slots)]; slot++) { + if (atomic_cmpxchg(&slot->in_use, 0, 1) == 0) + break; + } + if (WARN_ON_ONCE(slot == &state->slots[ARRAY_SIZE(state->slots)])) + return false; + /* + * This execution context now has exclusive access to 'slot'. + * Next, execute the next operation that the slot is set to perform. + */ + + data_len = sizeof(slot->pt); + if (slot->phase == 0) { + /* Phase 0: Prepare slot's key in current context. */ + ok = ok && AEAD_PREPAREKEY(&slot->key, slot->raw_key, + sizeof(slot->raw_key), + AEAD_MAX_TAG_LEN) == 0; + } else if (slot->phase == 1) { + /* + * Phase 1: Encrypt plaintext using key that may have been + * prepared in a different context. + */ + ok = ok && AEAD_ENCRYPT(slot->scratch_buf, slot->pt, data_len, + &slot->scratch_buf[data_len], slot->ad, + sizeof(slot->ad), slot->nonce, + sizeof(slot->nonce), &slot->key) == 0; + /* Verify the ciphertext (with concatenated auth tag) matches */ + ok = ok && + memcmp(slot->scratch_buf, slot->ct, sizeof(slot->ct)) == 0; + } else { + /* + * Phase 2: Decrypt ciphertext using key that may have been + * prepared in a different context. + */ + ok = ok && AEAD_DECRYPT(slot->scratch_buf, slot->ct, data_len, + &slot->ct[data_len], slot->ad, + sizeof(slot->ad), slot->nonce, + sizeof(slot->nonce), &slot->key) == 0; + /* Verify the plaintext matches. */ + ok = ok && memcmp(slot->scratch_buf, slot->pt, data_len) == 0; + } + slot->phase = (slot->phase + 1) % 3; + atomic_set_release(&slot->in_use, 0); + return ok; +} + +/* + * Test that encryption and decryption produce the correct results in task, + * softirq, and hardirq contexts running concurrently -- including with keys + * prepared in other contexts. This is needed to cover fallback code paths that + * execute in contexts where FPU or vector registers cannot be used. + */ +static void test_aead_interrupt_context(struct kunit *test) +{ + struct aead_irq_test_state *state = alloc_buf(test, sizeof(*state)); + + memset(state, 0, sizeof(*state)); + + /* + * For each slot, generate a set of AEAD inputs: a key, a nonce, a + * plaintext, and some associated data. Then generate the corresponding + * ciphertext with concatenated auth tag. + */ + for (int i = 0; i < IRQ_TEST_NUM_BUFFERS; i++) { + struct aead_irq_test_slot *slot = &state->slots[i]; + int err; + + rand_bytes(slot->raw_key, sizeof(slot->raw_key)); + rand_bytes(slot->nonce, sizeof(slot->nonce)); + rand_bytes(slot->pt, sizeof(slot->pt)); + rand_bytes(slot->ad, sizeof(slot->ad)); + err = AEAD_PREPAREKEY(&slot->key, slot->raw_key, + sizeof(slot->raw_key), AEAD_MAX_TAG_LEN); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_ENCRYPT(slot->ct, slot->pt, sizeof(slot->pt), + &slot->ct[sizeof(slot->pt)], slot->ad, + sizeof(slot->ad), slot->nonce, + sizeof(slot->nonce), &slot->key); + KUNIT_ASSERT_EQ(test, 0, err); + } + + kunit_run_irq_test(test, aead_irq_test_func, 100000, state); +} + +/* Benchmark AEAD encryption and decryption on various data lengths. */ +static void benchmark_aead(struct kunit *test) +{ + static const size_t data_lens_to_test[] = { + 16, 64, 128, 256, 512, 1024, 1420, 4096, 16384, + }; + const size_t max_data_len = 16384; + const size_t ad_len = 16; + const size_t key_len = AEAD_MAX_KEY_LEN; + const size_t nonce_len = AEAD_MAX_NONCE_LEN; + const size_t tag_len = AEAD_MAX_TAG_LEN; + const u8 *raw_key, *nonce, *ad; + u8 *pt, *ct, *tag; + struct AEAD_KEY *key; + int err; + + if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK)) + kunit_skip(test, "not enabled"); + + raw_key = aead_alloc_random_data(test, key_len); + nonce = aead_alloc_random_data(test, nonce_len); + ad = aead_alloc_random_data(test, ad_len); + pt = aead_alloc_random_data(test, max_data_len); + ct = alloc_buf(test, max_data_len); + tag = alloc_buf(test, tag_len); + + key = alloc_buf(test, sizeof(*key)); + err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); + KUNIT_ASSERT_EQ(test, 0, err); + + /* Warm-up */ + for (size_t i = 0; i < 10000000; i += max_data_len) { + err = AEAD_ENCRYPT(ct, pt, max_data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + err = AEAD_DECRYPT(pt, ct, max_data_len, tag, ad, ad_len, nonce, + nonce_len, key); + KUNIT_ASSERT_EQ(test, 0, err); + } + + for (size_t i = 0; i < ARRAY_SIZE(data_lens_to_test); i++) { + size_t data_len = data_lens_to_test[i]; + size_t num_iters = 10000000 / (data_len + 128); + u64 t_enc, t_dec; + bool ok = true; + + KUNIT_ASSERT_LE(test, data_len, max_data_len); + + preempt_disable(); + + t_enc = ktime_get_ns(); + for (size_t j = 0; j < num_iters; j++) { + err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + ok &= (err == 0); + } + t_enc = ktime_get_ns() - t_enc; + + t_dec = ktime_get_ns(); + for (size_t j = 0; j < num_iters; j++) { + err = AEAD_DECRYPT(pt, ct, data_len, tag, ad, ad_len, + nonce, nonce_len, key); + ok &= (err == 0); + } + t_dec = ktime_get_ns() - t_dec; + + preempt_enable(); + + KUNIT_ASSERT_TRUE_MSG(test, ok, "data_len=%zu", data_len); + + kunit_info(test, "data_len=%zu: enc %llu MB/s, dec %llu MB/s", + data_len, + div64_u64((u64)data_len * num_iters * 1000, + t_enc ?: 1), + div64_u64((u64)data_len * num_iters * 1000, + t_dec ?: 1)); + } +} + +/* clang-format off */ +#define AEAD_KUNIT_CASES \ + KUNIT_CASE(test_aead_all_key_lens), \ + KUNIT_CASE(test_aead_all_nonce_lens), \ + KUNIT_CASE(test_aead_all_tag_lens), \ + KUNIT_CASE(test_aead_incremental_updates), \ + KUNIT_CASE(test_aead_data_buffer_overruns), \ + KUNIT_CASE(test_aead_alignment_consistency), \ + KUNIT_CASE(test_aead_inplace), \ + KUNIT_CASE(test_aead_monte_carlo), \ + KUNIT_CASE(test_aead_interrupt_context), \ + KUNIT_CASE(benchmark_aead) diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c index ae3745212f03..6afdb3a04c17 100644 --- a/lib/crypto/tests/aes_cbc_macs_kunit.c +++ b/lib/crypto/tests/aes_cbc_macs_kunit.c @@ -34,18 +34,9 @@ static void aes_cmac_withtestkey(const u8 *data, size_t data_len, static int aes_cbc_macs_suite_init(struct kunit_suite *suite) { u8 raw_key[AES_KEYSIZE_256]; - int err; rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); - err = aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key)); - if (err) - return err; - return hash_suite_init(suite); -} - -static void aes_cbc_macs_suite_exit(struct kunit_suite *suite) -{ - hash_suite_exit(suite); + return aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key)); } /* Verify compatibility of the AES-CMAC implementation with RFC 4493. */ @@ -218,7 +209,6 @@ static struct kunit_suite aes_cbc_macs_test_suite = { .name = "aes_cbc_macs", .test_cases = aes_cbc_macs_test_cases, .suite_init = aes_cbc_macs_suite_init, - .suite_exit = aes_cbc_macs_suite_exit, }; kunit_test_suite(aes_cbc_macs_test_suite); diff --git a/lib/crypto/tests/aes_ccm_kunit.c b/lib/crypto/tests/aes_ccm_kunit.c new file mode 100644 index 000000000000..e8f2b3446017 --- /dev/null +++ b/lib/crypto/tests/aes_ccm_kunit.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * KUnit test suite for AES-CCM + * + * Copyright 2026 Google LLC + */ +#include <crypto/aes-ccm.h> +#include <crypto/blake2s.h> +#include "test-utils.h" + +/* AES-CCM test vectors from external sources */ +static const struct aes_ccm_testvec { + const char *name; + const char *key; + size_t key_len; + const char *nonce; + size_t nonce_len; + const char *ad; + size_t ad_len; + const char *ptext; + const char *ctext; + size_t data_len; + const char *tag; + size_t tag_len; +} aes_ccm_testvecs[] = { + { + .name = "RFC 3610 Packet Vector #1", + .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", + .key_len = 16, + .nonce = "\x00\x00\x00\x03\x02\x01\x00\xa0" + "\xa1\xa2\xa3\xa4\xa5", + .nonce_len = 13, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07", + .ad_len = 8, + .ptext = "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e", + .ctext = "\x58\x8c\x97\x9a\x61\xc6\x63\xd2" + "\xf0\x66\xd0\xc2\xc0\xf9\x89\x80" + "\x6d\x5f\x6b\x61\xda\xc3\x84", + .data_len = 23, + .tag = "\x17\xe8\xd1\x2c\xfd\xf9\x26\xe0", + .tag_len = 8, + }, + { + .name = "RFC 3610 Packet Vector #5", + .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", + .key_len = 16, + .nonce = "\x00\x00\x00\x07\x06\x05\x04\xa0" + "\xa1\xa2\xa3\xa4\xa5", + .nonce_len = 13, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b", + .ad_len = 12, + .ptext = "\x0c\x0d\x0e\x0f\x10\x11\x12\x13" + "\x14\x15\x16\x17\x18\x19\x1a\x1b" + "\x1c\x1d\x1e\x1f", + .ctext = "\xdc\xf1\xfb\x7b\x5d\x9e\x23\xfb" + "\x9d\x4e\x13\x12\x53\x65\x8a\xd8" + "\x6e\xbd\xca\x3e", + .data_len = 20, + .tag = "\x51\xe8\x3f\x07\x7d\x9c\x2d\x93", + .tag_len = 8, + }, + { + .name = "RFC 3610 Packet Vector #9", + .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", + .key_len = 16, + .nonce = "\x00\x00\x00\x0b\x0a\x09\x08\xa0" + "\xa1\xa2\xa3\xa4\xa5", + .nonce_len = 13, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07", + .ad_len = 8, + .ptext = "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + "\x20", + .ctext = "\x82\x53\x1a\x60\xcc\x24\x94\x5a" + "\x4b\x82\x79\x18\x1a\xb5\xc8\x4d" + "\xf2\x1c\xe7\xf9\xb7\x3f\x42\xe1" + "\x97", + .data_len = 25, + .tag = "\xea\x9c\x07\xe5\x6b\x5e\xb1\x7e" + "\x5f\x4e", + .tag_len = 10, + }, + { + .name = "NIST SP 800-38C Example 1", + .key = "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f", + .key_len = 16, + .nonce = "\x10\x11\x12\x13\x14\x15\x16", + .nonce_len = 7, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07", + .ad_len = 8, + .ptext = "\x20\x21\x22\x23", + .ctext = "\x71\x62\x01\x5b", + .data_len = 4, + .tag = "\x4d\xac\x25\x5d", + .tag_len = 4, + }, + { + .name = "NIST SP 800-38C Example 2", + .key = "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f", + .key_len = 16, + .nonce = "\x10\x11\x12\x13\x14\x15\x16\x17", + .nonce_len = 8, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .ad_len = 16, + .ptext = "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + .ctext = "\xd2\xa1\xf0\xe0\x51\xea\x5f\x62" + "\x08\x1a\x77\x92\x07\x3d\x59\x3d", + .data_len = 16, + .tag = "\x1f\xc6\x4f\xbf\xac\xcd", + .tag_len = 6, + }, + { + .name = "NIST SP 800-38C Example 3", + .key = "\x40\x41\x42\x43\x44\x45\x46\x47" + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f", + .key_len = 16, + .nonce = "\x10\x11\x12\x13\x14\x15\x16\x17" + "\x18\x19\x1a\x1b", + .nonce_len = 12, + .ad = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + "\x10\x11\x12\x13", + .ad_len = 20, + .ptext = "\x20\x21\x22\x23\x24\x25\x26\x27" + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + "\x30\x31\x32\x33\x34\x35\x36\x37", + .ctext = "\xe3\xb2\x01\xa9\xf5\xb7\x1a\x7a" + "\x9b\x1c\xea\xec\xcd\x97\xe7\x0b" + "\x61\x76\xaa\xd9\xa4\x42\x8a\xa5", + .data_len = 24, + .tag = "\x48\x43\x92\xfb\xc1\xb0\x99\x51", + .tag_len = 8, + }, +}; + +static void test_aes_ccm_one_test_vector(struct kunit *test, + const struct aes_ccm_testvec *tv) +{ + u8 *ctext = alloc_buf(test, tv->data_len); + u8 *decrypted = alloc_buf(test, tv->data_len); + u8 *tag = alloc_buf(test, tv->tag_len); + struct aes_ccm_key key; + int err; + + err = aes_ccm_preparekey(&key, tv->key, tv->key_len, tv->tag_len); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Failed to prepare key for %s", + tv->name); + + err = aes_ccm_encrypt(ctext, tv->ptext, tv->data_len, tag, tv->ad, + tv->ad_len, tv->nonce, tv->nonce_len, &key); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Encryption failed for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ctext, ctext, tv->data_len, + "Wrong ciphertext for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tag, tv->tag, tv->tag_len, + "Wrong tag for %s", tv->name); + + err = aes_ccm_decrypt(decrypted, ctext, tv->data_len, tag, tv->ad, + tv->ad_len, tv->nonce, tv->nonce_len, &key); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Decryption failed for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ptext, decrypted, tv->data_len, + "Wrong plaintext for %s", tv->name); +} + +static void test_aes_ccm_test_vectors(struct kunit *test) +{ + for (size_t i = 0; i < ARRAY_SIZE(aes_ccm_testvecs); i++) + test_aes_ccm_one_test_vector(test, &aes_ccm_testvecs[i]); +} + +/* + * Test NIST SP 800-38C Example 4, which uses a deterministically-generated + * 65536-byte associated data string. + */ +static void test_aes_ccm_nist_sp800_38c_example4(struct kunit *test) +{ + struct aes_ccm_testvec tv = { + .name = "NIST SP 800-38C Example 4", + .key_len = 16, + .nonce_len = 13, + .ad_len = 65536, + .ctext = "\x69\x91\x5d\xad\x1e\x84\xc6\x37" + "\x6a\x68\xc2\x96\x7e\x4d\xab\x61" + "\x5a\xe0\xfd\x1f\xae\xc4\x4c\xc4" + "\x84\x82\x85\x29\x46\x3c\xcf\x72", + .data_len = 32, + .tag = "\xb4\xac\x6b\xec\x93\xe8\x59\x8e" + "\x7f\x0d\xad\xbc\xea\x5b", + .tag_len = 14, + }; + u8 *key, *nonce, *ad, *ptext; + + key = alloc_buf(test, tv.key_len); + for (size_t i = 0; i < tv.key_len; i++) + key[i] = 0x40 + i; + + nonce = alloc_guarded_buf(test, tv.nonce_len); + for (size_t i = 0; i < tv.nonce_len; i++) + nonce[i] = 0x10 + i; + + ad = alloc_guarded_buf(test, tv.ad_len); + for (size_t i = 0; i < tv.ad_len; i++) + ad[i] = (u8)i; + + ptext = alloc_guarded_buf(test, tv.data_len); + for (size_t i = 0; i < tv.data_len; i++) + ptext[i] = 0x20 + i; + + tv.key = key; + tv.nonce = nonce; + tv.ad = ad; + tv.ptext = ptext; + + test_aes_ccm_one_test_vector(test, &tv); +} + +static const size_t aes_ccm_valid_key_lens[] = { 16, 24, 32 }; +#define AEAD_MAX_KEY_LEN 32 +#define AEAD_VALID_KEY_LENS aes_ccm_valid_key_lens + +static const size_t aes_ccm_valid_nonce_lens[] = { 7, 8, 9, 10, 11, 12, 13 }; +#define AEAD_MAX_NONCE_LEN 13 +#define AEAD_VALID_NONCE_LENS aes_ccm_valid_nonce_lens + +static const size_t aes_ccm_valid_tag_lens[] = { 4, 6, 8, 10, 12, 14, 16 }; +#define AEAD_MAX_TAG_LEN 16 +#define AEAD_VALID_TAG_LENS aes_ccm_valid_tag_lens + +#define AEAD_KEY aes_ccm_key +#define AEAD_CTX aes_ccm_ctx +#define AEAD_PREPAREKEY aes_ccm_preparekey +#define AEAD_ENCRYPT aes_ccm_encrypt +#define AEAD_DECRYPT aes_ccm_decrypt + +#define AEAD_INIT aes_ccm_init +#define AEAD_AUTH_UPDATE aes_ccm_auth_update +#define AEAD_ENCRYPT_UPDATE aes_ccm_encrypt_update +#define AEAD_DECRYPT_UPDATE aes_ccm_decrypt_update +#define AEAD_ENCRYPT_FINAL aes_ccm_encrypt_final +#define AEAD_DECRYPT_FINAL aes_ccm_decrypt_final + +/* This value was generated by gen-aead-testvecs.py. */ +static const u8 aes_ccm_monte_carlo_checksum[BLAKE2S_HASH_SIZE] = { + 0x70, 0x1c, 0xde, 0xa4, 0xe2, 0x03, 0x50, 0xb2, 0xf5, 0x9e, 0x61, + 0x66, 0xe4, 0xe5, 0x13, 0x1a, 0x00, 0x95, 0x34, 0x03, 0xb7, 0x61, + 0x2c, 0xdb, 0xc3, 0x15, 0x36, 0x84, 0x93, 0x7f, 0xb4, 0x5b, +}; +#define AEAD_MONTE_CARLO_CHECKSUM aes_ccm_monte_carlo_checksum + +#include "aead-test-template.h" + +/* + * Test that for each AES-CCM nonce length, the message length is validated + * against the correct corresponding maximum message length. + */ +static void test_aes_ccm_data_len_too_large(struct kunit *test) +{ + static const struct { + size_t nonce_len; + u64 max_data_len; + } lens[] = { + /* clang-format off */ + { 7, 0xffffffffffffffff }, /* U64_MAX */ + { 8, 0xffffffffffffff }, + { 9, 0xffffffffffff }, + { 10, 0xffffffffff }, + { 11, 0xffffffff }, + { 12, 0xffffff }, + { 13, 0xffff }, + /* clang-format on */ + }; + u8 nonce[13] = {}; + u8 raw_key[AES_KEYSIZE_256] = {}; + int err; + struct aes_ccm_key *key = alloc_buf(test, sizeof(*key)); + struct aes_ccm_ctx ctx; + + err = aes_ccm_preparekey(key, raw_key, sizeof(raw_key), 16); + KUNIT_ASSERT_EQ(test, 0, err); + + for (size_t i = 0; i < ARRAY_SIZE(lens); i++) { + size_t nonce_len = lens[i].nonce_len; + u64 max_data_len = lens[i].max_data_len; + + /* data_len <= max_data_len should be accepted. */ + err = aes_ccm_init(&ctx, 0, 0, nonce, nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "data_len=0 wasn't accepted with nonce_len=%zu", + nonce_len); + err = aes_ccm_init(&ctx, max_data_len, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, 0, err, + "data_len=%llu wasn't accepted with nonce_len=%zu", + max_data_len, nonce_len); + + /* data_len > max_data_len should be rejected. */ + if (max_data_len == U64_MAX) + continue; + err = aes_ccm_init(&ctx, max_data_len + 1, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=%llu wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_init)", + max_data_len + 1, nonce_len); + if (max_data_len + 1 <= SIZE_MAX) { + err = aes_ccm_encrypt(NULL, NULL, max_data_len + 1, + NULL, NULL, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=%llu wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_encrypt)", + max_data_len + 1, nonce_len); + err = aes_ccm_decrypt(NULL, NULL, max_data_len + 1, + NULL, NULL, 0, nonce, nonce_len, + key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=%llu wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_decrypt)", + max_data_len + 1, nonce_len); + } + err = aes_ccm_init(&ctx, U64_MAX, 0, nonce, nonce_len, key); + KUNIT_ASSERT_EQ_MSG( + test, -EOVERFLOW, err, + "data_len=U64_MAX wasn't rejected with -EOVERFLOW with nonce_len=%zu (aes_ccm_init)", + nonce_len); + } +} + +static struct kunit_case aes_ccm_test_cases[] = { + KUNIT_CASE(test_aes_ccm_test_vectors), + KUNIT_CASE(test_aes_ccm_nist_sp800_38c_example4), + KUNIT_CASE(test_aes_ccm_data_len_too_large), + AEAD_KUNIT_CASES, + {}, +}; + +static struct kunit_suite aes_ccm_test_suite = { + .name = "aes_ccm", + .test_cases = aes_ccm_test_cases, +}; +kunit_test_suite(aes_ccm_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for AES-CCM"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/aes_gcm_kunit.c b/lib/crypto/tests/aes_gcm_kunit.c new file mode 100644 index 000000000000..6959d86b316a --- /dev/null +++ b/lib/crypto/tests/aes_gcm_kunit.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * KUnit test suite for AES-GCM + * + * Copyright 2026 Google LLC + */ +#include <crypto/aes-gcm.h> +#include <crypto/blake2s.h> +#include "test-utils.h" + +/* The kernel's AES-GCM implementation supports only 12-byte nonces. */ +#define AES_GCM_NONCE_LEN 12 + +/* + * AES-GCM test vectors from the original paper "The Galois/Counter Mode of + * Operation (GCM)" by McGrew & Viega. Vectors with nonce lengths other than 12 + * bytes are excluded. + */ +static const struct aes_gcm_testvec { + const char *name; + const char *key; + size_t key_len; + const char *nonce; + size_t nonce_len; + const char *ad; + size_t ad_len; + const char *ptext; + const char *ctext; + size_t data_len; + const char *tag; + size_t tag_len; +} aes_gcm_testvecs[] = { + /* AES-128 Test Vectors */ + { + .name = "McGrew & Viega Test Case 1", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 16, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "", + .ctext = "", + .data_len = 0, + .ad = "", + .ad_len = 0, + .tag = "\x58\xe2\xfc\xce\xfa\x7e\x30\x61" + "\x36\x7f\x1d\x57\xa4\xe7\x45\x5a", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 2", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 16, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\x03\x88\xda\xce\x60\xb6\xa3\x92" + "\xf3\x28\xc2\xb9\x71\xb2\xfe\x78", + .data_len = 16, + .ad = "", + .ad_len = 0, + .tag = "\xab\x6e\x47\xd4\x2c\xec\x13\xbd" + "\xf5\x3a\x67\xb2\x12\x57\xbd\xdf", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 3", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 16, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39\x1a\xaf\xd2\x55", + .ctext = "\x42\x83\x1e\xc2\x21\x77\x74\x24" + "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" + "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" + "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" + "\x21\xd5\x14\xb2\x54\x66\x93\x1c" + "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" + "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" + "\x3d\x58\xe0\x91\x47\x3f\x59\x85", + .data_len = 64, + .ad = "", + .ad_len = 0, + .tag = "\x4d\x5c\x2a\xf3\x27\xcd\x64\xa6" + "\x2c\xf3\x5a\xbd\x2b\xa6\xfa\xb4", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 4", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 16, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39", + .ctext = "\x42\x83\x1e\xc2\x21\x77\x74\x24" + "\x4b\x72\x21\xb7\x84\xd0\xd4\x9c" + "\xe3\xaa\x21\x2f\x2c\x02\xa4\xe0" + "\x35\xc1\x7e\x23\x29\xac\xa1\x2e" + "\x21\xd5\x14\xb2\x54\x66\x93\x1c" + "\x7d\x8f\x6a\x5a\xac\x84\xaa\x05" + "\x1b\xa3\x0b\x39\x6a\x0a\xac\x97" + "\x3d\x58\xe0\x91", + .data_len = 60, + .ad = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xab\xad\xda\xd2", + .ad_len = 20, + .tag = "\x5b\xc9\x4f\xbc\x32\x21\xa5\xdb" + "\x94\xfa\xe9\x5a\xe7\x12\x1a\x47", + .tag_len = 16, + }, + + /* AES-192 Test Vectors */ + { + .name = "McGrew & Viega Test Case 7", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 24, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "", + .ctext = "", + .data_len = 0, + .ad = "", + .ad_len = 0, + .tag = "\xcd\x33\xb2\x8a\xc7\x73\xf7\x4b" + "\xa0\x0e\xd1\xf3\x12\x57\x24\x35", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 8", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 24, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\x98\xe7\x24\x7c\x07\xf0\xfe\x41" + "\x1c\x26\x7e\x43\x84\xb0\xf6\x00", + .data_len = 16, + .ad = "", + .ad_len = 0, + .tag = "\x2f\xf5\x8d\x80\x03\x39\x27\xab" + "\x8e\xf4\xd4\x58\x75\x14\xf0\xfb", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 9", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c", + .key_len = 24, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39\x1a\xaf\xd2\x55", + .ctext = "\x39\x80\xca\x0b\x3c\x00\xe8\x41" + "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" + "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" + "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" + "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" + "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" + "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" + "\xcc\xda\x27\x10\xac\xad\xe2\x56", + .data_len = 64, + .ad = "", + .ad_len = 0, + .tag = "\x99\x24\xa7\xc8\x58\x73\x36\xbf" + "\xb1\x18\x02\x4d\xb8\x67\x4a\x14", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 10", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c", + .key_len = 24, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39", + .ctext = "\x39\x80\xca\x0b\x3c\x00\xe8\x41" + "\xeb\x06\xfa\xc4\x87\x2a\x27\x57" + "\x85\x9e\x1c\xea\xa6\xef\xd9\x84" + "\x62\x85\x93\xb4\x0c\xa1\xe1\x9c" + "\x7d\x77\x3d\x00\xc1\x44\xc5\x25" + "\xac\x61\x9d\x18\xc8\x4a\x3f\x47" + "\x18\xe2\x44\x8b\x2f\xe3\x24\xd9" + "\xcc\xda\x27\x10", + .data_len = 60, + .ad = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xab\xad\xda\xd2", + .ad_len = 20, + .tag = "\x25\x19\x49\x8e\x80\xf1\x47\x8f" + "\x37\xba\x55\xbd\x6d\x27\x61\x8c", + .tag_len = 16, + }, + + /* AES-256 Test Vectors */ + { + .name = "McGrew & Viega Test Case 13", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 32, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "", + .ctext = "", + .data_len = 0, + .ad = "", + .ad_len = 0, + .tag = "\x53\x0f\x8a\xfb\xc7\x45\x36\xb9" + "\xa9\x63\xb4\xf1\xc4\xcb\x73\x8b", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 14", + .key = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .key_len = 32, + .nonce = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00", + .nonce_len = 12, + .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", + .ctext = "\xce\xa7\x40\x3d\x4d\x60\x6b\x6e" + "\x07\x4e\xc5\xd3\xba\xf3\x9d\x18", + .data_len = 16, + .ad = "", + .ad_len = 0, + .tag = "\xd0\xd1\xc8\xa7\x99\x99\x6b\xf0" + "\x26\x5b\x98\xb5\xd4\x8a\xb9\x19", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 15", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 32, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39\x1a\xaf\xd2\x55", + .ctext = "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" + "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" + "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" + "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" + "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" + "\xa7\xb0\x8b\x10\x56\x82\x88\x38" + "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" + "\xbc\xc9\xf6\x62\x89\x80\x15\xad", + .data_len = 64, + .ad = "", + .ad_len = 0, + .tag = "\xb0\x94\xda\xc5\xd9\x34\x71\xbd" + "\xec\x1a\x50\x22\x70\xe3\xcc\x6c", + .tag_len = 16, + }, + { + .name = "McGrew & Viega Test Case 16", + .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08" + "\xfe\xff\xe9\x92\x86\x65\x73\x1c" + "\x6d\x6a\x8f\x94\x67\x30\x83\x08", + .key_len = 32, + .nonce = "\xca\xfe\xba\xbe\xfa\xce\xdb\xad" + "\xde\xca\xf8\x88", + .nonce_len = 12, + .ptext = "\xd9\x31\x32\x25\xf8\x84\x06\xe5" + "\xa5\x59\x09\xc5\xaf\xf5\x26\x9a" + "\x86\xa7\xa9\x53\x15\x34\xf7\xda" + "\x2e\x4c\x30\x3d\x8a\x31\x8a\x72" + "\x1c\x3c\x0c\x95\x95\x68\x09\x53" + "\x2f\xcf\x0e\x24\x49\xa6\xb5\x25" + "\xb1\x6a\xed\xf5\xaa\x0d\xe6\x57" + "\xba\x63\x7b\x39", + .ctext = "\x52\x2d\xc1\xf0\x99\x56\x7d\x07" + "\xf4\x7f\x37\xa3\x2a\x84\x42\x7d" + "\x64\x3a\x8c\xdc\xbf\xe5\xc0\xc9" + "\x75\x98\xa2\xbd\x25\x55\xd1\xaa" + "\x8c\xb0\x8e\x48\x59\x0d\xbb\x3d" + "\xa7\xb0\x8b\x10\x56\x82\x88\x38" + "\xc5\xf6\x1e\x63\x93\xba\x7a\x0a" + "\xbc\xc9\xf6\x62", + .data_len = 60, + .ad = "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xfe\xed\xfa\xce\xde\xad\xbe\xef" + "\xab\xad\xda\xd2", + .ad_len = 20, + .tag = "\x76\xfc\x6e\xce\x0f\x4e\x17\x68" + "\xcd\xdf\x88\x53\xbb\x2d\x55\x1b", + .tag_len = 16, + }, +}; + +static void test_aes_gcm_one_test_vector(struct kunit *test, + const struct aes_gcm_testvec *tv) +{ + u8 *ctext = alloc_buf(test, tv->data_len); + u8 *decrypted = alloc_buf(test, tv->data_len); + u8 *tag = alloc_buf(test, tv->tag_len); + struct aes_gcm_key key; + int err; + + KUNIT_ASSERT_EQ(test, AES_GCM_NONCE_LEN, tv->nonce_len); + + err = aes_gcm_preparekey(&key, tv->key, tv->key_len, tv->tag_len); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Failed to prepare key for %s", + tv->name); + + aes_gcm_encrypt(ctext, tv->ptext, tv->data_len, tag, tv->ad, tv->ad_len, + tv->nonce, &key); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ctext, ctext, tv->data_len, + "Wrong ciphertext for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tag, tv->tag, tv->tag_len, + "Wrong tag for %s", tv->name); + + err = aes_gcm_decrypt(decrypted, ctext, tv->data_len, tag, tv->ad, + tv->ad_len, tv->nonce, &key); + KUNIT_ASSERT_EQ_MSG(test, 0, err, "Decryption failed for %s", tv->name); + KUNIT_ASSERT_MEMEQ_MSG(test, tv->ptext, decrypted, tv->data_len, + "Wrong plaintext for %s", tv->name); +} + +static void test_aes_gcm_test_vectors(struct kunit *test) +{ + for (size_t i = 0; i < ARRAY_SIZE(aes_gcm_testvecs); i++) + test_aes_gcm_one_test_vector(test, &aes_gcm_testvecs[i]); +} + +static int aes_gcm_init_test(struct aes_gcm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_gcm_key *key) +{ + if (nonce_len != AES_GCM_NONCE_LEN) + return -EINVAL; + /* + * Ignore data_len and ad_len. Incremental AES-GCM doesn't need them at + * initialization time. + */ + aes_gcm_init(ctx, nonce, key); + return 0; +} + +static int aes_gcm_encrypt_test(u8 *dst, const u8 *src, size_t data_len, + u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_gcm_key *key) +{ + if (nonce_len != AES_GCM_NONCE_LEN) + return -EINVAL; + /* aes_gcm_encrypt() returns void. */ + aes_gcm_encrypt(dst, src, data_len, authtag, ad, ad_len, nonce, key); + return 0; +} + +static int aes_gcm_decrypt_test(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_gcm_key *key) +{ + if (nonce_len != AES_GCM_NONCE_LEN) + return -EINVAL; + return aes_gcm_decrypt(dst, src, data_len, authtag, ad, ad_len, nonce, + key); +} + +static const size_t aes_gcm_valid_key_lens[] = { 16, 24, 32 }; +#define AEAD_MAX_KEY_LEN 32 +#define AEAD_VALID_KEY_LENS aes_gcm_valid_key_lens + +static const size_t aes_gcm_valid_nonce_lens[] = { AES_GCM_NONCE_LEN }; +#define AEAD_MAX_NONCE_LEN AES_GCM_NONCE_LEN +#define AEAD_VALID_NONCE_LENS aes_gcm_valid_nonce_lens + +static const size_t aes_gcm_valid_tag_lens[] = { 4, 8, 12, 13, 14, 15, 16 }; +#define AEAD_MAX_TAG_LEN 16 +#define AEAD_VALID_TAG_LENS aes_gcm_valid_tag_lens + +#define AEAD_KEY aes_gcm_key +#define AEAD_CTX aes_gcm_ctx +#define AEAD_PREPAREKEY aes_gcm_preparekey +#define AEAD_ENCRYPT aes_gcm_encrypt_test +#define AEAD_DECRYPT aes_gcm_decrypt_test + +#define AEAD_INIT aes_gcm_init_test +#define AEAD_AUTH_UPDATE aes_gcm_auth_update +#define AEAD_ENCRYPT_UPDATE aes_gcm_encrypt_update +#define AEAD_DECRYPT_UPDATE aes_gcm_decrypt_update +#define AEAD_ENCRYPT_FINAL aes_gcm_encrypt_final +#define AEAD_DECRYPT_FINAL aes_gcm_decrypt_final + +/* This value was generated by gen-aead-testvecs.py. */ +static const u8 aes_gcm_monte_carlo_checksum[BLAKE2S_HASH_SIZE] = { + 0x6d, 0xd0, 0x6e, 0x6b, 0xde, 0x3e, 0x92, 0x9f, 0xae, 0x1f, 0xf1, + 0x84, 0x99, 0x5a, 0x9e, 0x7b, 0xfe, 0x20, 0x9e, 0x22, 0x7c, 0x5f, + 0x15, 0xb3, 0x59, 0x89, 0xd0, 0xb7, 0x74, 0x5d, 0xd2, 0xa5, +}; +#define AEAD_MONTE_CARLO_CHECKSUM aes_gcm_monte_carlo_checksum + +#include "aead-test-template.h" + +static struct kunit_case aes_gcm_test_cases[] = { + KUNIT_CASE(test_aes_gcm_test_vectors), + AEAD_KUNIT_CASES, + {}, +}; + +static struct kunit_suite aes_gcm_test_suite = { + .name = "aes_gcm", + .test_cases = aes_gcm_test_cases, +}; +kunit_test_suite(aes_gcm_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for AES-GCM"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/blake2b_kunit.c b/lib/crypto/tests/blake2b_kunit.c index bc0be7da1e76..e2ce6cf3a99d 100644 --- a/lib/crypto/tests/blake2b_kunit.c +++ b/lib/crypto/tests/blake2b_kunit.c @@ -41,9 +41,9 @@ static void blake2b_init_default(struct blake2b_ctx *ctx) static void test_blake2b_all_key_and_hash_lens(struct kunit *test) { const size_t data_len = 100; - u8 *data = &test_buf[0]; - u8 *key = data + data_len; - u8 *hash = key + BLAKE2B_KEY_SIZE; + u8 *data = alloc_buf(test, data_len); + u8 *key = alloc_buf(test, BLAKE2B_KEY_SIZE); + u8 *hash = alloc_buf(test, BLAKE2B_HASH_SIZE); struct blake2b_ctx main_ctx; u8 main_hash[BLAKE2B_HASH_SIZE]; @@ -68,11 +68,13 @@ static void test_blake2b_all_key_and_hash_lens(struct kunit *test) static void test_blake2b_with_guarded_key_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *guarded_key_buf = alloc_guarded_buf(test, BLAKE2B_KEY_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int key_len = 0; key_len <= BLAKE2B_KEY_SIZE; key_len++) { u8 key[BLAKE2B_KEY_SIZE]; - u8 *guarded_key = &test_buf[TEST_BUF_LEN - key_len]; + u8 *guarded_key = &guarded_key_buf[BLAKE2B_KEY_SIZE - key_len]; u8 hash1[BLAKE2B_HASH_SIZE]; u8 hash2[BLAKE2B_HASH_SIZE]; struct blake2b_ctx ctx; @@ -80,14 +82,13 @@ static void test_blake2b_with_guarded_key_buf(struct kunit *test) rand_bytes(key, key_len); memcpy(guarded_key, key, key_len); - blake2b(key, key_len, test_buf, data_len, - hash1, BLAKE2B_HASH_SIZE); - blake2b(guarded_key, key_len, test_buf, data_len, - hash2, BLAKE2B_HASH_SIZE); + blake2b(key, key_len, data, data_len, hash1, BLAKE2B_HASH_SIZE); + blake2b(guarded_key, key_len, data, data_len, hash2, + BLAKE2B_HASH_SIZE); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2B_HASH_SIZE); blake2b_init_key(&ctx, BLAKE2B_HASH_SIZE, guarded_key, key_len); - blake2b_update(&ctx, test_buf, data_len); + blake2b_update(&ctx, data, data_len); blake2b_final(&ctx, hash2); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2B_HASH_SIZE); } @@ -100,14 +101,16 @@ static void test_blake2b_with_guarded_key_buf(struct kunit *test) static void test_blake2b_with_guarded_out_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *out_buf = alloc_guarded_buf(test, BLAKE2B_HASH_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int out_len = 1; out_len <= BLAKE2B_HASH_SIZE; out_len++) { u8 hash[BLAKE2B_HASH_SIZE]; - u8 *guarded_hash = &test_buf[TEST_BUF_LEN - out_len]; + u8 *guarded_hash = &out_buf[BLAKE2B_HASH_SIZE - out_len]; - blake2b(NULL, 0, test_buf, data_len, hash, out_len); - blake2b(NULL, 0, test_buf, data_len, guarded_hash, out_len); + blake2b(NULL, 0, data, data_len, hash, out_len); + blake2b(NULL, 0, data, data_len, guarded_hash, out_len); KUNIT_ASSERT_MEMEQ(test, hash, guarded_hash, out_len); } } @@ -124,8 +127,6 @@ static struct kunit_case blake2b_test_cases[] = { static struct kunit_suite blake2b_test_suite = { .name = "blake2b", .test_cases = blake2b_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(blake2b_test_suite); diff --git a/lib/crypto/tests/blake2s_kunit.c b/lib/crypto/tests/blake2s_kunit.c index 6832d9aa7b82..682f695f8a09 100644 --- a/lib/crypto/tests/blake2s_kunit.c +++ b/lib/crypto/tests/blake2s_kunit.c @@ -41,9 +41,9 @@ static void blake2s_init_default(struct blake2s_ctx *ctx) static void test_blake2s_all_key_and_hash_lens(struct kunit *test) { const size_t data_len = 100; - u8 *data = &test_buf[0]; - u8 *key = data + data_len; - u8 *hash = key + BLAKE2S_KEY_SIZE; + u8 *data = alloc_buf(test, data_len); + u8 *key = alloc_buf(test, BLAKE2S_KEY_SIZE); + u8 *hash = alloc_buf(test, BLAKE2S_HASH_SIZE); struct blake2s_ctx main_ctx; u8 main_hash[BLAKE2S_HASH_SIZE]; @@ -68,11 +68,13 @@ static void test_blake2s_all_key_and_hash_lens(struct kunit *test) static void test_blake2s_with_guarded_key_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *guarded_key_buf = alloc_guarded_buf(test, BLAKE2S_KEY_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int key_len = 0; key_len <= BLAKE2S_KEY_SIZE; key_len++) { u8 key[BLAKE2S_KEY_SIZE]; - u8 *guarded_key = &test_buf[TEST_BUF_LEN - key_len]; + u8 *guarded_key = &guarded_key_buf[BLAKE2S_KEY_SIZE - key_len]; u8 hash1[BLAKE2S_HASH_SIZE]; u8 hash2[BLAKE2S_HASH_SIZE]; struct blake2s_ctx ctx; @@ -80,14 +82,13 @@ static void test_blake2s_with_guarded_key_buf(struct kunit *test) rand_bytes(key, key_len); memcpy(guarded_key, key, key_len); - blake2s(key, key_len, test_buf, data_len, - hash1, BLAKE2S_HASH_SIZE); - blake2s(guarded_key, key_len, test_buf, data_len, - hash2, BLAKE2S_HASH_SIZE); + blake2s(key, key_len, data, data_len, hash1, BLAKE2S_HASH_SIZE); + blake2s(guarded_key, key_len, data, data_len, hash2, + BLAKE2S_HASH_SIZE); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2S_HASH_SIZE); blake2s_init_key(&ctx, BLAKE2S_HASH_SIZE, guarded_key, key_len); - blake2s_update(&ctx, test_buf, data_len); + blake2s_update(&ctx, data, data_len); blake2s_final(&ctx, hash2); KUNIT_ASSERT_MEMEQ(test, hash1, hash2, BLAKE2S_HASH_SIZE); } @@ -100,14 +101,16 @@ static void test_blake2s_with_guarded_key_buf(struct kunit *test) static void test_blake2s_with_guarded_out_buf(struct kunit *test) { const size_t data_len = 100; + u8 *data = alloc_buf(test, data_len); + u8 *out_buf = alloc_guarded_buf(test, BLAKE2S_HASH_SIZE); - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (int out_len = 1; out_len <= BLAKE2S_HASH_SIZE; out_len++) { u8 hash[BLAKE2S_HASH_SIZE]; - u8 *guarded_hash = &test_buf[TEST_BUF_LEN - out_len]; + u8 *guarded_hash = &out_buf[BLAKE2S_HASH_SIZE - out_len]; - blake2s(NULL, 0, test_buf, data_len, hash, out_len); - blake2s(NULL, 0, test_buf, data_len, guarded_hash, out_len); + blake2s(NULL, 0, data, data_len, hash, out_len); + blake2s(NULL, 0, data, data_len, guarded_hash, out_len); KUNIT_ASSERT_MEMEQ(test, hash, guarded_hash, out_len); } } @@ -124,8 +127,6 @@ static struct kunit_case blake2s_test_cases[] = { static struct kunit_suite blake2s_test_suite = { .name = "blake2s", .test_cases = blake2s_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(blake2s_test_suite); diff --git a/lib/crypto/tests/chacha20poly1305_kunit.c b/lib/crypto/tests/chacha20poly1305_kunit.c index 97a68fab88a7..d5504d0f4ad7 100644 --- a/lib/crypto/tests/chacha20poly1305_kunit.c +++ b/lib/crypto/tests/chacha20poly1305_kunit.c @@ -11,7 +11,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/kernel.h> -#include <linux/slab.h> +#include "test-utils.h" struct chacha20poly1305_testvec { const u8 *input, *output, *assoc, *nonce, *key; @@ -8890,11 +8890,8 @@ static void test_chacha20poly1305(struct kunit *test) bool ret; struct scatterlist sg_src[3]; - computed_output = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, - GFP_KERNEL); - input = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, computed_output); - KUNIT_ASSERT_NOT_NULL(test, input); + computed_output = alloc_buf(test, MAXIMUM_TEST_BUFFER_LEN); + input = alloc_buf(test, MAXIMUM_TEST_BUFFER_LEN); for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); diff --git a/lib/crypto/tests/ghash_kunit.c b/lib/crypto/tests/ghash_kunit.c index 68b3837a3607..4d7aa79b3cb2 100644 --- a/lib/crypto/tests/ghash_kunit.c +++ b/lib/crypto/tests/ghash_kunit.c @@ -39,17 +39,18 @@ static void ghash_withtestkey(const u8 *data, size_t len, */ static void test_ghash_allones_key_and_message(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct ghash_key key; struct ghash_ctx hashofhashes_ctx; u8 hash[GHASH_BLOCK_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - memset(test_buf, 0xff, 4096); + memset(data, 0xff, max_len); - ghash_preparekey(&key, test_buf); + ghash_preparekey(&key, data); ghash_init(&hashofhashes_ctx, &key); - for (size_t len = 0; len <= 4096; len += 16) { - ghash(&key, test_buf, len, hash); + for (size_t len = 0; len <= max_len; len += 16) { + ghash(&key, data, len, hash); ghash_update(&hashofhashes_ctx, hash, sizeof(hash)); } ghash_final(&hashofhashes_ctx, hash); @@ -68,7 +69,7 @@ static void check_key_consistency(struct kunit *test, const struct ghash_key *key1, const struct ghash_key *key2) { - u8 *data = test_buf; + u8 *data = alloc_buf(test, MAX_LEN_FOR_KEY_CHECK); u8 hash1[GHASH_BLOCK_SIZE]; u8 hash2[GHASH_BLOCK_SIZE]; @@ -88,10 +89,10 @@ static void check_key_consistency(struct kunit *test, static void test_ghash_with_guarded_key(struct kunit *test) { u8 raw_key[GHASH_BLOCK_SIZE]; - u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)]; + u8 *guarded_raw_key = alloc_guarded_buf(test, sizeof(raw_key)); struct ghash_key key1, key2; struct ghash_key *guarded_key = - (struct ghash_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)]; + alloc_guarded_buf(test, sizeof(*guarded_key)); /* Prepare with regular buffers. */ rand_bytes(raw_key, sizeof(raw_key)); @@ -116,14 +117,14 @@ static void test_ghash_with_minimally_aligned_key(struct kunit *test) { u8 raw_key[GHASH_BLOCK_SIZE]; struct ghash_key key; + const size_t align = __alignof__(struct ghash_key); + u8 *key_buf = alloc_buf(test, sizeof(struct ghash_key) + 3 * align); struct ghash_key *minaligned_key = - (struct ghash_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK + - __alignof__(struct ghash_key)]; + (struct ghash_key *)(PTR_ALIGN(key_buf, 2 * align) + align); - KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, - __alignof__(struct ghash_key))); - KUNIT_ASSERT_TRUE(test, !IS_ALIGNED((uintptr_t)minaligned_key, - 2 * __alignof__(struct ghash_key))); + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, align)); + KUNIT_ASSERT_TRUE(test, + !IS_ALIGNED((uintptr_t)minaligned_key, 2 * align)); rand_bytes(raw_key, sizeof(raw_key)); ghash_preparekey(&key, raw_key); @@ -164,12 +165,7 @@ static int ghash_suite_init(struct kunit_suite *suite) rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); ghash_preparekey(&test_key, raw_key); - return hash_suite_init(suite); -} - -static void ghash_suite_exit(struct kunit_suite *suite) -{ - hash_suite_exit(suite); + return 0; } static struct kunit_case ghash_test_cases[] = { @@ -186,7 +182,6 @@ static struct kunit_suite ghash_test_suite = { .name = "ghash", .test_cases = ghash_test_cases, .suite_init = ghash_suite_init, - .suite_exit = ghash_suite_exit, }; kunit_test_suite(ghash_test_suite); diff --git a/lib/crypto/tests/hash-test-template.h b/lib/crypto/tests/hash-test-template.h index 61b43e62779f..bb6eaa509d0f 100644 --- a/lib/crypto/tests/hash-test-template.h +++ b/lib/crypto/tests/hash-test-template.h @@ -7,93 +7,7 @@ */ #include <kunit/run-in-irq-context.h> #include <kunit/test.h> -#include <linux/vmalloc.h> - -/* test_buf is a guarded buffer, i.e. &test_buf[TEST_BUF_LEN] is not mapped. */ -#define TEST_BUF_LEN 16384 -static u8 *test_buf; - -static u8 *orig_test_buf; - -static u64 random_seed; - -/* - * This is a simple linear congruential generator. It is used only for testing, - * which does not require cryptographically secure random numbers. A hard-coded - * algorithm is used instead of <linux/prandom.h> so that it matches the - * algorithm used by the test vector generation script. This allows the input - * data in random test vectors to be concisely stored as just the seed. - */ -static u32 rand32(void) -{ - random_seed = (random_seed * 25214903917 + 11) & ((1ULL << 48) - 1); - return random_seed >> 16; -} - -static void rand_bytes(u8 *out, size_t len) -{ - for (size_t i = 0; i < len; i++) - out[i] = rand32(); -} - -static void rand_bytes_seeded_from_len(u8 *out, size_t len) -{ - random_seed = len; - rand_bytes(out, len); -} - -static bool rand_bool(void) -{ - return rand32() % 2; -} - -/* Generate a random length, preferring small lengths. */ -static size_t rand_length(size_t max_len) -{ - size_t len; - - switch (rand32() % 3) { - case 0: - len = rand32() % 128; - break; - case 1: - len = rand32() % 3072; - break; - default: - len = rand32(); - break; - } - return len % (max_len + 1); -} - -static size_t rand_offset(size_t max_offset) -{ - return min(rand32() % 128, max_offset); -} - -static int hash_suite_init(struct kunit_suite *suite) -{ - /* - * Allocate the test buffer using vmalloc() with a page-aligned length - * so that it is immediately followed by a guard page. This allows - * buffer overreads to be detected, even in assembly code. - */ - size_t alloc_len = round_up(TEST_BUF_LEN, PAGE_SIZE); - - orig_test_buf = vmalloc(alloc_len); - if (!orig_test_buf) - return -ENOMEM; - - test_buf = orig_test_buf + alloc_len - TEST_BUF_LEN; - return 0; -} - -static void hash_suite_exit(struct kunit_suite *suite) -{ - vfree(orig_test_buf); - orig_test_buf = NULL; - test_buf = NULL; -} +#include "test-utils.h" /* * Test the hash function against a list of test vectors. @@ -104,14 +18,16 @@ static void hash_suite_exit(struct kunit_suite *suite) */ static void test_hash_test_vectors(struct kunit *test) { + const size_t max_len = 16384; + u8 *data = alloc_buf(test, max_len); + for (size_t i = 0; i < ARRAY_SIZE(hash_testvecs); i++) { size_t data_len = hash_testvecs[i].data_len; u8 actual_hash[HASH_SIZE]; - KUNIT_ASSERT_LE(test, data_len, TEST_BUF_LEN); - rand_bytes_seeded_from_len(test_buf, data_len); - - HASH(test_buf, data_len, actual_hash); + KUNIT_ASSERT_LE(test, data_len, max_len); + rand_bytes_seeded_from_len(data, data_len); + HASH(data, data_len, actual_hash); KUNIT_ASSERT_MEMEQ_MSG( test, actual_hash, hash_testvecs[i].digest, HASH_SIZE, "Wrong result with test vector %zu; data_len=%zu", i, @@ -127,14 +43,15 @@ static void test_hash_test_vectors(struct kunit *test) */ static void test_hash_all_lens_up_to_4096(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct HASH_CTX ctx; u8 hash[HASH_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - rand_bytes_seeded_from_len(test_buf, 4096); + rand_bytes_seeded_from_len(data, max_len); HASH_INIT(&ctx); - for (size_t len = 0; len <= 4096; len++) { - HASH(test_buf, len, hash); + for (size_t len = 0; len <= max_len; len++) { + HASH(data, len, hash); HASH_UPDATE(&ctx, hash, HASH_SIZE); } HASH_FINAL(&ctx, hash); @@ -147,6 +64,9 @@ static void test_hash_all_lens_up_to_4096(struct kunit *test) */ static void test_hash_incremental_updates(struct kunit *test) { + const size_t max_len = 16384; + u8 *data = alloc_guarded_buf(test, max_len); + for (int i = 0; i < 1000; i++) { size_t total_len, offset; struct HASH_CTX ctx; @@ -155,12 +75,12 @@ static void test_hash_incremental_updates(struct kunit *test) size_t num_parts = 0; size_t remaining_len, cur_offset; - total_len = rand_length(TEST_BUF_LEN); - offset = rand_offset(TEST_BUF_LEN - total_len); - rand_bytes(&test_buf[offset], total_len); + total_len = rand_length(max_len); + offset = rand_offset(max_len - total_len); + rand_bytes(&data[offset], total_len); /* Compute the hash value in one shot. */ - HASH(&test_buf[offset], total_len, hash1); + HASH(&data[offset], total_len, hash1); /* * Compute the hash value incrementally, using a randomly @@ -172,13 +92,13 @@ static void test_hash_incremental_updates(struct kunit *test) while (rand_bool()) { size_t part_len = rand_length(remaining_len); - HASH_UPDATE(&ctx, &test_buf[cur_offset], part_len); + HASH_UPDATE(&ctx, &data[cur_offset], part_len); num_parts++; cur_offset += part_len; remaining_len -= part_len; } if (remaining_len != 0 || rand_bool()) { - HASH_UPDATE(&ctx, &test_buf[cur_offset], remaining_len); + HASH_UPDATE(&ctx, &data[cur_offset], remaining_len); num_parts++; } HASH_FINAL(&ctx, hash2); @@ -197,11 +117,13 @@ static void test_hash_incremental_updates(struct kunit *test) */ static void test_hash_buffer_overruns(struct kunit *test) { - const size_t max_tested_len = TEST_BUF_LEN - sizeof(struct HASH_CTX); - void *const buf_end = &test_buf[TEST_BUF_LEN]; + const size_t buf_len = 16384; + u8 *buf = alloc_guarded_buf(test, buf_len); + void *const buf_end = &buf[buf_len]; + const size_t max_tested_len = buf_len - sizeof(struct HASH_CTX); struct HASH_CTX *guarded_ctx = buf_end - sizeof(*guarded_ctx); - rand_bytes(test_buf, TEST_BUF_LEN); + rand_bytes(buf, buf_len); for (int i = 0; i < 100; i++) { size_t len = rand_length(max_tested_len); @@ -215,14 +137,14 @@ static void test_hash_buffer_overruns(struct kunit *test) HASH_FINAL(&ctx, hash); /* Check for overruns of the hash value buffer. */ - HASH(test_buf, len, buf_end - HASH_SIZE); + HASH(buf, len, buf_end - HASH_SIZE); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, test_buf, len); + HASH_UPDATE(&ctx, buf, len); HASH_FINAL(&ctx, buf_end - HASH_SIZE); - /* Check for overuns of the hash context. */ + /* Check for overruns of the hash context. */ HASH_INIT(guarded_ctx); - HASH_UPDATE(guarded_ctx, test_buf, len); + HASH_UPDATE(guarded_ctx, buf, len); HASH_FINAL(guarded_ctx, hash); } } @@ -233,30 +155,32 @@ static void test_hash_buffer_overruns(struct kunit *test) */ static void test_hash_overlaps(struct kunit *test) { - const size_t max_tested_len = TEST_BUF_LEN - HASH_SIZE; + const size_t buf_len = 16384; + u8 *buf = alloc_guarded_buf(test, buf_len); + const size_t max_tested_len = buf_len - HASH_SIZE; struct HASH_CTX ctx; u8 hash[HASH_SIZE]; - rand_bytes(test_buf, TEST_BUF_LEN); + rand_bytes(buf, buf_len); for (int i = 0; i < 100; i++) { size_t len = rand_length(max_tested_len); size_t offset = HASH_SIZE + rand_offset(max_tested_len - len); bool left_end = rand_bool(); - u8 *ovl_hash = left_end ? &test_buf[offset] : - &test_buf[offset + len - HASH_SIZE]; + u8 *ovl_hash = left_end ? &buf[offset] : + &buf[offset + len - HASH_SIZE]; - HASH(&test_buf[offset], len, hash); - HASH(&test_buf[offset], len, ovl_hash); + HASH(&buf[offset], len, hash); + HASH(&buf[offset], len, ovl_hash); KUNIT_ASSERT_MEMEQ_MSG( test, hash, ovl_hash, HASH_SIZE, "Overlap test 1 failed with len=%zu offset=%zu left_end=%d", len, offset, left_end); /* Repeat the above test, but this time use init+update+final */ - HASH(&test_buf[offset], len, hash); + HASH(&buf[offset], len, hash); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, &test_buf[offset], len); + HASH_UPDATE(&ctx, &buf[offset], len); HASH_FINAL(&ctx, ovl_hash); KUNIT_ASSERT_MEMEQ_MSG( test, hash, ovl_hash, HASH_SIZE, @@ -264,10 +188,10 @@ static void test_hash_overlaps(struct kunit *test) len, offset, left_end); /* Test modifying the source data after it was used. */ - HASH(&test_buf[offset], len, hash); + HASH(&buf[offset], len, hash); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, &test_buf[offset], len); - rand_bytes(&test_buf[offset], len); + HASH_UPDATE(&ctx, &buf[offset], len); + rand_bytes(&buf[offset], len); HASH_FINAL(&ctx, ovl_hash); KUNIT_ASSERT_MEMEQ_MSG( test, hash, ovl_hash, HASH_SIZE, @@ -282,20 +206,22 @@ static void test_hash_overlaps(struct kunit *test) */ static void test_hash_alignment_consistency(struct kunit *test) { + const size_t max_len = 16384; + u8 *data = alloc_guarded_buf(test, max_len); u8 hash1[128 + HASH_SIZE]; u8 hash2[128 + HASH_SIZE]; for (int i = 0; i < 100; i++) { - size_t len = rand_length(TEST_BUF_LEN); - size_t data_offs1 = rand_offset(TEST_BUF_LEN - len); - size_t data_offs2 = rand_offset(TEST_BUF_LEN - len); + size_t len = rand_length(max_len); + size_t data_offs1 = rand_offset(max_len - len); + size_t data_offs2 = rand_offset(max_len - len); size_t hash_offs1 = rand_offset(128); size_t hash_offs2 = rand_offset(128); - rand_bytes(&test_buf[data_offs1], len); - HASH(&test_buf[data_offs1], len, &hash1[hash_offs1]); - memmove(&test_buf[data_offs2], &test_buf[data_offs1], len); - HASH(&test_buf[data_offs2], len, &hash2[hash_offs2]); + rand_bytes(&data[data_offs1], len); + HASH(&data[data_offs1], len, &hash1[hash_offs1]); + memmove(&data[data_offs2], &data[data_offs1], len); + HASH(&data[data_offs2], len, &hash2[hash_offs2]); KUNIT_ASSERT_MEMEQ_MSG( test, &hash1[hash_offs1], &hash2[hash_offs2], HASH_SIZE, "Alignment consistency test failed with len=%zu data_offs=(%zu,%zu) hash_offs=(%zu,%zu)", @@ -308,11 +234,14 @@ static void test_hash_ctx_zeroization(struct kunit *test) { static const u8 zeroes[sizeof(struct HASH_CTX)]; struct HASH_CTX ctx; + const size_t data_len = 128; + u8 *data = alloc_buf(test, data_len); + u8 hash[HASH_SIZE]; - rand_bytes(test_buf, 128); + rand_bytes(data, data_len); HASH_INIT(&ctx); - HASH_UPDATE(&ctx, test_buf, 128); - HASH_FINAL(&ctx, test_buf); + HASH_UPDATE(&ctx, data, data_len); + HASH_FINAL(&ctx, hash); KUNIT_ASSERT_MEMEQ_MSG(test, &ctx, zeroes, sizeof(ctx), "Hash context was not zeroized by finalization"); } @@ -321,6 +250,7 @@ static void test_hash_ctx_zeroization(struct kunit *test) #define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */ struct hash_irq_test1_state { + u8 *data; u8 expected_hashes[IRQ_TEST_NUM_BUFFERS][HASH_SIZE]; atomic_t seqno; }; @@ -336,7 +266,8 @@ static bool hash_irq_test1_func(void *state_) u32 i = (u32)atomic_inc_return(&state->seqno) % IRQ_TEST_NUM_BUFFERS; u8 actual_hash[HASH_SIZE]; - HASH(&test_buf[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, actual_hash); + HASH(&state->data[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, + actual_hash); return memcmp(actual_hash, state->expected_hashes[i], HASH_SIZE) == 0; } @@ -346,12 +277,14 @@ static bool hash_irq_test1_func(void *state_) */ static void test_hash_interrupt_context_1(struct kunit *test) { + const size_t total_data_len = IRQ_TEST_NUM_BUFFERS * IRQ_TEST_DATA_LEN; struct hash_irq_test1_state state = {}; /* Prepare some test messages and compute the expected hash of each. */ - rand_bytes(test_buf, IRQ_TEST_NUM_BUFFERS * IRQ_TEST_DATA_LEN); + state.data = alloc_buf(test, total_data_len); + rand_bytes(state.data, total_data_len); for (int i = 0; i < IRQ_TEST_NUM_BUFFERS; i++) - HASH(&test_buf[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, + HASH(&state.data[i * IRQ_TEST_DATA_LEN], IRQ_TEST_DATA_LEN, state.expected_hashes[i]); kunit_run_irq_test(test, hash_irq_test1_func, 100000, &state); @@ -365,6 +298,8 @@ struct hash_irq_test2_hash_ctx { }; struct hash_irq_test2_state { + u8 *data; + size_t data_len; struct hash_irq_test2_hash_ctx ctxs[IRQ_TEST_NUM_BUFFERS]; u8 expected_hash[HASH_SIZE]; u16 update_lens[32]; @@ -397,7 +332,7 @@ static bool hash_irq_test2_func(void *state_) ctx->step++; } else if (ctx->step < state->num_steps - 1) { /* Update step */ - HASH_UPDATE(&ctx->hash_ctx, &test_buf[ctx->offset], + HASH_UPDATE(&ctx->hash_ctx, &state->data[ctx->offset], state->update_lens[ctx->step - 1]); ctx->offset += state->update_lens[ctx->step - 1]; ctx->step++; @@ -405,7 +340,7 @@ static bool hash_irq_test2_func(void *state_) /* Final step */ u8 actual_hash[HASH_SIZE]; - if (WARN_ON_ONCE(ctx->offset != TEST_BUF_LEN)) + if (WARN_ON_ONCE(ctx->offset != state->data_len)) ret = false; HASH_FINAL(&ctx->hash_ctx, actual_hash); if (memcmp(actual_hash, state->expected_hash, HASH_SIZE) != 0) @@ -426,20 +361,23 @@ static bool hash_irq_test2_func(void *state_) */ static void test_hash_interrupt_context_2(struct kunit *test) { + const size_t data_len = 16384; struct hash_irq_test2_state *state; - int remaining = TEST_BUF_LEN; + size_t remaining = data_len; state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, state); + state->data_len = data_len; + state->data = alloc_buf(test, data_len); - rand_bytes(test_buf, TEST_BUF_LEN); - HASH(test_buf, TEST_BUF_LEN, state->expected_hash); + rand_bytes(state->data, data_len); + HASH(state->data, data_len, state->expected_hash); /* * Generate a list of update lengths to use. Ensure that it contains * multiple entries but is limited to a maximum length. */ - static_assert(TEST_BUF_LEN / 4096 > 1); + KUNIT_ASSERT_GT(test, data_len / 4096, 1); for (state->num_steps = 0; state->num_steps < ARRAY_SIZE(state->update_lens) - 1 && remaining; state->num_steps++) { @@ -485,21 +423,23 @@ static void test_hash_interrupt_context_2(struct kunit *test) */ static void test_hmac(struct kunit *test) { + const size_t max_data_len = 4096; + const size_t max_key_len = 293; + const size_t outer_key_len = 32; + u8 *data = alloc_guarded_buf(test, max_data_len); + u8 *raw_key = alloc_guarded_buf(test, max_key_len); static const u8 zeroes[sizeof(struct HMAC_CTX)]; - u8 *raw_key; struct HMAC_KEY key; struct HMAC_CTX ctx; u8 mac[HASH_SIZE]; u8 mac2[HASH_SIZE]; - static_assert(TEST_BUF_LEN >= 4096 + 293); - rand_bytes_seeded_from_len(test_buf, 4096); - raw_key = &test_buf[4096]; + rand_bytes_seeded_from_len(data, max_data_len); + rand_bytes_seeded_from_len(raw_key, outer_key_len); - rand_bytes_seeded_from_len(raw_key, 32); - HMAC_PREPAREKEY(&key, raw_key, 32); + HMAC_PREPAREKEY(&key, raw_key, outer_key_len); HMAC_INIT(&ctx, &key); - for (size_t data_len = 0; data_len <= 4096; data_len++) { + for (size_t data_len = 0; data_len <= max_data_len; data_len++) { /* * Cycle through key lengths as well. Somewhat arbitrarily go * up to 293, which is somewhat larger than the largest hash @@ -507,17 +447,17 @@ static void test_hmac(struct kunit *test) * hashed down to one block); going higher would not be useful. * To reduce correlation with data_len, use a prime number here. */ - size_t key_len = data_len % 293; + size_t key_len = data_len % max_key_len; - HMAC_UPDATE(&ctx, test_buf, data_len); + HMAC_UPDATE(&ctx, data, data_len); rand_bytes_seeded_from_len(raw_key, key_len); - HMAC_USINGRAWKEY(raw_key, key_len, test_buf, data_len, mac); + HMAC_USINGRAWKEY(raw_key, key_len, data, data_len, mac); HMAC_UPDATE(&ctx, mac, HASH_SIZE); /* Verify that HMAC() is consistent with HMAC_USINGRAWKEY(). */ HMAC_PREPAREKEY(&key, raw_key, key_len); - HMAC(&key, test_buf, data_len, mac2); + HMAC(&key, data, data_len, mac2); KUNIT_ASSERT_MEMEQ_MSG( test, mac, mac2, HASH_SIZE, "HMAC gave different results with raw and prepared keys"); @@ -540,14 +480,17 @@ static void benchmark_hash(struct kunit *test) 1, 16, 64, 127, 128, 200, 256, 511, 512, 1024, 3173, 4096, 16384, }; + const size_t max_len = 16384; + u8 *data = alloc_buf(test, max_len); u8 hash[HASH_SIZE]; if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK)) kunit_skip(test, "not enabled"); /* Warm-up */ - for (size_t i = 0; i < 10000000; i += TEST_BUF_LEN) - HASH(test_buf, TEST_BUF_LEN, hash); + memset(data, 0, max_len); + for (size_t i = 0; i < 10000000; i += max_len) + HASH(data, max_len, hash); for (size_t i = 0; i < ARRAY_SIZE(lens_to_test); i++) { size_t len = lens_to_test[i]; @@ -555,11 +498,11 @@ static void benchmark_hash(struct kunit *test) size_t num_iters = 10000000 / (len + 128); u64 t; - KUNIT_ASSERT_LE(test, len, TEST_BUF_LEN); + KUNIT_ASSERT_LE(test, len, max_len); preempt_disable(); t = ktime_get_ns(); for (size_t j = 0; j < num_iters; j++) - HASH(test_buf, len, hash); + HASH(data, len, hash); t = ktime_get_ns() - t; preempt_enable(); kunit_info(test, "len=%zu: %llu MB/s", len, diff --git a/lib/crypto/tests/md5_kunit.c b/lib/crypto/tests/md5_kunit.c index 38bd52c25ae3..1598f8585b45 100644 --- a/lib/crypto/tests/md5_kunit.c +++ b/lib/crypto/tests/md5_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "md5", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/mldsa_kunit.c b/lib/crypto/tests/mldsa_kunit.c index 67f8f93e3dc6..ec64e10c97f2 100644 --- a/lib/crypto/tests/mldsa_kunit.c +++ b/lib/crypto/tests/mldsa_kunit.c @@ -8,6 +8,7 @@ #include <kunit/test.h> #include <linux/random.h> #include <linux/unaligned.h> +#include "test-utils.h" #define Q 8380417 /* The prime q = 2^23 - 2^13 + 1 */ @@ -60,14 +61,6 @@ static void do_mldsa_and_assert_success(struct kunit *test, KUNIT_ASSERT_EQ(test, err, 0); } -static u8 *kunit_kmemdup_or_fail(struct kunit *test, const u8 *src, size_t len) -{ - u8 *dst = kunit_kmalloc(test, len, GFP_KERNEL); - - KUNIT_ASSERT_NOT_NULL(test, dst); - return memcpy(dst, src, len); -} - /* * Test that changing coefficients in a valid signature's z vector results in * the following behavior from mldsa_verify(): @@ -83,7 +76,7 @@ static u8 *kunit_kmemdup_or_fail(struct kunit *test, const u8 *src, size_t len) static void test_mldsa_z_range(struct kunit *test, const struct mldsa_testvector *tv) { - u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len); + u8 *sig = memdup_buf(test, tv->sig, tv->sig_len); const int lambda = params[tv->alg].lambda; const s32 gamma1 = params[tv->alg].gamma1; const int beta = params[tv->alg].beta; @@ -146,7 +139,7 @@ static void test_mldsa_bad_hints(struct kunit *test, { const int omega = params[tv->alg].omega; const int k = params[tv->alg].k; - u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, tv->sig_len); + u8 *sig = memdup_buf(test, tv->sig, tv->sig_len); /* Pointer to the encoded hint vector in the signature */ u8 *hintvec = &sig[tv->sig_len - omega - k]; u8 h; @@ -202,9 +195,9 @@ static void test_mldsa_mutation(struct kunit *test, const int msg_len = tv->msg_len; const int pk_len = tv->pk_len; const int num_iter = 200; - u8 *sig = kunit_kmemdup_or_fail(test, tv->sig, sig_len); - u8 *msg = kunit_kmemdup_or_fail(test, tv->msg, msg_len); - u8 *pk = kunit_kmemdup_or_fail(test, tv->pk, pk_len); + u8 *sig = memdup_buf(test, tv->sig, sig_len); + u8 *msg = memdup_buf(test, tv->msg, msg_len); + u8 *pk = memdup_buf(test, tv->pk, pk_len); /* Initially the signature is valid. */ do_mldsa_and_assert_success(test, tv); diff --git a/lib/crypto/tests/nh_kunit.c b/lib/crypto/tests/nh_kunit.c index a8a3c3f345cb..49e0fb3dd294 100644 --- a/lib/crypto/tests/nh_kunit.c +++ b/lib/crypto/tests/nh_kunit.c @@ -5,14 +5,13 @@ #include <crypto/nh.h> #include <kunit/test.h> #include "nh-testvecs.h" +#include "test-utils.h" static void test_nh(struct kunit *test) { - u32 *key = kunit_kmalloc(test, NH_KEY_BYTES, GFP_KERNEL); + u32 *key = memdup_buf(test, nh_test_key, NH_KEY_BYTES); __le64 hash[NH_NUM_PASSES]; - KUNIT_ASSERT_NOT_NULL(test, key); - memcpy(key, nh_test_key, NH_KEY_BYTES); le32_to_cpu_array(key, NH_KEY_WORDS); nh(key, nh_test_msg, 16, hash); diff --git a/lib/crypto/tests/poly1305_kunit.c b/lib/crypto/tests/poly1305_kunit.c index 7ac191bd96b6..f3cb6245bc29 100644 --- a/lib/crypto/tests/poly1305_kunit.c +++ b/lib/crypto/tests/poly1305_kunit.c @@ -46,12 +46,7 @@ static void poly1305_withtestkey(const u8 *data, size_t len, static int poly1305_suite_init(struct kunit_suite *suite) { rand_bytes_seeded_from_len(test_key, POLY1305_KEY_SIZE); - return hash_suite_init(suite); -} - -static void poly1305_suite_exit(struct kunit_suite *suite) -{ - hash_suite_exit(suite); + return 0; } /* @@ -81,19 +76,20 @@ static void poly1305_suite_exit(struct kunit_suite *suite) */ static void test_poly1305_allones_keys_and_message(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct poly1305_desc_ctx mac_ctx, macofmacs_ctx; u8 mac[POLY1305_DIGEST_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - memset(test_buf, 0xff, 4096); + memset(data, 0xff, max_len); - poly1305_init(&mac_ctx, test_buf); - poly1305_init(&macofmacs_ctx, test_buf); + poly1305_init(&mac_ctx, data); + poly1305_init(&macofmacs_ctx, data); for (int i = 0; i < 32; i++) { - for (size_t len = 0; len <= 4096; len += 16) { + for (size_t len = 0; len <= max_len; len += 16) { struct poly1305_desc_ctx tmp_ctx; - poly1305_update(&mac_ctx, test_buf, len); + poly1305_update(&mac_ctx, data, len); tmp_ctx = mac_ctx; poly1305_final(&tmp_ctx, mac); poly1305_update(&macofmacs_ctx, mac, @@ -157,7 +153,6 @@ static struct kunit_suite poly1305_test_suite = { .name = "poly1305", .test_cases = poly1305_test_cases, .suite_init = poly1305_suite_init, - .suite_exit = poly1305_suite_exit, }; kunit_test_suite(poly1305_test_suite); diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c index d1f53a690ab8..2835929e890c 100644 --- a/lib/crypto/tests/polyval_kunit.c +++ b/lib/crypto/tests/polyval_kunit.c @@ -66,17 +66,18 @@ static void test_polyval_rfc8452_testvec(struct kunit *test) */ static void test_polyval_allones_key_and_message(struct kunit *test) { + const size_t max_len = 4096; + u8 *data = alloc_buf(test, max_len); struct polyval_key key; struct polyval_ctx hashofhashes_ctx; u8 hash[POLYVAL_BLOCK_SIZE]; - static_assert(TEST_BUF_LEN >= 4096); - memset(test_buf, 0xff, 4096); + memset(data, 0xff, max_len); - polyval_preparekey(&key, test_buf); + polyval_preparekey(&key, data); polyval_init(&hashofhashes_ctx, &key); - for (size_t len = 0; len <= 4096; len += 16) { - polyval(&key, test_buf, len, hash); + for (size_t len = 0; len <= max_len; len += 16) { + polyval(&key, data, len, hash); polyval_update(&hashofhashes_ctx, hash, sizeof(hash)); } polyval_final(&hashofhashes_ctx, hash); @@ -95,7 +96,7 @@ static void check_key_consistency(struct kunit *test, const struct polyval_key *key1, const struct polyval_key *key2) { - u8 *data = test_buf; + u8 *data = alloc_buf(test, MAX_LEN_FOR_KEY_CHECK); u8 hash1[POLYVAL_BLOCK_SIZE]; u8 hash2[POLYVAL_BLOCK_SIZE]; @@ -115,10 +116,10 @@ static void check_key_consistency(struct kunit *test, static void test_polyval_with_guarded_key(struct kunit *test) { u8 raw_key[POLYVAL_BLOCK_SIZE]; - u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)]; + u8 *guarded_raw_key = alloc_guarded_buf(test, sizeof(raw_key)); struct polyval_key key1, key2; struct polyval_key *guarded_key = - (struct polyval_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)]; + alloc_guarded_buf(test, sizeof(*guarded_key)); /* Prepare with regular buffers. */ rand_bytes(raw_key, sizeof(raw_key)); @@ -137,21 +138,20 @@ static void test_polyval_with_guarded_key(struct kunit *test) /* * Test that polyval_key only needs to be aligned to * __alignof__(struct polyval_key), i.e. 8 bytes. The assembly code may prefer - * 16-byte or higher alignment, but it musn't require it. + * 16-byte or higher alignment, but it mustn't require it. */ static void test_polyval_with_minimally_aligned_key(struct kunit *test) { u8 raw_key[POLYVAL_BLOCK_SIZE]; struct polyval_key key; + const size_t align = __alignof__(struct polyval_key); + u8 *key_buf = alloc_buf(test, sizeof(struct polyval_key) + 3 * align); struct polyval_key *minaligned_key = - (struct polyval_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK + - __alignof__(struct polyval_key)]; + (struct polyval_key *)(PTR_ALIGN(key_buf, 2 * align) + align); - KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, - __alignof__(struct polyval_key))); + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, align)); KUNIT_ASSERT_TRUE(test, - !IS_ALIGNED((uintptr_t)minaligned_key, - 2 * __alignof__(struct polyval_key))); + !IS_ALIGNED((uintptr_t)minaligned_key, 2 * align)); rand_bytes(raw_key, sizeof(raw_key)); polyval_preparekey(&key, raw_key); @@ -192,12 +192,7 @@ static int polyval_suite_init(struct kunit_suite *suite) rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); polyval_preparekey(&test_key, raw_key); - return hash_suite_init(suite); -} - -static void polyval_suite_exit(struct kunit_suite *suite) -{ - hash_suite_exit(suite); + return 0; } static struct kunit_case polyval_test_cases[] = { @@ -215,7 +210,6 @@ static struct kunit_suite polyval_test_suite = { .name = "polyval", .test_cases = polyval_test_cases, .suite_init = polyval_suite_init, - .suite_exit = polyval_suite_exit, }; kunit_test_suite(polyval_test_suite); diff --git a/lib/crypto/tests/sha1_kunit.c b/lib/crypto/tests/sha1_kunit.c index 24ba8d5669c8..27286afaa407 100644 --- a/lib/crypto/tests/sha1_kunit.c +++ b/lib/crypto/tests/sha1_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha1", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha224_kunit.c b/lib/crypto/tests/sha224_kunit.c index 962ad46b9c99..bcf8b90f9ae2 100644 --- a/lib/crypto/tests/sha224_kunit.c +++ b/lib/crypto/tests/sha224_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha224", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha256_kunit.c b/lib/crypto/tests/sha256_kunit.c index 5dccdee79693..17daa09adfe7 100644 --- a/lib/crypto/tests/sha256_kunit.c +++ b/lib/crypto/tests/sha256_kunit.c @@ -4,6 +4,7 @@ */ #include <crypto/sha2.h> #include "sha256-testvecs.h" +#include "test-utils.h" /* Generate the HASH_KUNIT_CASES using hash-test-template.h. */ #define HASH sha256 @@ -22,26 +23,6 @@ #define HMAC_USINGRAWKEY hmac_sha256_usingrawkey #include "hash-test-template.h" -static void free_guarded_buf(void *buf) -{ - vfree(buf); -} - -/* - * Allocate a KUnit-managed buffer that has length @len bytes immediately - * followed by an unmapped page, and assert that the allocation succeeds. - */ -static void *alloc_guarded_buf(struct kunit *test, size_t len) -{ - size_t full_len = round_up(len, PAGE_SIZE); - void *buf = vmalloc(full_len); - - KUNIT_ASSERT_NOT_NULL(test, buf); - KUNIT_ASSERT_EQ(test, 0, - kunit_add_action_or_reset(test, free_guarded_buf, buf)); - return buf + full_len - len; -} - /* * Test for sha256_finup_2x(). Specifically, choose various data lengths and * salt lengths, and for each one, verify that sha256_finup_2x() produces the @@ -105,19 +86,20 @@ static void test_sha256_finup_2x(struct kunit *test) static void test_sha256_finup_2x_defaultctx(struct kunit *test) { const size_t data_len = 128; + u8 *data = alloc_buf(test, 2 * data_len); struct sha256_ctx ctx; u8 hash1_a[SHA256_DIGEST_SIZE]; u8 hash2_a[SHA256_DIGEST_SIZE]; u8 hash1_b[SHA256_DIGEST_SIZE]; u8 hash2_b[SHA256_DIGEST_SIZE]; - rand_bytes(test_buf, 2 * data_len); + rand_bytes(data, 2 * data_len); sha256_init(&ctx); - sha256_finup_2x(&ctx, test_buf, &test_buf[data_len], data_len, hash1_a, + sha256_finup_2x(&ctx, data, &data[data_len], data_len, hash1_a, hash2_a); - sha256_finup_2x(NULL, test_buf, &test_buf[data_len], data_len, hash1_b, + sha256_finup_2x(NULL, data, &data[data_len], data_len, hash1_b, hash2_b); KUNIT_ASSERT_MEMEQ(test, hash1_a, hash1_b, SHA256_DIGEST_SIZE); @@ -131,18 +113,19 @@ static void test_sha256_finup_2x_defaultctx(struct kunit *test) static void test_sha256_finup_2x_hugelen(struct kunit *test) { const size_t data_len = 4 * SHA256_BLOCK_SIZE; + u8 *data = alloc_buf(test, data_len); struct sha256_ctx ctx = {}; u8 expected_hash[SHA256_DIGEST_SIZE]; u8 hash[SHA256_DIGEST_SIZE]; - rand_bytes(test_buf, data_len); + rand_bytes(data, data_len); for (size_t align = 0; align < SHA256_BLOCK_SIZE; align++) { sha256_init(&ctx); ctx.ctx.bytecount = 0x123456789abcd00 + align; - sha256_finup_2x(&ctx, test_buf, test_buf, data_len, hash, hash); + sha256_finup_2x(&ctx, data, data, data_len, hash, hash); - sha256_update(&ctx, test_buf, data_len); + sha256_update(&ctx, data, data_len); sha256_final(&ctx, expected_hash); KUNIT_ASSERT_MEMEQ(test, hash, expected_hash, @@ -161,6 +144,7 @@ static void benchmark_sha256_finup_2x(struct kunit *test) static const size_t salt_lens_to_test[] = { 0, 32, 64 }; const size_t data_len = 4096; const size_t num_iters = 4096; + u8 *data = alloc_buf(test, data_len * 2); struct sha256_ctx ctx; u8 hash1[SHA256_DIGEST_SIZE]; u8 hash2[SHA256_DIGEST_SIZE]; @@ -170,12 +154,12 @@ static void benchmark_sha256_finup_2x(struct kunit *test) if (!sha256_finup_2x_is_optimized()) kunit_skip(test, "not relevant"); - rand_bytes(test_buf, data_len * 2); + rand_bytes(data, data_len * 2); /* Warm-up */ for (size_t i = 0; i < num_iters; i++) - sha256_finup_2x(NULL, &test_buf[0], &test_buf[data_len], - data_len, hash1, hash2); + sha256_finup_2x(NULL, &data[0], &data[data_len], data_len, + hash1, hash2); for (size_t i = 0; i < ARRAY_SIZE(salt_lens_to_test); i++) { size_t salt_len = salt_lens_to_test[i]; @@ -186,12 +170,12 @@ static void benchmark_sha256_finup_2x(struct kunit *test) * not measured; we're just interested in sha256_finup_2x(). */ sha256_init(&ctx); - sha256_update(&ctx, test_buf, salt_len); + sha256_update(&ctx, data, salt_len); preempt_disable(); t0 = ktime_get_ns(); for (size_t j = 0; j < num_iters; j++) - sha256_finup_2x(&ctx, &test_buf[0], &test_buf[data_len], + sha256_finup_2x(&ctx, &data[0], &data[data_len], data_len, hash1, hash2); t1 = ktime_get_ns(); preempt_enable(); @@ -215,8 +199,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha256", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha384_kunit.c b/lib/crypto/tests/sha384_kunit.c index e1ef5c995bb6..76409822d965 100644 --- a/lib/crypto/tests/sha384_kunit.c +++ b/lib/crypto/tests/sha384_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha384", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sha3_kunit.c b/lib/crypto/tests/sha3_kunit.c index ed5fbe80337f..81b113079755 100644 --- a/lib/crypto/tests/sha3_kunit.c +++ b/lib/crypto/tests/sha3_kunit.c @@ -273,12 +273,10 @@ static void test_shake_all_lens_up_to_4096(struct kunit *test) { struct sha3_ctx main_ctx; const size_t max_len = 4096; - u8 *const in = test_buf; - u8 *const out = &test_buf[TEST_BUF_LEN - max_len]; + u8 *const in = alloc_buf(test, max_len); + u8 *const out = alloc_buf(test, max_len); u8 main_hash[SHA3_256_DIGEST_SIZE]; - KUNIT_ASSERT_LE(test, 2 * max_len, TEST_BUF_LEN); - rand_bytes_seeded_from_len(in, max_len); for (int alg = 0; alg < 2; alg++) { sha3_256_init(&main_ctx); @@ -309,12 +307,8 @@ static void test_shake_all_lens_up_to_4096(struct kunit *test) static void test_shake_multiple_squeezes(struct kunit *test) { const size_t max_len = 512; - u8 *ref_out; - - KUNIT_ASSERT_GE(test, TEST_BUF_LEN, 2 * max_len); - - ref_out = kunit_kzalloc(test, max_len, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, ref_out); + u8 *buf = alloc_buf(test, max_len); + u8 *ref_out = alloc_buf(test, max_len); for (int i = 0; i < 2000; i++) { const int alg = rand32() % 2; @@ -322,8 +316,8 @@ static void test_shake_multiple_squeezes(struct kunit *test) const size_t out_len = rand_length(max_len); const size_t in_offs = rand_offset(max_len - in_len); const size_t out_offs = rand_offset(max_len - out_len); - u8 *const in = &test_buf[in_offs]; - u8 *const out = &test_buf[out_offs]; + u8 *const in = &buf[in_offs]; + u8 *const out = &buf[out_offs]; struct shake_ctx ctx; size_t remaining_len, j, num_parts; @@ -368,16 +362,12 @@ static void test_shake_multiple_squeezes(struct kunit *test) static void test_shake_with_guarded_bufs(struct kunit *test) { const size_t max_len = 512; - u8 *reg_buf; - - KUNIT_ASSERT_GE(test, TEST_BUF_LEN, max_len); - - reg_buf = kunit_kzalloc(test, max_len, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, reg_buf); + u8 *buf = alloc_guarded_buf(test, max_len); + u8 *reg_buf = alloc_buf(test, max_len); for (int alg = 0; alg < 2; alg++) { for (size_t len = 0; len <= max_len; len++) { - u8 *guarded_buf = &test_buf[TEST_BUF_LEN - len]; + u8 *guarded_buf = &buf[max_len - len]; rand_bytes(reg_buf, len); memcpy(guarded_buf, reg_buf, len); @@ -413,8 +403,6 @@ static struct kunit_case sha3_test_cases[] = { static struct kunit_suite sha3_test_suite = { .name = "sha3", .test_cases = sha3_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(sha3_test_suite); diff --git a/lib/crypto/tests/sha512_kunit.c b/lib/crypto/tests/sha512_kunit.c index 8923e2d7d3d4..e03926c24403 100644 --- a/lib/crypto/tests/sha512_kunit.c +++ b/lib/crypto/tests/sha512_kunit.c @@ -30,8 +30,6 @@ static struct kunit_case hash_test_cases[] = { static struct kunit_suite hash_test_suite = { .name = "sha512", .test_cases = hash_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(hash_test_suite); diff --git a/lib/crypto/tests/sm3_kunit.c b/lib/crypto/tests/sm3_kunit.c index dc8136acdff6..1cba384946e9 100644 --- a/lib/crypto/tests/sm3_kunit.c +++ b/lib/crypto/tests/sm3_kunit.c @@ -22,8 +22,6 @@ static struct kunit_case sm3_test_cases[] = { static struct kunit_suite sm3_test_suite = { .name = "sm3", .test_cases = sm3_test_cases, - .suite_init = hash_suite_init, - .suite_exit = hash_suite_exit, }; kunit_test_suite(sm3_test_suite); diff --git a/lib/crypto/tests/test-utils.h b/lib/crypto/tests/test-utils.h new file mode 100644 index 000000000000..73ca21f9176e --- /dev/null +++ b/lib/crypto/tests/test-utils.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Test utility functions shared by the crypto library tests. + * + * For now this is simply a header that's included into the KUnit test suites + * that need it. If this gets too large it could be made its own translation + * unit and libcrypto_test_utils module, but that seems overkill for now. + */ +#ifndef LIB_CRYPTO_TEST_UTILS_H +#define LIB_CRYPTO_TEST_UTILS_H + +#include <kunit/test.h> +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/string.h> +#include <linux/vmalloc.h> + +static u64 random_seed; + +static __maybe_unused void action_free_guarded_buf(void *buf) +{ + vfree(buf); +} + +/* + * Allocate a KUnit-managed buffer that has length @size bytes (> 0) immediately + * followed by an unmapped page, and assert that the allocation succeeds. + */ +static __maybe_unused void *alloc_guarded_buf(struct kunit *test, size_t size) +{ + size_t full_size = round_up(size, PAGE_SIZE); + void *buf = vmalloc(full_size); + + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_EQ(test, 0, + kunit_add_action_or_reset(test, action_free_guarded_buf, + buf)); + return buf + full_size - size; +} + +static __maybe_unused void *alloc_buf(struct kunit *test, size_t size) +{ + void *buf = kunit_kmalloc(test, size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, buf); + return buf; +} + +static __maybe_unused void *memdup_buf(struct kunit *test, const void *src, + size_t size) +{ + void *dst = alloc_buf(test, size); + + return memcpy(dst, src, size); +} + +/* + * This is a simple linear congruential generator. It is used only for testing, + * which does not require cryptographically secure random numbers. A hard-coded + * algorithm is used instead of <linux/prandom.h> so that it matches the + * algorithm used by the test vector generation script. This allows the input + * data in random test vectors to be concisely stored as just the seed. + */ +static __maybe_unused u32 rand32(void) +{ + random_seed = (random_seed * 25214903917 + 11) & ((1ULL << 48) - 1); + return random_seed >> 16; +} + +static __maybe_unused void rand_bytes(u8 *out, size_t len) +{ + for (size_t i = 0; i < len; i++) + out[i] = rand32(); +} + +static __maybe_unused void rand_bytes_seeded_from_len(u8 *out, size_t len) +{ + random_seed = len; + rand_bytes(out, len); +} + +static __maybe_unused bool rand_bool(void) +{ + return rand32() % 2; +} + +/* Generate a random length, preferring small lengths. */ +static __maybe_unused size_t rand_length(size_t max_len) +{ + size_t len; + + switch (rand32() % 3) { + case 0: + len = rand32() % 128; + break; + case 1: + len = rand32() % 3072; + break; + default: + len = rand32(); + break; + } + return len % (max_len + 1); +} + +static __maybe_unused size_t rand_offset(size_t max_offset) +{ + return min(rand32() % 128, max_offset); +} + +#endif /* LIB_CRYPTO_TEST_UTILS_H */ diff --git a/lib/embedded-cmdline.S b/lib/embedded-cmdline.S new file mode 100644 index 000000000000..bda81b4a42be --- /dev/null +++ b/lib/embedded-cmdline.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Embed the build-time-rendered bootconfig "kernel" subtree as a flat + * cmdline string. setup_arch() prepends this to boot_command_line on + * architectures that select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates + * Copyright (c) 2026 Breno Leitao <leitao@debian.org> + */ + .section .init.rodata.embed_cmdline, "a", %progbits + .global embedded_kernel_cmdline +embedded_kernel_cmdline: + .incbin "lib/embedded_cmdline.bin" + .byte 0 + .global embedded_kernel_cmdline_end +embedded_kernel_cmdline_end: diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c index 00d9dc61cd46..05305e655f99 100644 --- a/lib/find_bit_benchmark.c +++ b/lib/find_bit_benchmark.c @@ -149,6 +149,21 @@ static int __init test_find_next_and_bit(const void *bitmap, return 0; } +static int __init +test_bitmap_find_next_zero_area_off(unsigned long *bitmap, unsigned long len) +{ + unsigned long i, cnt; + ktime_t time; + + time = ktime_get(); + for (cnt = i = 0; i < BITMAP_LEN; cnt++) + i = bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN, i, 8, 0, 0) + 1; + time = ktime_get() - time; + pr_err("bitmap_find_next_zero_area_off:%7llu ns, %6ld iterations\n", time, cnt); + + return 0; +} + static int __init find_bit_test(void) { unsigned long nbits = BITMAP_LEN / SPARSE; @@ -158,6 +173,7 @@ static int __init find_bit_test(void) get_random_bytes(bitmap, sizeof(bitmap)); get_random_bytes(bitmap2, sizeof(bitmap2)); + test_bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN); test_find_next_bit(bitmap, BITMAP_LEN); test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); @@ -181,6 +197,7 @@ static int __init find_bit_test(void) __set_bit(get_random_u32_below(BITMAP_LEN), bitmap2); } + test_bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN); test_find_next_bit(bitmap, BITMAP_LEN); test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index f5d5333450a0..4fc66722d00d 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -178,6 +178,37 @@ unsigned int font_data_size(font_data_t *fd) } EXPORT_SYMBOL_GPL(font_data_size); +static unsigned int font_data_num_glyphs(font_data_t *fd, unsigned int width, unsigned int height) +{ + return font_data_size(fd) / font_glyph_size(width, height); +} + +/** + * font_data_glyph_buf() - Returns the glyph for a specific character as raw bytes + * @fd: The font data + * @width: The glyph width in bits per scanline + * @vpitch: The number of scanlines per glyph + * @c: The character + * + * Glyphs start at fixed intervals within the font data. font_data_glyph_buf() + * returns the glyph shape of the specified character. If no such glyph + * exists in the font, it returns NULL. + * + * Returns: + * The character's raw glyph shape, or NULL if no glyph exists for the character. The + * provided buffer is read-only. + */ +const unsigned char *font_data_glyph_buf(font_data_t *fd, + unsigned int width, unsigned int vpitch, + unsigned int c) +{ + if (c >= font_data_num_glyphs(fd, width, vpitch)) + return NULL; + + return font_data_buf(fd) + font_glyph_size(width, vpitch) * c; +} +EXPORT_SYMBOL_GPL(font_data_glyph_buf); + /** * font_data_is_equal - Compares font data for equality * @lhs: Left-hand side font data diff --git a/lib/glob.c b/lib/glob.c index 7aca76c25bcb..c80d9dd736b4 100644 --- a/lib/glob.c +++ b/lib/glob.c @@ -11,6 +11,9 @@ MODULE_DESCRIPTION("glob(7) matching"); MODULE_LICENSE("Dual MIT/GPL"); +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end); + /** * glob_match - Shell-style pattern matching, like !fnmatch(pat, str, 0) * @pat: Shell-style pattern to match, e.g. "*.[ch]". @@ -41,6 +44,29 @@ MODULE_LICENSE("Dual MIT/GPL"); */ bool __pure glob_match(char const *pat, char const *str) { + return glob_match_str(pat, str, NULL); +} +EXPORT_SYMBOL(glob_match); + +/** + * glob_match_len - glob match against a length-bounded string + * @pat: Shell-style pattern to match. + * @str: String to match. Need not be NUL-terminated. + * @len: Number of bytes of @str that may be read. + * + * Like glob_match(), but @str is only read up to @len bytes, so it can be + * used on buffers that are not NUL-terminated (e.g. trace event fields). + * A NUL byte within @len still terminates the string. + */ +bool __pure glob_match_len(char const *pat, char const *str, size_t len) +{ + return glob_match_str(pat, str, str + len); +} +EXPORT_SYMBOL(glob_match_len); + +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end) +{ /* * Backtrack to previous * on mismatch and retry starting one * character later in the string. Because * matches all characters @@ -55,9 +81,11 @@ bool __pure glob_match(char const *pat, char const *str) * on mismatch, or true after matching the trailing nul bytes. */ for (;;) { - unsigned char c = *str++; + unsigned char c = (str_end && str >= str_end) ? '\0' : *str; unsigned char d = *pat++; + str++; + switch (d) { case '?': /* Wildcard: anything but nul */ if (c == '\0') @@ -125,4 +153,3 @@ backtrack: } } } -EXPORT_SYMBOL(glob_match); diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index eba2d3e28980..b0b07270ce7c 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -140,13 +140,13 @@ static int intersection_range_check(void) intxn1 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn1) { - WARN_ON_ONCE("Failed to allocate intxn1\n"); + WARN_ONCE(1, "Failed to allocate intxn1\n"); return -ENOMEM; } intxn2 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn2) { - WARN_ON_ONCE("Failed to allocate intxn2\n"); + WARN_ONCE(1, "Failed to allocate intxn2\n"); bitmap_free(intxn1); return -ENOMEM; } diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 273919b16161..6665372ecf71 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1491,6 +1491,7 @@ void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) i->__iov -= state->nr_segs - i->nr_segs; i->nr_segs = state->nr_segs; } +EXPORT_SYMBOL_FOR_MODULES(iov_iter_restore, "vmw_vsock_virtio_transport_common"); /* * Extract a list of contiguous pages from an ITER_FOLIOQ iterator. This does @@ -1568,6 +1569,7 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, struct folio *folio; unsigned int nr = 0, offset; loff_t pos = i->xarray_start + i->iov_offset; + bool will_alloc = !*pages; XA_STATE(xas, i->xarray, pos >> PAGE_SHIFT); offset = pos & ~PAGE_MASK; @@ -1595,6 +1597,14 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, } rcu_read_unlock(); + if (!nr) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } + return 0; + } + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); iov_iter_advance(i, maxsize); return maxsize; @@ -1625,9 +1635,11 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, } bi.bi_idx = 0; bi.bi_size = maxsize; - bi.bi_bvec_done = skip; + bi.bi_offset = skip; maxpages = want_pages_array(pages, maxsize, skip, maxpages); + if (!maxpages) + return -ENOMEM; while (bi.bi_size && bi.bi_idx < i->nr_segs) { struct bio_vec bv = bvec_iter_bvec(i->bvec, bi); @@ -1745,6 +1757,7 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, unsigned long addr; unsigned int gup_flags = 0; size_t offset; + bool will_alloc = !*pages; int res; if (i->data_source == ITER_DEST) @@ -1761,8 +1774,14 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, if (!maxpages) return -ENOMEM; res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages); - if (unlikely(res <= 0)) + if (unlikely(res <= 0)) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } return res; + } + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset); iov_iter_advance(i, maxsize); return maxsize; @@ -1886,6 +1905,8 @@ static unsigned int get_contig_folio_len(struct page **pages, * @max_size: maximum size to extract from @iter * @nr_vecs: number of vectors in @bv (on in and output) * @max_vecs: maximum vectors in @bv, including those filled before calling + * @mem_align_mask: reject with -EINVAL if the source address or + * length is not aligned to this mask * @extraction_flags: flags to qualify request * * Like iov_iter_extract_pages(), but returns physically contiguous ranges @@ -1897,14 +1918,19 @@ static unsigned int get_contig_folio_len(struct page **pages, */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags) + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags) { + unsigned long start = (unsigned long)iter_iov_addr(iter); unsigned short entries_left = max_vecs - *nr_vecs; unsigned short nr_pages, i = 0; size_t left, offset, len; struct page **pages; ssize_t size; + if ((start | iter_iov_len(iter)) & mem_align_mask) + return -EINVAL; + /* * Move page array up in the allocated memory for the bio vecs as far as * possible so that we can start filling biovecs from the beginning diff --git a/lib/kobject.c b/lib/kobject.c index 9c9ff0f5175f..e7b010a989fb 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -823,9 +823,11 @@ static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr, struct kobj_attribute *kattr; ssize_t ret = -EIO; - kattr = container_of(attr, struct kobj_attribute, attr); + kattr = container_of_const(attr, struct kobj_attribute, attr); if (kattr->show) ret = kattr->show(kobj, kattr, buf); + else if (kattr->show_const) + ret = kattr->show_const(kobj, kattr, buf); return ret; } @@ -835,9 +837,11 @@ static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr, struct kobj_attribute *kattr; ssize_t ret = -EIO; - kattr = container_of(attr, struct kobj_attribute, attr); + kattr = container_of_const(attr, struct kobj_attribute, attr); if (kattr->store) ret = kattr->store(kobj, kattr, buf, count); + else if (kattr->store_const) + ret = kattr->store_const(kobj, kattr, buf, count); return ret; } diff --git a/lib/kstrtox.c b/lib/kstrtox.c index edc4eb7c1bca..bac1c057e1b0 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -17,6 +17,7 @@ #include <linux/export.h> #include <linux/kstrtox.h> #include <linux/math64.h> +#include <linux/overflow.h> #include <linux/types.h> #include <linux/uaccess.h> @@ -45,6 +46,7 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) * @base: Radix * @p: Where to store result * @max_chars: Maximum amount of characters to convert + * @init: Initial value of the multiply-accumulate result * * Convert non-negative integer string representation in explicitly given * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX. @@ -56,12 +58,12 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) */ noinline unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p, - size_t max_chars) + size_t max_chars, unsigned long long init) { unsigned int rv, overflow = 0; unsigned long long res; - res = 0; + res = init; for (rv = 0; rv < max_chars; rv++, s++) { unsigned int c = *s; unsigned int lc = _tolower(c); @@ -94,12 +96,6 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon return rv | overflow; } -noinline -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p) -{ - return _parse_integer_limit(s, base, p, INT_MAX); -} - static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res) { unsigned long long _res; @@ -399,6 +395,109 @@ int kstrtobool(const char *s, bool *res) } EXPORT_SYMBOL(kstrtobool); +static int _kstrtoudec64(const char *s, unsigned int scale, u64 *res) +{ + unsigned int rv_int, rv_frac; + u64 _res = 0; + + rv_int = _parse_integer(s, 10, &_res); + if (rv_int & KSTRTOX_OVERFLOW) + return -ERANGE; + s += rv_int; + + if (*s == '.') + s++; /* skip decimal point */ + + rv_frac = _parse_integer(s, 10, &_res, scale, _res); + if (rv_frac & KSTRTOX_OVERFLOW) + return -ERANGE; + s += rv_frac; + + /* + * Check input beyond rv_int and rv_frac to cover cases like ".5" with + * scale 0, which is considered a valid input, being parsed as 0. + */ + if (!rv_int && !rv_frac && !isdigit(*s)) + return -EINVAL; + + while (isdigit(*s)) /* truncate digits */ + s++; + + if (*s == '\n') + s++; + if (*s) + return -EINVAL; + + if (_res && ((scale - rv_frac) > 19 /* log10(2^64) = 19.26 */ || + check_mul_overflow(_res, int_pow(10, scale - rv_frac), &_res))) + return -ERANGE; + + *res = _res; + return 0; +} + +/** + * kstrtoudec64() - Convert a string to an unsigned 64-bit scaled decimal value. + * @s: The start of the string. The string must be null-terminated, and may also + * include a single newline before its terminating null. The first character + * may also be a plus sign, but not a minus sign. + * @scale: The number of digits to the right of the decimal point. + * @res: Where to write the result of the conversion on success. + * + * For example, a scale of 3 with input "123.45" results in 123450. Note that + * trailing zeros in the fractional part input to match the scale are not + * required. Also, digits beyond the specified scale are ignored. + * + * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error. + */ +noinline +int kstrtoudec64(const char *s, unsigned int scale, u64 *res) +{ + if (s[0] == '+') + s++; + return _kstrtoudec64(s, scale, res); +} +EXPORT_SYMBOL(kstrtoudec64); + +/** + * kstrtodec64() - Convert a string to a signed 64-bit scaled decimal value. + * @s: The start of the string. The string must be null-terminated, and may also + * include a single newline before its terminating null. The first character + * may also be a plus sign or a minus sign. + * @scale: The number of digits to the right of the decimal point. + * @res: Where to write the result of the conversion on success. + * + * For example, a scale of 4 with input "-3.141592" results in -31415. Note + * that digits beyond the specified scale are ignored. Also, trailing zeros in + * the fractional part input to match the scale are not required. + * + * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error. + */ +noinline +int kstrtodec64(const char *s, unsigned int scale, s64 *res) +{ + u64 tmp; + int rv; + + if (s[0] == '-') { + rv = _kstrtoudec64(s + 1, scale, &tmp); + if (rv < 0) + return rv; + if ((s64)-tmp > 0) + return -ERANGE; + *res = -tmp; + } else { + rv = kstrtoudec64(s, scale, &tmp); + if (rv < 0) + return rv; + if ((s64)tmp < 0) + return -ERANGE; + *res = tmp; + } + return 0; +} +EXPORT_SYMBOL(kstrtodec64); + /* * Since "base" would be a nonsense argument, this open-codes the * _from_user helper instead of using the helper macro below. diff --git a/lib/kstrtox.h b/lib/kstrtox.h index 158c400ca865..73dee79fd8ed 100644 --- a/lib/kstrtox.h +++ b/lib/kstrtox.h @@ -2,10 +2,23 @@ #ifndef _LIB_KSTRTOX_H #define _LIB_KSTRTOX_H +#include <linux/args.h> + #define KSTRTOX_OVERFLOW (1U << 31) const char *_parse_integer_fixup_radix(const char *s, unsigned int *base); unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *res, - size_t max_chars); -unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *res); + size_t max_chars, unsigned long long init); + +#define _parse_integer0(s, base, res, ...) \ + _parse_integer_limit(s, base, res, INT_MAX, 0) + +#define _parse_integer1(s, base, res, max_chars, ...) \ + _parse_integer_limit(s, base, res, max_chars, 0) + +#define _parse_integer2(s, base, res, max_chars, init, ...) \ + _parse_integer_limit(s, base, res, max_chars, init) + +#define _parse_integer(s, base, res, ...) \ + CONCATENATE(_parse_integer, COUNT_ARGS(__VA_ARGS__))(s, base, res, __VA_ARGS__) #endif diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index 2e8a6b71a2ab..204e02b10eba 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -11,6 +11,7 @@ kunit-objs += test.o \ attributes.o \ device.o \ platform.o \ + fwnode.o \ bug.o ifeq ($(CONFIG_KUNIT_DEBUGFS),y) diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 9c326f1837bd..442b2ceb955b 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -76,18 +76,30 @@ static int debugfs_print_results(struct seq_file *seq, void *v) seq_puts(seq, "KTAP version 1\n"); seq_puts(seq, "1..1\n"); - /* Print suite header because it is not stored in the test logs. */ - seq_puts(seq, KUNIT_SUBTEST_INDENT "KTAP version 1\n"); - seq_printf(seq, KUNIT_SUBTEST_INDENT "# Subtest: %s\n", suite->name); - seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite)); - - kunit_suite_for_each_test_case(suite, test_case) - debugfs_print_result(seq, test_case->log); + if (suite->status != KUNIT_SKIPPED) { + /* Print suite header because it is not stored in the test logs. */ + seq_puts(seq, + KUNIT_SUBTEST_INDENT "KTAP version 1\n"); + seq_printf(seq, + KUNIT_SUBTEST_INDENT "# Subtest: %s\n", + suite->name); + seq_printf(seq, + KUNIT_SUBTEST_INDENT "1..%zd\n", + kunit_suite_num_test_cases(suite)); + + kunit_suite_for_each_test_case(suite, test_case) + debugfs_print_result(seq, test_case->log); + } debugfs_print_result(seq, suite->log); - seq_printf(seq, "%s %d %s\n", - kunit_status_to_ok_not_ok(success), 1, suite->name); + if (suite->status != KUNIT_SKIPPED) + seq_printf(seq, "%s %d %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name); + else + seq_printf(seq, "%s %d %s # SKIP %s\n", + kunit_status_to_ok_not_ok(success), 1, suite->name, + suite->status_comment); return 0; } diff --git a/lib/kunit/fwnode.c b/lib/kunit/fwnode.c new file mode 100644 index 000000000000..a58ce0984d76 --- /dev/null +++ b/lib/kunit/fwnode.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Qualcomm Technologies, Inc. and/or its subsidiaries + */ + +#include <kunit/fwnode.h> +#include <kunit/test.h> + +#include <linux/fwnode.h> +#include <linux/property.h> + +KUNIT_DEFINE_ACTION_WRAPPER(fwnode_remove_software_node_wrapper, + fwnode_remove_software_node, + struct fwnode_handle *); + +/** + * kunit_fwnode_create_software_node() - Create a kunit-managed software node + * @test: Test context + * @properties: Properties to use to create the new software node + * @parent: Parent of this software node + * + * Create a test-managed software node and return its firmware node handle. + * The software node is removed after the test case completes. + * + * Returns: + * Firmware node handle of the newly created software node or IS_ERR() on + * failure. + */ +struct fwnode_handle * +kunit_fwnode_create_software_node(struct kunit *test, + const struct property_entry *properties, + const struct fwnode_handle *parent) +{ + struct fwnode_handle *fwnode; + int ret; + + fwnode = fwnode_create_software_node(properties, parent); + if (IS_ERR(fwnode)) + return fwnode; + + ret = kunit_add_action_or_reset(test, fwnode_remove_software_node_wrapper, + fwnode); + if (ret) + return ERR_PTR(ret); + + return fwnode; +} +EXPORT_SYMBOL_GPL(kunit_fwnode_create_software_node); + +KUNIT_DEFINE_ACTION_WRAPPER(software_node_unregister_wrapper, + software_node_unregister, + const struct software_node *); + +/** + * kunit_software_node_register() - Register a kunit-managed software node + * @test: Test context + * @swnode: Software node to register + * + * Register a test-managed software node and return its firmware node handle. + * The software node is unregistered after the test case completes. + * + * Returns: + * Firmware node handle of the registered software node or IS_ERR() on failure. + */ +struct fwnode_handle * +kunit_software_node_register(struct kunit *test, + const struct software_node *swnode) +{ + struct fwnode_handle *fwnode; + int ret; + + ret = software_node_register(swnode); + if (ret) + return ERR_PTR(ret); + + fwnode = software_node_fwnode(swnode); + if (WARN_ON(!fwnode)) + return ERR_PTR(-ENOENT); + + ret = kunit_add_action_or_reset(test, software_node_unregister_wrapper, + (void *)swnode); + if (ret) + return ERR_PTR(ret); + + return fwnode; +} +EXPORT_SYMBOL_GPL(kunit_software_node_register); + +KUNIT_DEFINE_ACTION_WRAPPER(software_node_unregister_node_group_wrapper, + software_node_unregister_node_group, + const struct software_node *const *); + +/** + * kunit_software_node_register_node_group() - Register a kunit-managed software node group + * @test: Test context + * @nodes: Software node group to register + * + * Register a test-managed software node group. The nodes are unregistered + * after the test case completes. + * + * Returns: + * 0 on success, negative error number on failure. + */ +int kunit_software_node_register_node_group(struct kunit *test, + const struct software_node *const *nodes) +{ + int ret; + + ret = software_node_register_node_group(nodes); + if (ret) + return ret; + + return kunit_add_action_or_reset(test, software_node_unregister_node_group_wrapper, + (void *)nodes); +} +EXPORT_SYMBOL_GPL(kunit_software_node_register_node_group); + +KUNIT_DEFINE_ACTION_WRAPPER(device_remove_software_node_wrapper, + device_remove_software_node, + struct device *); + +/** + * kunit_device_add_software_node() - Assign a kunit-managed software node to a device + * @test: Test context + * @dev: Device to assign the software node for + * @node: The software node to assign + * + * Make @node the secondary firmware node of @dev. If @dev has no primary + * firmware node, @node will become the primary node. The software node will + * be automatically removed from @dev when the test case completes. + * + * Returns: + * 0 on success, negative error number on failure. + */ +int kunit_device_add_software_node(struct kunit *test, struct device *dev, + const struct software_node *node) +{ + int ret; + + ret = device_add_software_node(dev, node); + if (ret) + return ret; + + return kunit_add_action_or_reset(test, device_remove_software_node_wrapper, dev); +} +EXPORT_SYMBOL_GPL(kunit_device_add_software_node); diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index 0bae7b7ca0b0..b8ded54fa46d 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -591,5 +591,34 @@ static struct kunit_suite example_init_test_suite = { */ kunit_test_init_section_suites(&example_init_test_suite); +/* + * This test should always be skipped. + */ +static void example_skip_suite_test(struct kunit *test) +{ + /* This line should never be seen */ + KUNIT_FAIL(test, "You should not see a this."); +} + +static struct kunit_case example_skip_suite_test_cases[] = { + KUNIT_CASE(example_skip_suite_test), + {} +}; + +static int example_skip_suite_init(struct kunit_suite *suite) +{ + kunit_mark_skipped(suite, "Test suite expected to be skipped"); + return 0; +} + +static struct kunit_suite example_test_skip_suite = { + .name = "example_skip_suite", + .suite_init = example_skip_suite_init, + .test_cases = example_skip_suite_test_cases, +}; + +/* This registers a test suite that will be skipped */ +kunit_test_suite(example_test_skip_suite); + MODULE_DESCRIPTION("Example KUnit test suite"); MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index 0d8f1b30559b..51ba40ebf19f 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -9,6 +9,7 @@ #include <kunit/static_stub.h> #include <kunit/test.h> #include <linux/list.h> +#include <linux/seq_buf.h> #include <linux/slab.h> #include "string-stream.h" @@ -74,7 +75,8 @@ int string_stream_vadd(struct string_stream *stream, /* Append newline if necessary. */ if (frag_container->fragment[result_len - 1] != '\n') - result_len = strlcat(frag_container->fragment, "\n", buf_len); + result_len += strscpy(frag_container->fragment + result_len, + "\n", buf_len - result_len); } else { result_len = vsnprintf(frag_container->fragment, buf_len, fmt, args); } @@ -118,15 +120,18 @@ char *string_stream_get_string(struct string_stream *stream) { struct string_stream_fragment *frag_container; size_t buf_len = stream->length + 1; /* +1 for null byte. */ + struct seq_buf sb; char *buf; buf = kzalloc(buf_len, stream->gfp); if (!buf) return NULL; + seq_buf_init(&sb, buf, buf_len); + spin_lock(&stream->lock); list_for_each_entry(frag_container, &stream->fragments, node) - strlcat(buf, frag_container->fragment, buf_len); + seq_buf_puts(&sb, frag_container->fragment); spin_unlock(&stream->lock); return buf; diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 99773e000e1b..09e3dabfac0c 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -214,12 +214,18 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) const struct kunit_case *test_case; enum kunit_status status = KUNIT_SKIPPED; + if (suite->status == KUNIT_SKIPPED) + return KUNIT_SKIPPED; + if (suite->suite_init_err) return KUNIT_FAILURE; kunit_suite_for_each_test_case(suite, test_case) { - if (test_case->status == KUNIT_FAILURE) + if (test_case->status == KUNIT_FAILURE) { + /* Update the kunit_suite status also */ + suite->status = KUNIT_FAILURE; return KUNIT_FAILURE; + } else if (test_case->status == KUNIT_SUCCESS) status = KUNIT_SUCCESS; } @@ -795,12 +801,20 @@ int kunit_run_tests(struct kunit_suite *suite) /* Taint the kernel so we know we've run tests. */ add_taint(TAINT_TEST, LOCKDEP_STILL_OK); + if (suite->status == KUNIT_SKIPPED) + goto suite_end; + if (suite->suite_init) { suite->suite_init_err = suite->suite_init(suite); if (suite->suite_init_err) { + suite->status = KUNIT_FAILURE; kunit_err(suite, KUNIT_SUBTEST_INDENT "# failed to initialize (%d)", suite->suite_init_err); goto suite_end; + + } else if (suite->status == KUNIT_SKIPPED) { + /* Skip this kunit suite */ + goto suite_end; } } @@ -825,6 +839,7 @@ static void kunit_init_suite(struct kunit_suite *suite) kunit_debugfs_create_suite(suite); suite->status_comment[0] = '\0'; suite->suite_init_err = 0; + suite->status = KUNIT_SUCCESS; if (suite->log) string_stream_clear(suite->log); diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index bfafe1204c7b..c3d976c801bb 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures; static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { - int saved_preempt_count = preempt_count(); + long saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT int saved_mgd_count = current->migration_disabled; int saved_rcu_count = current->rcu_read_lock_nesting; diff --git a/lib/lockref.c b/lib/lockref.c index 5d8e3ef3860e..9b3dd688d8cd 100644 --- a/lib/lockref.c +++ b/lib/lockref.c @@ -131,7 +131,7 @@ EXPORT_SYMBOL(lockref_put_or_lock); void lockref_mark_dead(struct lockref *lockref) { assert_spin_locked(&lockref->lock); - lockref->count = -128; + lockref->count = __LOCKREF_DEAD_VAL; } EXPORT_SYMBOL(lockref_mark_dead); diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e52876435b77..1aba6cced713 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas) return mas->status == ma_underflow; } +static inline void mas_make_walkable(struct ma_state *mas) +{ + if (!mas_is_active(mas) && !mas_is_start(mas)) + mas->status = ma_start; +} + static __always_inline struct maple_node *mte_to_node( const struct maple_enode *entry) { @@ -356,11 +362,6 @@ static __always_inline bool mte_is_root(const struct maple_enode *node) return ma_is_root(mte_to_node(node)); } -static inline bool mas_is_root_limits(const struct ma_state *mas) -{ - return !mas->min && mas->max == ULONG_MAX; -} - static __always_inline bool mt_is_alloc(struct maple_tree *mt) { return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); @@ -456,46 +457,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode) } /* - * mas_set_parent() - Set the parent node and encode the slot - * @mas: The maple state - * @enode: The encoded maple node. - * @parent: The encoded maple node that is the parent of @enode. - * @slot: The slot that @enode resides in @parent. - * - * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the - * parent type. - */ -static inline -void mas_set_parent(struct ma_state *mas, struct maple_enode *enode, - const struct maple_enode *parent, unsigned char slot) -{ - unsigned long val = (unsigned long)parent; - unsigned long shift; - unsigned long type; - enum maple_type p_type = mte_node_type(parent); - - MAS_BUG_ON(mas, p_type == maple_dense); - MAS_BUG_ON(mas, p_type == maple_leaf_64); - - switch (p_type) { - case maple_range_64: - case maple_arange_64: - shift = MAPLE_PARENT_SLOT_SHIFT; - type = MAPLE_PARENT_RANGE64; - break; - default: - case maple_dense: - case maple_leaf_64: - shift = type = 0; - break; - } - - val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ - val |= (slot << shift) | type; - mte_to_node(enode)->parent = ma_parent_ptr(val); -} - -/* * mte_parent_slot() - get the parent slot of @enode. * @enode: The encoded maple node. * @@ -877,6 +838,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, } /* + * mas_set_parent_slots() - Bulk operation to set many slot parent pointers + * @mas: The maple state + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that of the @enode. + * @start_slot: The offset into @slot + * @count: The number of slots to set (eg: exclusive) + */ +static inline +void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent, + void __rcu **slots, unsigned char start_slot, unsigned char count) +{ + unsigned long val; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + unsigned char i; + + MAS_BUG_ON(mas, p_type != maple_range_64 && + p_type != maple_arange_64); + + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + + val = (unsigned long)parent; + val &= ~MAPLE_NODE_MASK; + + for (i = 0; i < count; i++) { + unsigned long pval = val | ((start_slot + i) << shift) | type; + struct maple_enode *child; + + child = mt_slot_locked(mas->tree, slots, i); + mte_to_node(child)->parent = ma_parent_ptr(pval); + } +} + +/* * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. * @mat: the ma_topiary, a linked list of dead nodes. * @dead_enode: the node to be marked as dead and added to the tail of the list @@ -1153,6 +1150,79 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used) ma_free_rcu(mte_to_node(used)); } + +#ifdef CONFIG_LOCKDEP +static struct lockdep_map *mas_lockdep_map(struct ma_state *mas) +{ + struct maple_tree *mt = mas->tree; + + if (mt_external_lock(mt)) + return mt->ma_external_lock; + + return &(mt->ma_lock).dep_map; +} + +#endif + +static void mas_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; + u32 seq; + + if (!mas_is_active(mas)) + return; + +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp)); + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (!map) + return; + + seq = lock_sequence(map); + if (seq != UINT_MAX && mas->ld_seq != UINT_MAX) + WARN_ON_ONCE(mas->ld_seq != seq); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP + struct lockdep_map *map; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (!mt_locked(mas->tree)) { + if (mt_in_rcu(mas->tree)) + mas->rcu_gp = get_state_synchronize_rcu(); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + + map = mas_lockdep_map(mas); + if (map) /* Update regardless of lock state */ + mas->ld_seq = lock_sequence(map); +#endif /* CONFIG_LOCKDEP */ + +} + +static void mas_may_init_lock_check(struct ma_state *mas) +{ +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + if (mas_is_start(mas) || mas_is_paused(mas)) { + mas_init_lock_check(mas); + return; + } +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ + mas_lock_check(mas); +#endif /* CONFIG_LOCKDEP */ +} + /* * mas_start() - Sets up maple state for operations. * @mas: The maple state. @@ -1171,6 +1241,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas) if (likely(mas_is_start(mas))) { struct maple_enode *root; + mas_init_lock_check(mas); mas->min = 0; mas->max = ULONG_MAX; @@ -1501,14 +1572,26 @@ ascend: goto ascend; } +static __always_inline void mas_update_gap_known(struct ma_state *mas, + unsigned long gap) +{ + unsigned char pslot; + unsigned long p_gap; + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), + mas_parent_type(mas, mas->node))[pslot]; + + if (p_gap != gap) + mas_parent_gap(mas, pslot, gap); +} + /* * mas_update_gap() - Update a nodes gaps and propagate up if necessary. * @mas: the maple state. */ static inline void mas_update_gap(struct ma_state *mas) { - unsigned char pslot; - unsigned long p_gap; unsigned long max_gap; if (!mt_is_alloc(mas->tree)) @@ -1518,13 +1601,7 @@ static inline void mas_update_gap(struct ma_state *mas) return; max_gap = mas_max_gap(mas); - - pslot = mte_parent_slot(mas->node); - p_gap = ma_gaps(mte_parent(mas->node), - mas_parent_type(mas, mas->node))[pslot]; - - if (p_gap != max_gap) - mas_parent_gap(mas, pslot, max_gap); + mas_update_gap_known(mas, max_gap); } /* @@ -1540,14 +1617,10 @@ static inline void mas_adopt_children(struct ma_state *mas, struct maple_node *node = mte_to_node(parent); void __rcu **slots = ma_slots(node, type); unsigned long *pivots = ma_pivots(node, type); - struct maple_enode *child; - unsigned char offset; + unsigned char end; - offset = ma_data_end(node, type, pivots, mas->max); - do { - child = mas_slot_locked(mas, slots, offset); - mas_set_parent(mas, child, parent, offset); - } while (offset--); + end = ma_data_end(node, type, pivots, mas->max); + mas_set_parent_slots(mas, parent, slots, 0, end + 1); } /* @@ -1929,15 +2002,10 @@ unsigned long node_copy(struct ma_state *mas, struct maple_node *src, s_slots = ma_slots(src, s_mt) + start; s_pivots = ma_pivots(src, s_mt) + start; memcpy(d_slots, s_slots, size * sizeof(void __rcu *)); - if (!ma_is_leaf(d_mt) && s_mt == maple_copy) { - struct maple_enode *edst = mt_mk_node(dst, d_mt); - - for (int i = 0; i < size; i++) - mas_set_parent(mas, - mt_slot_locked(mas->tree, d_slots, i), - edst, d_start + i); - } + if (!ma_is_leaf(d_mt) && s_mt == maple_copy) + mas_set_parent_slots(mas, mt_mk_node(dst, d_mt), + d_slots, d_start, size); d_gaps = ma_gaps(dst, d_mt); if (d_gaps) { @@ -2081,8 +2149,8 @@ static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp) mas->node = mt_slot_locked(mas->tree, cp->slot, 0); /* Insert the new data in the tree */ mas_topiary_replace(mas, old_enode, cp->height); - if (!mte_is_leaf(mas->node)) - mas_update_gap(mas); + if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node)) + mas_update_gap_known(mas, cp->gap[0]); mtree_range_walk(mas); } @@ -2928,13 +2996,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, if (r_mas->last < r_wr_mas->r_max) r_mas->last = r_wr_mas->r_max; r_mas->offset++; - } else if ((r_mas->last == r_wr_mas->r_max) && - (r_mas->last < r_mas->max) && - !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { - r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, - r_wr_mas->type, r_mas->offset + 1); - r_mas->offset++; - r_wr_mas->r_max = r_mas->last; } } @@ -3125,7 +3186,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas) static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) { unsigned char dst_offset, offset_end; - unsigned char copy_size, node_pivots; + unsigned char copy_size, node_pivots, node_slots; struct maple_node reuse, *newnode; unsigned long *dst_pivots; void __rcu **dst_slots; @@ -3138,6 +3199,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) in_rcu = mt_in_rcu(mas->tree); offset_end = wr_mas->offset_end; node_pivots = mt_pivots[wr_mas->type]; + node_slots = mt_slots[wr_mas->type]; /* Assume last adds an entry */ new_end = mas->end + 1 - offset_end + mas->offset; if (mas->last == wr_mas->end_piv) { @@ -3149,7 +3211,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) if (in_rcu) { newnode = mas_pop_node(mas); } else { - memset(&reuse, 0, sizeof(struct maple_node)); newnode = &reuse; } @@ -3193,7 +3254,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) dst_pivots[new_end] = mas->max; done: - mas_leaf_set_meta(newnode, maple_leaf_64, new_end); + if (!in_rcu && new_end + 2 < node_slots) { + unsigned char clear_from = new_end + 1; + + /* + * Note that the last slot is never cleared, since the metadata + * will be stored there or it has a value. + */ + memset(dst_slots + clear_from, 0, + sizeof(void __rcu *) * (node_slots - clear_from)); + if (clear_from < node_pivots) + memset(dst_pivots + clear_from, 0, + sizeof(unsigned long) * (node_pivots - clear_from)); + } + + mas_leaf_set_meta(newnode, wr_mas->type, new_end); if (in_rcu) { struct maple_enode *old_enode = mas->node; @@ -3218,7 +3293,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) void __rcu **slots = wr_mas->slots; bool gap = false; - gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !wr_mas->content; gap |= !mt_slot_locked(mas->tree, slots, offset + 1); if (wr_mas->offset_end - offset == 1) { @@ -3595,7 +3670,7 @@ set_content: /** * mas_prealloc_calc() - Calculate number of nodes needed for a - * given store oepration + * given store operation * @wr_mas: The maple write state * @entry: The entry to store into the tree * @@ -3661,6 +3736,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) { struct ma_state *mas = wr_mas->mas; unsigned char new_end; + bool appending; + bool one_slot; + bool in_rcu; if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) return wr_store_root; @@ -3680,21 +3758,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) return wr_new_root; new_end = mas_wr_new_end(wr_mas); + in_rcu = mt_in_rcu(mas->tree); + appending = mas->offset == mas->end; + one_slot = wr_mas->offset_end - mas->offset == 1; + /* Potential spanning rebalance collapsing a node */ if (new_end < mt_min_slots[wr_mas->type]) { if (!mte_is_root(mas->node)) return wr_rebalance; + if (!in_rcu) { + if (appending) + return wr_append; + else if (mas->end == new_end && one_slot) + return wr_slot_store; + } return wr_node_store; } if (new_end >= mt_slots[wr_mas->type]) return wr_split_store; - if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end)) + if (!in_rcu && appending) return wr_append; - if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) || - (wr_mas->offset_end - mas->offset == 1))) + if (new_end == mas->end && (!in_rcu || one_slot)) return wr_slot_store; return wr_node_store; @@ -3793,35 +3880,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } @@ -4360,8 +4452,8 @@ void *mas_walk(struct ma_state *mas) { void *entry; - if (!mas_is_active(mas) && !mas_is_start(mas)) - mas->status = ma_start; + mas_may_init_lock_check(mas); + mas_make_walkable(mas); retry: entry = mas_state_walk(mas); if (mas_is_start(mas)) { @@ -4831,6 +4923,7 @@ void *mas_store(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); trace_ma_write(TP_FCT, mas, 0, entry); #ifdef CONFIG_DEBUG_MAPLE_TREE if (MAS_WARN_ON(mas, mas->index > mas->last)) @@ -4889,6 +4982,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) MA_WR_STATE(wr_mas, mas, entry); int ret = 0; + mas_may_init_lock_check(mas); retry: mas_wr_preallocate(&wr_mas, entry); if (unlikely(mas_nomem(mas, gfp))) { @@ -4919,6 +5013,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) { MA_WR_STATE(wr_mas, mas, entry); + mas_lock_check(mas); if (mas->store_type == wr_store_root) { mas_wr_prealloc_setup(&wr_mas); goto store; @@ -4951,6 +5046,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) { MA_WR_STATE(wr_mas, mas, entry); + mas_may_init_lock_check(mas); mas_wr_prealloc_setup(&wr_mas); mas->store_type = mas_wr_store_type(&wr_mas); mas_prealloc_calc(&wr_mas, entry); @@ -4997,6 +5093,7 @@ static void mas_may_activate(struct ma_state *mas) mas->status = ma_start; } else { mas->status = ma_active; + mas_lock_check(mas); } } @@ -5074,6 +5171,7 @@ void *mas_next(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5097,6 +5195,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_next_setup(mas, max, &entry)) return entry; @@ -5205,6 +5304,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5228,6 +5328,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_prev_setup(mas, min, &entry)) return entry; @@ -5274,6 +5375,7 @@ EXPORT_SYMBOL_GPL(mt_prev); */ void mas_pause(struct ma_state *mas) { + mas_lock_check(mas); mas->status = ma_pause; mas->node = NULL; } @@ -5382,6 +5484,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5409,6 +5512,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_setup(mas, max, &entry)) return entry; @@ -5428,7 +5532,6 @@ EXPORT_SYMBOL_GPL(mas_find_range); static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) { - switch (mas->status) { case ma_active: goto active; @@ -5521,6 +5624,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5547,6 +5651,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min) { void *entry = NULL; + mas_may_init_lock_check(mas); if (mas_find_rev_setup(mas, min, &entry)) return entry; @@ -5564,6 +5669,11 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev); * Searches for @mas->index, sets @mas->index and @mas->last to the range and * erases that range. * + * Note that erase requires allocations and will use GFP_KERNEL | __GFP_NOFAIL + * to do so if necessary. If the allocation fails, the internal lock will be + * dropped to retry. An externally locked tree must be protected by a lock that + * allows blocking for this API. + * * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. */ void *mas_erase(struct ma_state *mas) @@ -5572,25 +5682,25 @@ void *mas_erase(struct ma_state *mas) unsigned long index = mas->index; MA_WR_STATE(wr_mas, mas, NULL); - if (!mas_is_active(mas) || !mas_is_start(mas)) - mas->status = ma_start; + /* + * In low memory situations, the allocation is retried with the gfp flag + * GFP_KERNEL | __GFP_NOFAIL. The internal spinlock is dropped in + * mas_nomem_nofail(), however the external lock is not dropped. + */ + if (mt_external_lock(mas->tree)) + might_alloc(GFP_KERNEL); + mas_make_walkable(mas); write_retry: entry = mas_state_walk(mas); if (!entry) - return NULL; + goto out; /* Must reset to ensure spanning writes of last slot are detected */ mas_reset(mas); mas_wr_preallocate(&wr_mas, NULL); - if (mas_nomem(mas, GFP_KERNEL)) { - /* in case the range of entry changed when unlocked */ - mas->index = mas->last = index; + if (mas_nomem_nofail(mas, index, index)) goto write_retry; - } - - if (mas_is_err(mas)) - goto out; mas_wr_store_entry(&wr_mas); out: @@ -5601,10 +5711,11 @@ EXPORT_SYMBOL_GPL(mas_erase); /** * mas_nomem() - Check if there was an error allocating and do the allocation - * if necessary If there are allocations, then free them. + * if necessary. + * * @mas: The maple state * @gfp: The GFP_FLAGS to use for allocations - * Return: true on allocation, false otherwise. + * Return: False on no memory. True otherwise (partial success as well) */ bool mas_nomem(struct ma_state *mas, gfp_t gfp) __must_hold(mas->tree->ma_lock) @@ -5612,6 +5723,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) if (likely(mas->node != MA_ERROR(-ENOMEM))) return false; + /* Allocations can fail, don't do this. */ + WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) && + mt_external_lock(mas->tree)); + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { mtree_unlock(mas->tree); mas_alloc_nodes(mas, gfp); @@ -5620,10 +5735,49 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) mas_alloc_nodes(mas, gfp); } + /* + * Return false on zero forward progress. Partial allocations are kept + * so the retry path will attempt to get the rest. The failure should + * not happen as we try our best to reclaim. The user would need an + * external lock with a non-blocking gfp in a low memory situation. + */ if (!mas->sheaf && !mas->alloc) return false; - mas->status = ma_start; + mas_reset(mas); + return true; +} + +/** + * mas_nomem_nofail() - Retry allocations with __GFP_NOFAIL, if the maple state + * has stored the -ENOMEM error. + * @mas: The maple state + * @index: The start of the range for the @mas reset + * @last: The end of the range for the @mas reset + * + * Return: false if @mas isn't in an -ENOMEM state. True if the allocation + * happens, the state is reset. The internal lock will be dropped and external + * locks must allow blocking. + */ +bool mas_nomem_nofail(struct ma_state *mas, unsigned long index, + unsigned long last) + __must_hold(mas->tree->ma_lock) +{ + gfp_t gfp; + + if (likely(mas->node != MA_ERROR(-ENOMEM))) + return false; + + gfp = GFP_KERNEL | __GFP_NOFAIL; + if (!mt_external_lock(mas->tree)) { + mtree_unlock(mas->tree); + mas_alloc_nodes(mas, gfp); + mtree_lock(mas->tree); + } else { + mas_alloc_nodes(mas, gfp); + } + + mas_set_range(mas, index, last); return true; } @@ -5913,6 +6067,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange); * Erasing is the same as a walk to an entry then a store of a NULL to that * ENTIRE range. In fact, it is implemented as such using the advanced API. * + * Note that erase requires allocations and will use GFP_KERNEL | __GFP_NOFAIL + * to do so if necessary. If the allocation fails, the internal lock will be + * dropped to retry. + * * Return: The entry stored at the @index or %NULL */ void *mtree_erase(struct maple_tree *mt, unsigned long index) @@ -5922,6 +6080,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index) MA_STATE(mas, mt, index, index); trace_ma_op(TP_FCT, &mas); + might_alloc(GFP_KERNEL); mtree_lock(mt); entry = mas_erase(&mas); mtree_unlock(mt); @@ -6447,7 +6606,7 @@ static void mt_dump_range(unsigned long min, unsigned long max, { static const char spaces[] = " "; - switch(format) { + switch (format) { case mt_dump_hex: if (min == max) pr_info("%.*s%lx: ", depth * 2, spaces, min); @@ -6489,7 +6648,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, pr_cont(" contents: "); for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) { - switch(format) { + switch (format) { case mt_dump_hex: pr_cont(PTR_FMT " %lX ", node->slot[i], node->pivot[i]); break; @@ -6517,7 +6676,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); @@ -6576,7 +6735,7 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry, if (last == max) break; if (last > max) { - switch(format) { + switch (format) { case mt_dump_hex: pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); diff --git a/lib/math/div64.c b/lib/math/div64.c index d1e92ea24fce..0b10ded09a9b 100644 --- a/lib/math/div64.c +++ b/lib/math/div64.c @@ -158,6 +158,21 @@ u64 div64_u64(u64 dividend, u64 divisor) EXPORT_SYMBOL(div64_u64); #endif +#ifndef div64_s64_rem +s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder) +{ + s64 quot, t, rem; + + quot = div64_u64_rem(abs(dividend), abs(divisor), (u64 *)&rem); + t = dividend >> 63; + *remainder = (rem ^ t) - t; + t = (dividend ^ divisor) >> 63; + + return (quot ^ t) - t; +} +EXPORT_SYMBOL(div64_s64_rem); +#endif + #ifndef div64_s64 s64 div64_s64(s64 dividend, s64 divisor) { diff --git a/lib/math/test_mul_u64_u64_div_u64.c b/lib/math/test_mul_u64_u64_div_u64.c index 338d014f0c73..d12dc05938fb 100644 --- a/lib/math/test_mul_u64_u64_div_u64.c +++ b/lib/math/test_mul_u64_u64_div_u64.c @@ -157,6 +157,7 @@ static void __exit test_exit(void) #define __div64_32 __div64_32 #define div_s64_rem div_s64_rem #define div64_u64_rem div64_u64_rem +#define div64_s64_rem div64_s64_rem #define div64_u64 div64_u64 #define div64_s64 div64_s64 #define iter_div_u64_rem iter_div_u64_rem diff --git a/lib/math/tests/Makefile b/lib/math/tests/Makefile index 13dc96e48408..85e1ad59f29d 100644 --- a/lib/math/tests/Makefile +++ b/lib/math/tests/Makefile @@ -4,5 +4,6 @@ obj-$(CONFIG_GCD_KUNIT_TEST) += gcd_kunit.o obj-$(CONFIG_INT_LOG_KUNIT_TEST) += int_log_kunit.o obj-$(CONFIG_INT_POW_KUNIT_TEST) += int_pow_kunit.o obj-$(CONFIG_INT_SQRT_KUNIT_TEST) += int_sqrt_kunit.o +obj-$(CONFIG_POLYNOMIAL_KUNIT_TEST) += polynomial_kunit.o obj-$(CONFIG_PRIME_NUMBERS_KUNIT_TEST) += prime_numbers_kunit.o obj-$(CONFIG_RATIONAL_KUNIT_TEST) += rational_kunit.o diff --git a/lib/math/tests/polynomial_kunit.c b/lib/math/tests/polynomial_kunit.c new file mode 100644 index 000000000000..ef443b57fc12 --- /dev/null +++ b/lib/math/tests/polynomial_kunit.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> +#include <linux/polynomial.h> + +struct polynomial_test_param { + const struct polynomial *poly; + long data; + long expected; + const char *name; +}; + +/* f(x) = 5 */ +static const struct polynomial poly_constant = { + .total_divider = 1, + .terms = { + {0, 5, 1, 1}, + } +}; + +/* f(x) = 2x^2 + 3x + 5 */ +static const struct polynomial poly_simple = { + .total_divider = 1, + .terms = { + {2, 2, 1, 1}, + {1, 3, 1, 1}, + {0, 5, 1, 1}, + } +}; + +/* f(x) = -5x + 100 */ +static const struct polynomial poly_negative_coef = { + .total_divider = 1, + .terms = { + {1, -5, 1, 1}, + {0, 100, 1, 1}, + } +}; + +/* f(x) = (150x + 50) / 10 */ +static const struct polynomial poly_total_divider = { + .total_divider = 10, + .terms = { + {1, 150, 1, 1}, + {0, 50, 1, 1}, + } +}; + +/* + * f(x) = x / 2 + * divider=2 applied once per multiply: mult_frac(coef, data, 2) = coef*data/2 + */ +static const struct polynomial poly_step_divider = { + .total_divider = 1, + .terms = { + {1, 1, 2, 1}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = (100/500) * x^2 = 0.2 * x^2 + * Encoded as coef=100, divider=10, divider_leftover=5: + * denom = 10^2 * 5 = 500 + */ +static const struct polynomial poly_leftover = { + .total_divider = 1, + .terms = { + {2, 100, 10, 5}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = 2x^3 (single high-degree term, no constant) + * Used to exercise the power loop alone. + */ +static const struct polynomial poly_cubic = { + .total_divider = 1, + .terms = { + {3, 2, 1, 1}, + {0, 0, 1, 1}, + } +}; + +/* + * f(x) = 4x + 1 with a zero-coefficient quadratic term. + * The deg-2 term contributes nothing regardless of input. + */ +static const struct polynomial poly_zero_coef = { + .total_divider = 1, + .terms = { + {2, 0, 1, 1}, + {1, 4, 1, 1}, + {0, 1, 1, 1}, + } +}; + +/* + * f(x) = 9 with total_divider = 0. + * The implementation treats 0 as 1 via `total_divider ?: 1`, so the + * result must equal the constant term unchanged. + */ +static const struct polynomial poly_zero_total_divider = { + .total_divider = 0, + .terms = { + {0, 9, 1, 1}, + } +}; + + +static const struct polynomial_test_param test_params[] = { + { + .poly = &poly_constant, + .data = 0, + .expected = 5, + .name = "Constant polynomial at x=0", + }, + { + .poly = &poly_constant, + .data = 42, + .expected = 5, + .name = "Constant polynomial is independent of input", + }, + { + .poly = &poly_simple, + .data = 0, + .expected = 5, /* zero input collapses all power terms */ + .name = "Zero input yields constant term only", + }, + { + .poly = &poly_simple, + .data = 10, + .expected = 235, /* 2*100 + 3*10 + 5 */ + .name = "Simple quadratic at x=10", + }, + { + .poly = &poly_negative_coef, + .data = 10, + .expected = 50, /* -5*10 + 100 */ + .name = "Negative coefficient at x=10", + }, + { + .poly = &poly_negative_coef, + .data = 20, + .expected = 0, /* -5*20 + 100 = 0 */ + .name = "Negative coefficient result is zero", + }, + { + .poly = &poly_total_divider, + .data = 3, + .expected = 50, /* (150*3 + 50) / 10 = 500/10 */ + .name = "total_divider scales the final sum", + }, + { + .poly = &poly_step_divider, + .data = 100, + .expected = 50, /* 1*100/2 */ + .name = "Per-step divider halves input", + }, + { + .poly = &poly_leftover, + .data = 30, + .expected = 180, /* 100*30^2 / (10^2 * 5) = 90000/500 */ + .name = "divider_leftover with quadratic term", + }, + /* Boundary: unit and negative-unit input */ + { + /* + * data=1: each mult_frac(tmp, 1, divider) strips one factor of + * divider from coef per degree, so coef is left-shifted right + * until intermediate precision is exhausted. + * 2*1 + 3*1 + 5 = 10 + */ + .poly = &poly_simple, + .data = 1, + .expected = 10, + .name = "Boundary: data=1 (unit input)", + }, + { + /* + * data=-1: even degrees produce positive contributions, + * odd degrees produce negative ones. + * 2*(-1)^2 + 3*(-1) + 5 = 2 - 3 + 5 = 4 + */ + .poly = &poly_simple, + .data = -1, + .expected = 4, + .name = "Boundary: data=-1 (negative unit input)", + }, + + /* Boundary: negative non-trivial input */ + { + /* + * 2*(-3)^2 + 3*(-3) + 5 = 18 - 9 + 5 = 14 + * Verifies sign handling for negative data across all degrees. + */ + .poly = &poly_simple, + .data = -3, + .expected = 14, + .name = "Boundary: negative data with quadratic", + }, + + /* Boundary: total_divider = 0 is treated as 1 */ + { + .poly = &poly_zero_total_divider, + .data = 42, + .expected = 9, + .name = "Boundary: total_divider=0 defaults to 1", + }, + + /* Boundary: zero-coefficient high-degree term */ + { + /* + * The deg-2 term has coef=0, so it contributes 0 regardless + * of data. Result: 0 + 4*10 + 1 = 41 + */ + .poly = &poly_zero_coef, + .data = 10, + .expected = 41, + .name = "Boundary: zero-coefficient term is inert", + }, + + /* Boundary: single high-degree term, no constant */ + { + /* 2 * 5^3 = 250; also verifies the loop terminates on deg-0 */ + .poly = &poly_cubic, + .data = 5, + .expected = 250, + .name = "Boundary: single cubic term", + }, + { + /* 2 * (-2)^3 = -16; odd power preserves sign of negative data */ + .poly = &poly_cubic, + .data = -2, + .expected = -16, + .name = "Boundary: single cubic term, negative data", + }, + +}; + +static void get_desc(const struct polynomial_test_param *param, char *desc) +{ + strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(polynomial, test_params, get_desc); + +static void polynomial_calc_test(struct kunit *test) +{ + const struct polynomial_test_param *param = test->param_value; + + KUNIT_EXPECT_EQ(test, polynomial_calc(param->poly, param->data), + param->expected); +} + +static struct kunit_case polynomial_test_cases[] = { + KUNIT_CASE_PARAM(polynomial_calc_test, polynomial_gen_params), + {} +}; + +static struct kunit_suite polynomial_test_suite = { + .name = "math-polynomial", + .test_cases = polynomial_test_cases, +}; + +kunit_test_suites(&polynomial_test_suite); + +MODULE_DESCRIPTION("math.polynomial_calc KUnit test suite"); +MODULE_LICENSE("GPL"); diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index a3bfa9360b23..f39c9eca8888 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -19,6 +19,7 @@ #include <linux/stringify.h> #include <linux/nmi.h> #include <linux/cpu.h> +#include <linux/export.h> #include <linux/sched/debug.h> #ifdef arch_trigger_cpumask_backtrace @@ -129,4 +130,10 @@ bool nmi_cpu_backtrace(struct pt_regs *regs) return false; } NOKPROBE_SYMBOL(nmi_cpu_backtrace); + +void cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + arch_trigger_cpumask_backtrace(mask, exclude_cpu); +} +EXPORT_SYMBOL_GPL(cpumask_backtrace); #endif diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 97772e42b9b2..f8d90689af9f 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref, * @confirm_switch: optional confirmation callback * * There's no reason to use this function for the usual reference counting. - * Use percpu_ref_kill[_and_confirm](). + * Use percpu_ref_kill() or percpu_ref_kill_and_confirm(). * * Schedule switching of @ref to atomic mode. All its percpu counts will * be collected to the main atomic counter. On completion, when all CPUs diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig index 978cd6ba08ac..01f007b2522c 100644 --- a/lib/raid/Kconfig +++ b/lib/raid/Kconfig @@ -29,6 +29,12 @@ config XOR_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. +config XOR_BENCHMARK + bool "Benchmark for xor_gen" + depends on XOR_KUNIT_TEST + help + Include benchmarks in the KUnit test suite for xor_gen. + config RAID6_PQ tristate @@ -54,6 +60,12 @@ config RAID6_PQ_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. +config RAID6_PQ_KUNIT_BENCHMARK + bool "Benchmark for RAID6 PQ" + depends on RAID6_PQ_KUNIT_TEST + help + Include benchmarks in the KUnit test suite for raid P/Q generation. + config RAID6_PQ_BENCHMARK bool "Automatically choose fastest RAID6 PQ functions" depends on RAID6_PQ diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile index 038d6c74d1ba..5cdb7223de2a 100644 --- a/lib/raid/raid6/Makefile +++ b/lib/raid/raid6/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y + ccflags-y += -I $(src) ifeq ($(CONFIG_RAID6_PQ_ARCH),y) diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c index 6f5c89ab2b17..011aa9d0eb3c 100644 --- a/lib/raid/raid6/algos.c +++ b/lib/raid/raid6/algos.c @@ -152,40 +152,37 @@ void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) } EXPORT_SYMBOL_GPL(raid6_recov_datap); -#define RAID6_TIME_JIFFIES_LG2 4 -#define RAID6_TEST_DISKS 8 +#define BENCH_SIZE SZ_4K +#define NR_SRCS 8 +#define NR_DISKS (NR_SRCS + 2) +#define REPS 800U -static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], - const int disks) +static int raid6_choose_gen(void *dptrs[NR_DISKS], const int disks) { - /* work on the second half of the disks */ - int start = (disks >> 1) - 1, stop = disks - 3; const struct raid6_calls *best = NULL; unsigned long bestgenperf = 0; unsigned int i; for (i = 0; i < raid6_nr_algos; i++) { const struct raid6_calls *algo = raid6_algos[i]; - unsigned long perf = 0, j0, j1; + unsigned long perf = 0; + u64 t; + int i; preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { - algo->gen_syndrome(disks, PAGE_SIZE, *dptrs); - perf++; - } + t = ktime_get_ns(); + for (i = 0; i < REPS; i++) + algo->gen_syndrome(disks, BENCH_SIZE, dptrs); + t = max(ktime_get_ns() - t, 1); preempt_enable(); + /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */ + perf = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t); if (perf > bestgenperf) { bestgenperf = perf; best = algo; } - pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, - (perf * HZ * (disks-2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + pr_info("raid6: %-8s gen() %5lu MB/s\n", algo->name, perf); } if (!best) { @@ -197,28 +194,24 @@ static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + best->name, bestgenperf); if (best->xor_syndrome) { - unsigned long perf = 0, j0, j1; + /* work on the second half of the disks */ + int start = (disks / 2) - 1, stop = disks - 3; + u64 t; preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { - best->xor_syndrome(disks, start, stop, - PAGE_SIZE, *dptrs); - perf++; - } + t = ktime_get_ns(); + for (i = 0; i < REPS; i++) + best->xor_syndrome(disks, start, stop, BENCH_SIZE, + dptrs); + t = max(ktime_get_ns() - t, 1); preempt_enable(); - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (perf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); + pr_info("raid6: .... xor() %llu MB/s, rmw enabled\n", + div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS / 2 * 1000, + t)); } return 0; @@ -230,24 +223,20 @@ static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], static int __init raid6_select_algo(void) { - const int disks = RAID6_TEST_DISKS; + const int disks = NR_DISKS; + void *dptrs[NR_DISKS]; char *disk_ptr, *p; - void *dptrs[RAID6_TEST_DISKS]; int i, cycle; int error; if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { pr_info("raid6: skipped pq benchmark and selected %s\n", raid6_algos[raid6_nr_algos - 1]->name); - static_call_update(raid6_gen_syndrome_impl, - raid6_algos[raid6_nr_algos - 1]->gen_syndrome); - static_call_update(raid6_xor_syndrome_impl, - raid6_algos[raid6_nr_algos - 1]->xor_syndrome); return 0; } /* prepare the buffer and fill it circularly with gfmul table */ - disk_ptr = kmalloc(PAGE_SIZE * RAID6_TEST_DISKS, GFP_KERNEL); + disk_ptr = kmalloc_array(NR_DISKS, BENCH_SIZE, GFP_KERNEL); if (!disk_ptr) { pr_err("raid6: Yikes! No memory available.\n"); return -ENOMEM; @@ -255,19 +244,19 @@ static int __init raid6_select_algo(void) p = disk_ptr; for (i = 0; i < disks; i++) - dptrs[i] = p + PAGE_SIZE * i; + dptrs[i] = p + BENCH_SIZE * i; - cycle = ((disks - 2) * PAGE_SIZE) / 65536; + cycle = ((disks - 2) * BENCH_SIZE) / 65536; for (i = 0; i < cycle; i++) { memcpy(p, raid6_gfmul, 65536); p += 65536; } - if ((disks - 2) * PAGE_SIZE % 65536) - memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); + if ((disks - 2) * BENCH_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * BENCH_SIZE % 65536); /* select raid gen_syndrome function */ - error = raid6_choose_gen(&dptrs, disks); + error = raid6_choose_gen(dptrs, disks); kfree(disk_ptr); @@ -329,13 +318,34 @@ static int __init raid6_init(void) static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); + /* + * Pick the last registered implementation as the temporary default until + * calibration happens. + */ + static_call_update(raid6_gen_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->xor_syndrome); + +#ifdef MODULE return raid6_select_algo(); +#else + return 0; +#endif } static void __exit raid6_exit(void) { } +/* + * When built-in we must register the default implementation before md + * initializes, but we don't want calibration to run that early as that + * would delay the boot process. + */ +#ifndef MODULE +device_initcall(raid6_select_algo); +#endif subsys_initcall(raid6_init); module_exit(raid6_exit); MODULE_LICENSE("GPL"); diff --git a/lib/raid/raid6/riscv/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c index 2305940276dd..78e158a3e332 100644 --- a/lib/raid/raid6/riscv/recov_rvv.c +++ b/lib/raid/raid6/riscv/recov_rvv.c @@ -8,6 +8,7 @@ #include <linux/raid/pq.h> #include "algos.h" #include "rvv.h" +#include "pq_arch.h" static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, u8 *dq, const u8 *pbmul, diff --git a/lib/raid/raid6/riscv/rvv.c b/lib/raid/raid6/riscv/rvv.c index 75c9dafedb28..4ac50606f3dc 100644 --- a/lib/raid/raid6/riscv/rvv.c +++ b/lib/raid/raid6/riscv/rvv.c @@ -10,6 +10,7 @@ */ #include "rvv.h" +#include "pq_arch.h" #ifdef __riscv_vector #error "This code must be built without compiler support for vector" diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c index 9f3e671a1224..7d0a1ec98001 100644 --- a/lib/raid/raid6/tests/raid6_kunit.c +++ b/lib/raid/raid6/tests/raid6_kunit.c @@ -18,6 +18,7 @@ MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); #define RAID6_KUNIT_MAX_BUFFERS 64 /* Including P and Q */ #define RAID6_KUNIT_MAX_FAILURES 2 #define RAID6_KUNIT_MAX_BYTES PAGE_SIZE +#define RAID6_KUNIT_ALLOC_BYTES SZ_16K static struct rnd_state rng; static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS]; @@ -229,6 +230,68 @@ static void raid6_test(struct kunit *test) raid6_test_one(test); } +static void raid6_benchmark(struct kunit *test) +{ + static const unsigned int nr_to_test[] = { + 4, 5, 6, 7, 8, 10, 12, 15, 16, 32, + }; + static const unsigned int len_to_test[] = { + SZ_4K, SZ_16K, + }; + unsigned int i, j, l; + u64 t; + + if (!IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_BENCHMARK)) + kunit_skip(test, "not enabled"); + + /* warm-up */ + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + for (l = 0; l < 10; l++) { + raid6_gen_syndrome(nr_to_test[i], + len_to_test[j], test_buffers); + } + } + } + + /* + * Preferably this would be a loop over len_to_test, but the kunit + * logging always adds a newline to each logged format string. + */ + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, " \t%5u bytes\t%5u bytes\n", + len_to_test[0], len_to_test[1]); + + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + unsigned int nr = nr_to_test[i]; + u64 speed[ARRAY_SIZE(len_to_test)]; + + KUNIT_ASSERT_LE(test, nr, RAID6_KUNIT_MAX_BUFFERS); + + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + unsigned int len = len_to_test[j]; + const unsigned long num_iters = 1000; + + KUNIT_ASSERT_GT(test, len, 0); + KUNIT_ASSERT_LE(test, len, RAID6_KUNIT_ALLOC_BYTES); + + preempt_disable(); + t = ktime_get_ns(); + for (l = 0; l < num_iters; l++) + raid6_gen_syndrome(nr_to_test[i], + len_to_test[j], test_buffers); + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + speed[j] = div64_u64((u64)len * num_iters * nr, t); + } + + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, "%3u disks:\t%5llu GB/s\t%5llu GB/s\n", + nr, speed[0], speed[1]); + } +} + static const void *raid6_gen_params(struct kunit *test, const void *prev, char *desc) { @@ -256,6 +319,7 @@ next_algo: static struct kunit_case raid6_test_cases[] = { KUNIT_CASE_PARAM(raid6_test, raid6_gen_params), + KUNIT_CASE(raid6_benchmark), {}, }; @@ -270,7 +334,7 @@ static int raid6_suite_init(struct kunit_suite *suite) * so that it is immediately followed by a guard page. This allows * buffer overreads to be detected, even in assembly code. */ - test_buflen = round_up(RAID6_KUNIT_MAX_BYTES, PAGE_SIZE); + test_buflen = round_up(RAID6_KUNIT_ALLOC_BYTES, PAGE_SIZE); for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) { test_recov_buffers[i] = vmalloc(test_buflen); if (!test_recov_buffers[i]) diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile index e8ecec3c09f9..9b0fad459cdb 100644 --- a/lib/raid/xor/Makefile +++ b/lib/raid/xor/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y + ccflags-y += -I $(src) obj-$(CONFIG_XOR_BLOCKS) += xor.o diff --git a/lib/raid/xor/tests/Makefile b/lib/raid/xor/tests/Makefile index 661e8f6ffd1f..1cce833cd7fd 100644 --- a/lib/raid/xor/tests/Makefile +++ b/lib/raid/xor/tests/Makefile @@ -1,3 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +CONTEXT_ANALYSIS := y + obj-$(CONFIG_XOR_KUNIT_TEST) += xor_kunit.o diff --git a/lib/raid/xor/tests/xor_kunit.c b/lib/raid/xor/tests/xor_kunit.c index 0c2a3a420bf9..1fb30b2c7c47 100644 --- a/lib/raid/xor/tests/xor_kunit.c +++ b/lib/raid/xor/tests/xor_kunit.c @@ -85,7 +85,7 @@ static void xor_test(struct kunit *test) xor_generate_random_data(); /* - * If we're not using the entire buffer size, inject randomize + * If we're not using the entire buffer size, inject randomized * alignment into the buffer. */ max_alignment = XOR_KUNIT_MAX_BYTES - len; @@ -125,8 +125,70 @@ static void xor_test(struct kunit *test) } } +static void xor_benchmark(struct kunit *test) +{ + static const unsigned int nr_to_test[] = { + 4, 5, 6, 7, 8, 10, 12, 15, 16, 32, + }; + static const unsigned int len_to_test[] = { + SZ_4K, SZ_16K, + }; + unsigned int i, j, l; + u64 t; + + if (!IS_ENABLED(CONFIG_XOR_BENCHMARK)) + kunit_skip(test, "not enabled"); + + /* warm-up */ + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + for (l = 0; l < 10; l++) { + xor_gen(test_dest, test_buffers, nr_to_test[i], + len_to_test[j]); + } + } + } + + /* + * Preferably this would be a loop over len_to_test, but the kunit + * logging always adds a newline to each logged format string. + */ + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, " \t%5u bytes\t%5u bytes\n", + len_to_test[0], len_to_test[1]); + + for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) { + unsigned int nr = nr_to_test[i]; + u64 speed[ARRAY_SIZE(len_to_test)]; + + KUNIT_ASSERT_LE(test, nr, XOR_KUNIT_MAX_BUFFERS); + + for (j = 0; j < ARRAY_SIZE(len_to_test); j++) { + unsigned int len = len_to_test[j]; + const unsigned long num_iters = 1000; + + KUNIT_ASSERT_GT(test, len, 0); + KUNIT_ASSERT_LE(test, len, XOR_KUNIT_MAX_BYTES); + + preempt_disable(); + t = ktime_get_ns(); + for (l = 0; l < num_iters; l++) + xor_gen(test_dest, test_buffers, nr, len); + t = max(ktime_get_ns() - t, 1); + preempt_enable(); + + speed[j] = div64_u64((u64)len * num_iters * nr, t); + } + + static_assert(ARRAY_SIZE(len_to_test) == 2); + kunit_info(test, "%3u disks:\t%5llu GB/s\t%5llu GB/s\n", + nr, speed[0], speed[1]); + } +} + static struct kunit_case xor_test_cases[] = { KUNIT_CASE(xor_test), + KUNIT_CASE(xor_benchmark), {}, }; diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c index 50931fbf0324..f9d61b096b11 100644 --- a/lib/raid/xor/xor-core.c +++ b/lib/raid/xor/xor-core.c @@ -10,7 +10,6 @@ #include <linux/gfp.h> #include <linux/slab.h> #include <linux/raid/xor.h> -#include <linux/jiffies.h> #include <linux/preempt.h> #include <linux/static_call.h> #include "xor_impl.h" @@ -73,59 +72,56 @@ void __init xor_force(struct xor_block_template *tmpl) forced_template = tmpl; } -#define BENCH_SIZE 4096 +#define BENCH_SIZE SZ_4K +#define NR_SRCS 4 #define REPS 800U -static void __init -do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2) +static void __init do_xor_speed(struct xor_block_template *tmpl, void *dest, + void *srcs[NR_SRCS]) { - int speed; - unsigned long reps; - ktime_t min, start, t0; - void *srcs[1] = { b2 }; + u64 t; + int i; preempt_disable(); - - reps = 0; - t0 = ktime_get(); - /* delay start until time has advanced */ - while ((start = ktime_get()) == t0) - cpu_relax(); - do { + t = ktime_get_ns(); + for (i = 0; i < REPS; i++) { mb(); /* prevent loop optimization */ - tmpl->xor_gen(b1, srcs, 1, BENCH_SIZE); + tmpl->xor_gen(dest, srcs, NR_SRCS, BENCH_SIZE); mb(); - } while (reps++ < REPS || (t0 = ktime_get()) == start); - min = ktime_sub(t0, start); - + } + t = max(ktime_get_ns() - t, 1); preempt_enable(); - // bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] - speed = (1000 * reps * BENCH_SIZE) / (unsigned int)ktime_to_ns(min); - tmpl->speed = speed; + /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */ + tmpl->speed = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t); - pr_info(" %-16s: %5d MB/sec\n", tmpl->name, speed); + pr_info(" %-16s: %5d MB/sec\n", tmpl->name, tmpl->speed); } static int __init calibrate_xor_blocks(void) { - void *b1, *b2; struct xor_block_template *f, *fastest; + void *srcs[NR_SRCS]; + void *buf, *dest; + int i; if (forced_template) return 0; - b1 = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); - if (!b1) { + buf = kmalloc(BENCH_SIZE * (NR_SRCS + 1), GFP_KERNEL); + if (!buf) { pr_warn("xor: Yikes! No memory available.\n"); return -ENOMEM; } - b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE; + get_random_bytes(buf, BENCH_SIZE * (NR_SRCS + 1)); + dest = buf; + for (i = 0; i < NR_SRCS; i++) + srcs[i] = buf + (i + 1) * BENCH_SIZE; pr_info("xor: measuring software checksum speed\n"); fastest = template_list; for (f = template_list; f; f = f->next) { - do_xor_speed(f, b1, b2); + do_xor_speed(f, dest, srcs); if (f->speed > fastest->speed) fastest = f; } @@ -133,9 +129,10 @@ static int __init calibrate_xor_blocks(void) pr_info("xor: using function: %s (%d MB/sec)\n", fastest->name, fastest->speed); - kfree(b1); + kfree(buf); return 0; } +#undef NR_SRCS #ifdef CONFIG_XOR_BLOCKS_ARCH #include "xor_arch.h" /* $SRCARCH/xor_arch.h */ diff --git a/lib/random32.c b/lib/random32.c index 24e7acd9343f..dad90219c351 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -41,6 +41,7 @@ #include <linux/bitops.h> #include <linux/slab.h> #include <linux/unaligned.h> +#include <kunit/visibility.h> /** * prandom_u32_state - seeded pseudo-random number generator. @@ -92,7 +93,14 @@ void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes) } EXPORT_SYMBOL(prandom_bytes_state); -static void prandom_warmup(struct rnd_state *state) +/* + * Only declared here so that it has a prototype when made + * non-static for KUnit testing (avoids -Wmissing-prototypes). + */ +#if IS_ENABLED(CONFIG_KUNIT) +void prandom_warmup(struct rnd_state *state); +#endif +VISIBLE_IF_KUNIT void prandom_warmup(struct rnd_state *state) { /* Calling RNG ten times to satisfy recurrence condition */ prandom_u32_state(state); @@ -106,6 +114,7 @@ static void prandom_warmup(struct rnd_state *state) prandom_u32_state(state); prandom_u32_state(state); } +EXPORT_SYMBOL_IF_KUNIT(prandom_warmup); void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) { @@ -125,176 +134,3 @@ void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state) } } EXPORT_SYMBOL(prandom_seed_full_state); - -#ifdef CONFIG_RANDOM32_SELFTEST -static struct prandom_test1 { - u32 seed; - u32 result; -} test1[] = { - { 1U, 3484351685U }, - { 2U, 2623130059U }, - { 3U, 3125133893U }, - { 4U, 984847254U }, -}; - -static struct prandom_test2 { - u32 seed; - u32 iteration; - u32 result; -} test2[] = { - /* Test cases against taus113 from GSL library. */ - { 931557656U, 959U, 2975593782U }, - { 1339693295U, 876U, 3887776532U }, - { 1545556285U, 961U, 1615538833U }, - { 601730776U, 723U, 1776162651U }, - { 1027516047U, 687U, 511983079U }, - { 416526298U, 700U, 916156552U }, - { 1395522032U, 652U, 2222063676U }, - { 366221443U, 617U, 2992857763U }, - { 1539836965U, 714U, 3783265725U }, - { 556206671U, 994U, 799626459U }, - { 684907218U, 799U, 367789491U }, - { 2121230701U, 931U, 2115467001U }, - { 1668516451U, 644U, 3620590685U }, - { 768046066U, 883U, 2034077390U }, - { 1989159136U, 833U, 1195767305U }, - { 536585145U, 996U, 3577259204U }, - { 1008129373U, 642U, 1478080776U }, - { 1740775604U, 939U, 1264980372U }, - { 1967883163U, 508U, 10734624U }, - { 1923019697U, 730U, 3821419629U }, - { 442079932U, 560U, 3440032343U }, - { 1961302714U, 845U, 841962572U }, - { 2030205964U, 962U, 1325144227U }, - { 1160407529U, 507U, 240940858U }, - { 635482502U, 779U, 4200489746U }, - { 1252788931U, 699U, 867195434U }, - { 1961817131U, 719U, 668237657U }, - { 1071468216U, 983U, 917876630U }, - { 1281848367U, 932U, 1003100039U }, - { 582537119U, 780U, 1127273778U }, - { 1973672777U, 853U, 1071368872U }, - { 1896756996U, 762U, 1127851055U }, - { 847917054U, 500U, 1717499075U }, - { 1240520510U, 951U, 2849576657U }, - { 1685071682U, 567U, 1961810396U }, - { 1516232129U, 557U, 3173877U }, - { 1208118903U, 612U, 1613145022U }, - { 1817269927U, 693U, 4279122573U }, - { 1510091701U, 717U, 638191229U }, - { 365916850U, 807U, 600424314U }, - { 399324359U, 702U, 1803598116U }, - { 1318480274U, 779U, 2074237022U }, - { 697758115U, 840U, 1483639402U }, - { 1696507773U, 840U, 577415447U }, - { 2081979121U, 981U, 3041486449U }, - { 955646687U, 742U, 3846494357U }, - { 1250683506U, 749U, 836419859U }, - { 595003102U, 534U, 366794109U }, - { 47485338U, 558U, 3521120834U }, - { 619433479U, 610U, 3991783875U }, - { 704096520U, 518U, 4139493852U }, - { 1712224984U, 606U, 2393312003U }, - { 1318233152U, 922U, 3880361134U }, - { 855572992U, 761U, 1472974787U }, - { 64721421U, 703U, 683860550U }, - { 678931758U, 840U, 380616043U }, - { 692711973U, 778U, 1382361947U }, - { 677703619U, 530U, 2826914161U }, - { 92393223U, 586U, 1522128471U }, - { 1222592920U, 743U, 3466726667U }, - { 358288986U, 695U, 1091956998U }, - { 1935056945U, 958U, 514864477U }, - { 735675993U, 990U, 1294239989U }, - { 1560089402U, 897U, 2238551287U }, - { 70616361U, 829U, 22483098U }, - { 368234700U, 731U, 2913875084U }, - { 20221190U, 879U, 1564152970U }, - { 539444654U, 682U, 1835141259U }, - { 1314987297U, 840U, 1801114136U }, - { 2019295544U, 645U, 3286438930U }, - { 469023838U, 716U, 1637918202U }, - { 1843754496U, 653U, 2562092152U }, - { 400672036U, 809U, 4264212785U }, - { 404722249U, 965U, 2704116999U }, - { 600702209U, 758U, 584979986U }, - { 519953954U, 667U, 2574436237U }, - { 1658071126U, 694U, 2214569490U }, - { 420480037U, 749U, 3430010866U }, - { 690103647U, 969U, 3700758083U }, - { 1029424799U, 937U, 3787746841U }, - { 2012608669U, 506U, 3362628973U }, - { 1535432887U, 998U, 42610943U }, - { 1330635533U, 857U, 3040806504U }, - { 1223800550U, 539U, 3954229517U }, - { 1322411537U, 680U, 3223250324U }, - { 1877847898U, 945U, 2915147143U }, - { 1646356099U, 874U, 965988280U }, - { 805687536U, 744U, 4032277920U }, - { 1948093210U, 633U, 1346597684U }, - { 392609744U, 783U, 1636083295U }, - { 690241304U, 770U, 1201031298U }, - { 1360302965U, 696U, 1665394461U }, - { 1220090946U, 780U, 1316922812U }, - { 447092251U, 500U, 3438743375U }, - { 1613868791U, 592U, 828546883U }, - { 523430951U, 548U, 2552392304U }, - { 726692899U, 810U, 1656872867U }, - { 1364340021U, 836U, 3710513486U }, - { 1986257729U, 931U, 935013962U }, - { 407983964U, 921U, 728767059U }, -}; - -static void prandom_state_selftest_seed(struct rnd_state *state, u32 seed) -{ -#define LCG(x) ((x) * 69069U) /* super-duper LCG */ - state->s1 = __seed(LCG(seed), 2U); - state->s2 = __seed(LCG(state->s1), 8U); - state->s3 = __seed(LCG(state->s2), 16U); - state->s4 = __seed(LCG(state->s3), 128U); -} - -static int __init prandom_state_selftest(void) -{ - int i, j, errors = 0, runs = 0; - bool error = false; - - for (i = 0; i < ARRAY_SIZE(test1); i++) { - struct rnd_state state; - - prandom_state_selftest_seed(&state, test1[i].seed); - prandom_warmup(&state); - - if (test1[i].result != prandom_u32_state(&state)) - error = true; - } - - if (error) - pr_warn("prandom: seed boundary self test failed\n"); - else - pr_info("prandom: seed boundary self test passed\n"); - - for (i = 0; i < ARRAY_SIZE(test2); i++) { - struct rnd_state state; - - prandom_state_selftest_seed(&state, test2[i].seed); - prandom_warmup(&state); - - for (j = 0; j < test2[i].iteration - 1; j++) - prandom_u32_state(&state); - - if (test2[i].result != prandom_u32_state(&state)) - errors++; - - runs++; - cond_resched(); - } - - if (errors) - pr_warn("prandom: %d/%d self tests failed\n", errors, runs); - else - pr_info("prandom: %d self tests passed\n", runs); - return 0; -} -core_initcall(prandom_state_selftest); -#endif diff --git a/lib/region_alloc_benchmark.c b/lib/region_alloc_benchmark.c new file mode 100644 index 000000000000..e88b4cf55c62 --- /dev/null +++ b/lib/region_alloc_benchmark.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Benchmark bitmap, IDA and Maple Tree allocation of variable-sized regions. */ + +#include <linux/bitmap.h> +#include <linux/idr.h> +#include <linux/kernel.h> +#include <linux/maple_tree.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/xarray.h> + +#define REGION_MAX_SIZE 32 + +static unsigned long *bitmap __initdata; +/* One more request guarantees that even an all-ones trace reaches ENOSPC. */ +static u8 *reg_sz __initdata; +static unsigned long *reg_idx __initdata; +static unsigned long capacities[64] = { 1000000, 100000, 10000, 1000, 100, 10 }; +static unsigned int cap_cnt = 6; + +module_param_array(capacities, ulong, &cap_cnt, 0400); +MODULE_PARM_DESC(capacities, "Region capacities to benchmark"); + +static unsigned long __init benchmark_bitmap(unsigned long cap) +{ + unsigned long cnt, idx; + ktime_t alloc_time, free_time; + size_t sz; + + bitmap_zero(bitmap, cap); + alloc_time = ktime_get(); + for (cnt = 0; cnt <= cap; cnt++) { + idx = bitmap_find_next_zero_area(bitmap, cap, 0, reg_sz[cnt], 0); + if (idx >= cap) + break; + + reg_idx[cnt] = idx; + bitmap_set(bitmap, idx, reg_sz[cnt]); + } + alloc_time = ktime_get() - alloc_time; + + idx = cnt; + + free_time = ktime_get(); + while (idx--) + bitmap_clear(bitmap, reg_idx[idx], reg_sz[idx]); + free_time = ktime_get() - free_time; + + WARN_ON(!bitmap_empty(bitmap, cap)); + + sz = BITS_TO_LONGS(cap) * sizeof(unsigned long); + pr_err("Bitmap %12llu %12llu %8lu %8lu %10zu\n", + alloc_time, free_time, cnt, cap, sz); + + return cnt; +} + +static size_t __init ida_size(unsigned long nr_ids) +{ + unsigned long entries = DIV_ROUND_UP(nr_ids, IDA_BITMAP_BITS); + unsigned long bitmaps = nr_ids / IDA_BITMAP_BITS; + unsigned long nodes = 0; + + if (nr_ids % IDA_BITMAP_BITS > BITS_PER_XA_VALUE) + bitmaps++; + + while (entries > 1) { + entries = DIV_ROUND_UP(entries, XA_CHUNK_SIZE); + nodes += entries; + } + + return sizeof(struct ida) + + bitmaps * sizeof(struct ida_bitmap) + + nodes * sizeof(struct xa_node); +} + +static unsigned long __init benchmark_ida(unsigned long cap) +{ + struct ida ida = IDA_INIT(ida); + unsigned long cnt, idx, off, nr_ids = 0; + ktime_t alloc_time, free_time; + int id = -ENOSPC; + + alloc_time = ktime_get(); + for (cnt = 0; cnt <= cap; cnt++) { + for (off = 0; off < reg_sz[cnt]; off++) { + id = ida_alloc_max(&ida, cap - 1, GFP_KERNEL); + if (id < 0) + break; + + if (!off) + reg_idx[cnt] = id; + } + if (id < 0) { + while (off--) + ida_free(&ida, reg_idx[cnt] + off); + break; + } + WARN_ON(id != reg_idx[cnt] + reg_sz[cnt] - 1); + nr_ids += reg_sz[cnt]; + } + alloc_time = ktime_get() - alloc_time; + + WARN_ON(id != -ENOSPC); + + idx = cnt; + + free_time = ktime_get(); + while (idx--) { + for (off = 0; off < reg_sz[idx]; off++) + ida_free(&ida, reg_idx[idx] + off); + } + free_time = ktime_get() - free_time; + + WARN_ON(!ida_is_empty(&ida)); + + pr_err("IDA %12llu %12llu %8lu %8lu %10zu\n", + alloc_time, free_time, cnt, cap, ida_size(nr_ids)); + + ida_destroy(&ida); + return cnt; +} + +static unsigned long __init benchmark_maple_tree(unsigned long cap) +{ + struct maple_tree mt = MTREE_INIT(mt, MT_FLAGS_ALLOC_RANGE); + unsigned long cnt, idx; + ktime_t alloc_time, free_time; + size_t sz; + int ret; + + alloc_time = ktime_get(); + for (cnt = 0; cnt <= cap; cnt++) { + ret = mtree_alloc_range(&mt, &idx, xa_mk_value(cnt + 1), + reg_sz[cnt], 0, cap - 1, GFP_KERNEL); + if (ret) + break; + + reg_idx[cnt] = idx; + } + alloc_time = ktime_get() - alloc_time; + + WARN_ON(ret != -EBUSY); + + idx = cnt; + + free_time = ktime_get(); + while (idx--) + mtree_erase(&mt, reg_idx[idx]); + free_time = ktime_get() - free_time; + + WARN_ON(!mtree_empty(&mt)); + + /* Minimum storage assuming fully occupied allocation-range leaf nodes. */ + sz = sizeof(mt) + DIV_ROUND_UP(cnt, MAPLE_ARANGE64_SLOTS) * sizeof(struct maple_node); + pr_err("Maple %12llu %12llu %8lu %8lu %10zu\n", + alloc_time, free_time, cnt, cap, sz); + + mtree_destroy(&mt); + return cnt; +} + +static int __init region_alloc_benchmark(void) +{ + unsigned long bitmap_count, ida_count, maple_count; + unsigned long i, max_cap = 0; + int ret = -ENOMEM; + + for (i = 0; i < cap_cnt; i++) { + if (capacities[i] == 0) { + pr_err("capacity must be nonzero\n"); + return -EINVAL; + } + max_cap = max(max_cap, capacities[i]); + } + + bitmap = kvmalloc_array(BITS_TO_LONGS(max_cap), sizeof(*bitmap), GFP_KERNEL); + reg_sz = kvmalloc_array(max_cap + 1, sizeof(*reg_sz), GFP_KERNEL); + reg_idx = kvmalloc_array(max_cap, sizeof(*reg_idx), GFP_KERNEL); + if (!bitmap || !reg_sz || !reg_idx) + goto out; + + pr_err("\nStart testing bitmap vs IDA vs Maple Tree region allocation\n"); + pr_err("memory: bitmap is exact; IDA and Maple Tree are lower bounds\n"); + pr_err("Type alloc (ns) free (ns) regions capacity memory (B)\n"); + + for (i = 0; i < cap_cnt; i++) { + unsigned long idx, max_size; + + max_size = min(REGION_MAX_SIZE, capacities[i] / 10) ? : 1; + for (idx = 0; idx <= capacities[i]; idx++) + reg_sz[idx] = get_random_u32_below(max_size) + 1; + + bitmap_count = benchmark_bitmap(capacities[i]); + maple_count = benchmark_maple_tree(capacities[i]); + ida_count = benchmark_ida(capacities[i]); + + WARN_ON(bitmap_count != ida_count); + WARN_ON(bitmap_count != maple_count); + } + + /* Return an error so the benchmark can run repeatedly without rmmod. */ + pr_info("Region allocation benchmark complete\n"); + ret = -EAGAIN; +out: + kvfree(reg_idx); + kvfree(reg_sz); + kvfree(bitmap); + return ret; +} +module_init(region_alloc_benchmark); + +MODULE_AUTHOR("Yury Norov <ynorov@nvidia.com>"); +MODULE_DESCRIPTION("Benchmark bitmap, IDA and Maple Tree region allocation"); +MODULE_LICENSE("GPL"); diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 40cfb38ac919..6362896e4f09 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -14,6 +14,7 @@ #include <linux/atomic.h> #include <linux/kernel.h> #include <linux/init.h> +#include <linux/irq_work.h> #include <linux/log2.h> #include <linux/sched.h> #include <linux/rculist.h> @@ -25,6 +26,7 @@ #include <linux/rhashtable.h> #include <linux/err.h> #include <linux/export.h> +#include <linux/workqueue.h> #define HASH_DEFAULT_SIZE 64UL #define HASH_MIN_SIZE 4U @@ -878,6 +880,7 @@ int rhashtable_walk_start_check(struct rhashtable_iter *iter) iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht); iter->slot = 0; iter->skip = 0; + iter->p = NULL; return -EAGAIN; } @@ -1260,7 +1263,7 @@ static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, list = container_of(obj, struct rhlist_head, rhead); do { obj = &list->rhead; - list = rht_dereference(list->next, ht); + list = rcu_dereference_raw(list->next); free_fn(rht_obj(ht, obj), arg); } while (list); } diff --git a/lib/scatterlist.c b/lib/scatterlist.c index b7fe91ef35b8..6ea40d2e6247 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1366,6 +1366,7 @@ static ssize_t extract_xarray_to_sg(struct iov_iter *iter, sg_max--; maxsize -= len; + start += len; ret += len; if (maxsize <= 0 || sg_max == 0) break; diff --git a/lib/seq_buf.c b/lib/seq_buf.c index b59488fa8135..a92093f346da 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -321,7 +321,7 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) if (size) { char *p = d_path(path, buf, size); if (!IS_ERR(p)) { - char *end = mangle_path(buf, p, esc); + char *end = seq_mangle_path(buf, p, esc); if (end) res = end - buf; } diff --git a/lib/string.c b/lib/string.c index a4e8ad23577d..a3778d5aab4a 100644 --- a/lib/string.c +++ b/lib/string.c @@ -821,7 +821,8 @@ void *memchr_inv(const void *start, int c, size_t bytes) { u8 value = c; u64 value64; - unsigned int words, prefix; + size_t words; + unsigned int prefix; if (bytes <= 16) return check_bytes8(start, value, bytes); diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c index 811128d0df16..9255234752db 100644 --- a/lib/test-kstrtox.c +++ b/lib/test-kstrtox.c @@ -709,6 +709,182 @@ static void __init test_kstrtos8_fail(void) TEST_FAIL(kstrtos8, s8, "%hhd", test_s8_fail); } +static void __init test_kstrtoudec64_ok(void) +{ + DECLARE_TEST_OK(u64, struct test_udec64); + static DEFINE_TEST_OK(struct test_udec64, test_udec64_ok) = { + /* basic: integer.fraction, exact digits */ + {"0.0", 1, 0}, + {"1.5", 1, 15}, + {"1.234", 3, 1234}, + {"42.0", 1, 420}, + /* zero */ + {"0.0", 1, 0}, + {"0.000", 3, 0}, + /* integer only */ + {"0", 1, 0}, + {"42", 3, 42000}, + {"123.", 2, 12300}, + {"1", 1, 10}, + /* fractional only (leading dot) */ + {".5", 1, 5}, + {".5", 0, 0}, + {".123", 3, 123}, + {".001", 3, 1}, + /* zero padding: fewer fractional digits than scale */ + {"1.2", 3, 1200}, + {"1.2", 6, 1200000}, + {"0.01", 3, 10}, + {"0.1", 9, 100000000ULL}, + {"0.01", 9, 10000000}, + /* truncation: more fractional digits than scale */ + {"1.23456", 3, 1234}, + {"3.1415926535", 6, 3141592}, + {"0.999999999", 3, 999}, + {"1.99", 1, 19}, + {"1.234", 0, 1}, + /* trailing newline */ + {"1.5\n", 1, 15}, + {"42\n", 3, 42000}, + /* plus sign */ + {"+1.5", 1, 15}, + {"+.5", 1, 5}, + /* scale progression */ + {"1.", 0, 1}, + {"1.0", 1, 10}, + {"1.00", 2, 100}, + {"1.000", 3, 1000}, + {"1.000000", 6, 1000000}, + {"1.000000000", 9, 1000000000ULL}, + /* max limit check */ + {"18446744073.709551615", 9, ULLONG_MAX}, + {"18446744073709.551615", 6, ULLONG_MAX}, + {"0.18446744073709551615", 20, ULLONG_MAX}, + /* scale > 19: representable when integer part is small */ + {"0.00000000000000000001", 20, 1}, + {"0.1", 20, 10000000000000000000ULL}, + {"0.00000000000000000000001", 23, 1}, + /* truncation with scale > 19 */ + {"0.0000000000000000000012345", 23, 123}, + /* truncation with many excess digits */ + {"0.00000000000000000000000000000000423", 34, 42}, + {"1.99999999999999999999999999999999999", 3, 1999}, + }; + TEST_OK(kstrtoudec64, u64, "%llu", test_udec64_ok); +} + +static void __init test_kstrtoudec64_fail(void) +{ + static DEFINE_TEST_FAIL(test_udec64_fail) = { + /* empty / whitespace */ + {"", 3}, + {"\n", 3}, + /* minus sign (unsigned) */ + {"-1.5", 1}, + {"-0.5", 1}, + /* only a decimal point */ + {".", 3}, + {".", 0}, + /* only a sign */ + {"+", 3}, + /* non-digit characters */ + {"abc", 3}, + {"1.2x", 3}, + /* leading/trailing space */ + {" 1.5", 1}, + {"1.5 ", 1}, + /* overflow */ + {"18446744073710.551615", 6}, + {"99999999999999999999", 1}, + /* overflow with scale > 19 */ + {"1.0", 21}, + {"0.2", 20}, + {"0.18446744073709551616", 20}, + {"1", 20}, + }; + TEST_FAIL(kstrtoudec64, u64, "%llu", test_udec64_fail); +} + +static void __init test_kstrtodec64_ok(void) +{ + DECLARE_TEST_OK(s64, struct test_dec64); + static DEFINE_TEST_OK(struct test_dec64, test_dec64_ok) = { + /* basic positive */ + {"0.0", 1, 0}, + {"1.5", 1, 15}, + {"1.234", 3, 1234}, + /* basic negative */ + {"-1.5", 1, -15}, + {"-1.234", 3, -1234}, + {"-0.5", 1, -5}, + {"-0.001", 3, -1}, + /* zero (signed) */ + {"-0", 1, 0}, + {"-0.0", 1, 0}, + {"0.000", 3, 0}, + /* integer only */ + {"42", 3, 42000}, + {"-42", 3, -42000}, + /* fractional only */ + {".5", 1, 5}, + {"-.5", 1, -5}, + /* zero padding */ + {"1.2", 3, 1200}, + {"-1.2", 3, -1200}, + {"0.01", 3, 10}, + {"-0.01", 3, -10}, + /* truncation */ + {"1.23456", 3, 1234}, + {"-1.23456", 3, -1234}, + {"0.999999999", 3, 999}, + {"-0.999999999", 3, -999}, + /* trailing newline */ + {"1.5\n", 1, 15}, + {"-1.5\n", 1, -15}, + /* plus sign */ + {"+1.5", 1, 15}, + /* limits */ + {"9223372036.854775807", 9, LLONG_MAX}, + {"-9223372036.854775808", 9, LLONG_MIN}, + /* scale > 19 */ + {"0.0", 20, 0}, + {"-0.0", 20, 0}, + {"0.00000000000000000001", 20, 1}, + {"-0.00000000000000000001", 20, -1}, + {"0.009223372036854775807", 21, LLONG_MAX}, + {"-0.009223372036854775808", 21, LLONG_MIN}, + }; + TEST_OK(kstrtodec64, s64, "%lld", test_dec64_ok); +} + +static void __init test_kstrtodec64_fail(void) +{ + static DEFINE_TEST_FAIL(test_dec64_fail) = { + /* empty / whitespace */ + {"", 3}, + {"\n", 3}, + /* no digits after dot */ + {".", 3}, + {"-.", 3}, + /* no digits at all */ + {"+", 3}, + {"-", 3}, + /* non-digit characters */ + {"abc", 3}, + {"-1.2x", 3}, + /* signed overflow */ + {"9223372036.854775808", 9}, + {"-9223372036.854775809", 9}, + {"99999999999999999999", 1}, + /* signed overflow with scale > 19 */ + {"0.1", 20}, + {"-0.1", 20}, + {"0.09223372036854775808", 20}, + {"-0.09223372036854775809", 20}, + }; + TEST_FAIL(kstrtodec64, s64, "%lld", test_dec64_fail); +} + static int __init test_kstrtox_init(void) { test_kstrtoull_ok(); @@ -735,6 +911,12 @@ static int __init test_kstrtox_init(void) test_kstrtou8_fail(); test_kstrtos8_ok(); test_kstrtos8_fail(); + + test_kstrtoudec64_ok(); + test_kstrtoudec64_fail(); + test_kstrtodec64_ok(); + test_kstrtodec64_fail(); + return -EINVAL; } module_init(test_kstrtox_init); diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 69813c10e6c0..56bd23059b26 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -234,6 +234,43 @@ static void __init test_find_nth_bit(void) } } +static void __init +test_bitmap_find_next_zero_area_off(void) +{ + DECLARE_BITMAP(bmap, 192); + + bitmap_set(bmap, 0, 192); + + bitmap_clear(bmap, 0, 8); + __clear_bit(50, bmap); + bitmap_clear(bmap, 60, 18); + __set_bit(69, bmap); + __clear_bit(80, bmap); + bitmap_clear(bmap, 100, 10); + __clear_bit(120, bmap); + bitmap_clear(bmap, 145, 8); + bitmap_clear(bmap, 160, 32); + + expect_eq_uint(0, + bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 0, 0)); + expect_eq_uint(0, + bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 3, 0)); + expect_eq_uint(163, + bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 3, 1)); + expect_eq_uint(60, + bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 0, 0)); + expect_eq_uint(160, + bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 7, 0)); + expect_eq_uint(60, + bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 7, 4)); + expect_eq_uint(100, + bitmap_find_next_zero_area_off(bmap, 192, 0, 10, 0, 0)); + expect_eq_uint(160, + bitmap_find_next_zero_area_off(bmap, 192, 0, 32, 0, 0)); + expect_eq_uint(1, + !!(bitmap_find_next_zero_area_off(bmap, 192, 0, 33, 0, 0) >= 192)); +} + static void __init test_fill_set(void) { DECLARE_BITMAP(bmap, 1024); @@ -392,6 +429,7 @@ static void __init test_bitmap_sg(void) /* Scatter/gather relationship */ bitmap_zero(bmap_tmp, 100); + bitmap_zero(bmap_res, 100); bitmap_gather(bmap_tmp, bmap_scatter, sg_mask, nbits); bitmap_scatter(bmap_res, bmap_tmp, sg_mask, nbits); expect_eq_bitmap(bmap_scatter, bmap_res, 100); @@ -1559,6 +1597,7 @@ static void __init selftest(void) test_for_each_clear_bitrange_from(); test_for_each_set_clump8(); test_for_each_set_bit_wrap(); + test_bitmap_find_next_zero_area_off(); } KSTM_MODULE_LOADERS(test_bitmap); diff --git a/lib/test_fortify/Makefile b/lib/test_fortify/Makefile index 399cae880e1d..44cd5df41a81 100644 --- a/lib/test_fortify/Makefile +++ b/lib/test_fortify/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y := $(call cc-disable-warning,fortify-source) +ccflags-y += $(call cc-disable-warning,stringop-overread) quiet_cmd_test_fortify = TEST $@ cmd_test_fortify = $(CONFIG_SHELL) $(src)/test_fortify.sh \ diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 9c59d1ceb5b5..6911daa9f854 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -389,6 +389,67 @@ out: return ret; } +static int dmirror_range_fault_unlocked(struct dmirror *dmirror, + struct hmm_range *range, + unsigned long timeout) +{ + int ret; + + while (true) { + ret = hmm_range_fault_unlocked_timeout(range, timeout); + if (ret) + goto out; + + mutex_lock(&dmirror->mutex); + if (mmu_interval_read_retry(range->notifier, + range->notifier_seq)) { + mutex_unlock(&dmirror->mutex); + continue; + } + break; + } + + ret = dmirror_do_fault(dmirror, range); + + mutex_unlock(&dmirror->mutex); +out: + return ret; +} + +static int dmirror_fault_unlocked(struct dmirror *dmirror, + unsigned long start, + unsigned long end, bool write, + unsigned long timeout) +{ + struct mm_struct *mm = dmirror->notifier.mm; + unsigned long addr; + unsigned long pfns[32]; + struct hmm_range range = { + .notifier = &dmirror->notifier, + .hmm_pfns = pfns, + .pfn_flags_mask = 0, + .default_flags = + HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0), + .dev_private_owner = dmirror->mdevice, + }; + int ret = 0; + + if (!mmget_not_zero(mm)) + return -EFAULT; + + for (addr = start; addr < end; addr = range.end) { + range.start = addr; + range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end); + + ret = dmirror_range_fault_unlocked(dmirror, &range, timeout); + if (ret) + break; + } + + mmput(mm); + return ret; +} + static int dmirror_fault(struct dmirror *dmirror, unsigned long start, unsigned long end, bool write) { @@ -407,7 +468,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; @@ -488,6 +549,48 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) return ret; } +static int dmirror_read_unlocked(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd, + unsigned long timeout) +{ + struct dmirror_bounce bounce; + unsigned long start, end; + unsigned long size = cmd->npages << PAGE_SHIFT; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + + while (1) { + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret != -ENOENT) + break; + + start = cmd->addr + (bounce.cpages << PAGE_SHIFT); + ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout); + if (ret) + break; + cmd->faults++; + } + + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, unsigned long end, struct dmirror_bounce *bounce) { @@ -581,7 +684,7 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice, devmem->pagemap.type = MEMORY_DEVICE_PRIVATE; break; case HMM_DMIRROR_MEMORY_DEVICE_COHERENT: - devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ? + devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ? spm_addr_dev0 : spm_addr_dev1; devmem->pagemap.range.end = devmem->pagemap.range.start + @@ -1048,10 +1151,9 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, if (!dpage && !order) return VM_FAULT_OOM; - pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n", - page_to_pfn(spage), page_to_pfn(dpage)); - if (dpage) { + pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n", + page_to_pfn(spage), page_to_pfn(dpage)); lock_page(dpage); *dst |= migrate_pfn(page_to_pfn(dpage)); } @@ -1209,16 +1311,10 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror, if (!mmget_not_zero(mm)) return -EINVAL; - ret = -ENOMEM; src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - if (!src_pfns) - goto free_mem; - dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - if (!dst_pfns) - goto free_mem; ret = 0; mmap_read_lock(mm); @@ -1572,7 +1668,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, dmirror->flags = cmd.npages; ret = 0; break; - + case HMM_DMIRROR_READ_UNLOCKED: + ret = dmirror_read_unlocked(dmirror, &cmd, 0); + break; default: return -EINVAL; } diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index f94c6d457338..ea9b0ec404fb 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -38,6 +38,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd) #define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_READ_UNLOCKED _IOWR('H', 0x09, struct hmm_dmirror_cmd) #define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0) diff --git a/lib/test_sysctl.c b/lib/test_sysctl.c index c02aa9c868f2..909cfcf76dbf 100644 --- a/lib/test_sysctl.c +++ b/lib/test_sysctl.c @@ -7,7 +7,7 @@ /* * This module provides an interface to the proc sysctl interfaces. This - * driver requires CONFIG_PROC_SYSCTL. It will not normally be loaded by the + * driver requires CONFIG_SYSCTL. It will not normally be loaded by the * system unless explicitly requested by name. You can also build this driver * into your kernel. */ diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 4ead57602eac..3cac3b63a752 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -41,8 +41,7 @@ obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o obj-$(CONFIG_MIN_HEAP_KUNIT_TEST) += min_heap_kunit.o CFLAGS_overflow_kunit.o = $(call cc-disable-warning, tautological-constant-out-of-range-compare) obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o -# GCC < 12.1 can miscompile errptr() test when branch profiling is enabled. -CFLAGS_printf_kunit.o += -DDISABLE_BRANCH_PROFILING +obj-$(CONFIG_PRANDOM_KUNIT_TEST) += random32_kunit.o obj-$(CONFIG_PRINTF_KUNIT_TEST) += printf_kunit.o obj-$(CONFIG_RANDSTRUCT_KUNIT_TEST) += randstruct_kunit.o obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o diff --git a/lib/tests/blackhole_dev_kunit.c b/lib/tests/blackhole_dev_kunit.c index 06834ab35f43..fa3e0533038d 100644 --- a/lib/tests/blackhole_dev_kunit.c +++ b/lib/tests/blackhole_dev_kunit.c @@ -46,7 +46,7 @@ static void test_blackholedev(struct kunit *test) uh = (struct udphdr *)skb_push(skb, sizeof(struct udphdr)); skb_set_transport_header(skb, 0); uh->source = uh->dest = htons(UDP_PORT); - uh->len = htons(data_len); + udp_set_len_short(uh, data_len); uh->check = 0; /* (Network) IPv6 */ ip6h = (struct ipv6hdr *)skb_push(skb, sizeof(struct ipv6hdr)); diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c index 1e6fce9cb255..d9690ba1db88 100644 --- a/lib/tests/kunit_iov_iter.c +++ b/lib/tests/kunit_iov_iter.c @@ -283,7 +283,7 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) struct page **spages, **bpages; u8 *scratch, *buffer; size_t bufsize, npages, size, copied; - int i, b, patt; + int i, patt; bufsize = 0x100000; npages = bufsize / PAGE_SIZE; @@ -306,10 +306,9 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); /* Build the expected image in the scratch buffer. */ - b = 0; patt = 0; memset(scratch, 0, bufsize); - for (pr = bvec_test_ranges; pr->from >= 0; pr++, b++) { + for (pr = bvec_test_ranges; pr->from >= 0; pr++) { u8 *p = scratch + pr->page * PAGE_SIZE; for (i = pr->from; i < pr->to; i++) diff --git a/lib/tests/random32_kunit.c b/lib/tests/random32_kunit.c new file mode 100644 index 000000000000..0b4af2b09c01 --- /dev/null +++ b/lib/tests/random32_kunit.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for random32 functions. + */ + +#include <linux/prandom.h> +#include <kunit/test.h> + +/* prandom_warmup() is static in lib/random32.c; exposed for testing only. */ +void prandom_warmup(struct rnd_state *state); + +static const struct prandom_test1 { + u32 seed; + u32 result; +} test1[] = { + { 1U, 3484351685U }, + { 2U, 2623130059U }, + { 3U, 3125133893U }, + { 4U, 984847254U }, +}; + +static const struct prandom_test2 { + u32 seed; + u32 iteration; + u32 result; +} test2[] = { + /* Test cases against taus113 from GSL library. */ + { 931557656U, 959U, 2975593782U }, + { 1339693295U, 876U, 3887776532U }, + { 1545556285U, 961U, 1615538833U }, + { 601730776U, 723U, 1776162651U }, + { 1027516047U, 687U, 511983079U }, + { 416526298U, 700U, 916156552U }, + { 1395522032U, 652U, 2222063676U }, + { 366221443U, 617U, 2992857763U }, + { 1539836965U, 714U, 3783265725U }, + { 556206671U, 994U, 799626459U }, + { 684907218U, 799U, 367789491U }, + { 2121230701U, 931U, 2115467001U }, + { 1668516451U, 644U, 3620590685U }, + { 768046066U, 883U, 2034077390U }, + { 1989159136U, 833U, 1195767305U }, + { 536585145U, 996U, 3577259204U }, + { 1008129373U, 642U, 1478080776U }, + { 1740775604U, 939U, 1264980372U }, + { 1967883163U, 508U, 10734624U }, + { 1923019697U, 730U, 3821419629U }, + { 442079932U, 560U, 3440032343U }, + { 1961302714U, 845U, 841962572U }, + { 2030205964U, 962U, 1325144227U }, + { 1160407529U, 507U, 240940858U }, + { 635482502U, 779U, 4200489746U }, + { 1252788931U, 699U, 867195434U }, + { 1961817131U, 719U, 668237657U }, + { 1071468216U, 983U, 917876630U }, + { 1281848367U, 932U, 1003100039U }, + { 582537119U, 780U, 1127273778U }, + { 1973672777U, 853U, 1071368872U }, + { 1896756996U, 762U, 1127851055U }, + { 847917054U, 500U, 1717499075U }, + { 1240520510U, 951U, 2849576657U }, + { 1685071682U, 567U, 1961810396U }, + { 1516232129U, 557U, 3173877U }, + { 1208118903U, 612U, 1613145022U }, + { 1817269927U, 693U, 4279122573U }, + { 1510091701U, 717U, 638191229U }, + { 365916850U, 807U, 600424314U }, + { 399324359U, 702U, 1803598116U }, + { 1318480274U, 779U, 2074237022U }, + { 697758115U, 840U, 1483639402U }, + { 1696507773U, 840U, 577415447U }, + { 2081979121U, 981U, 3041486449U }, + { 955646687U, 742U, 3846494357U }, + { 1250683506U, 749U, 836419859U }, + { 595003102U, 534U, 366794109U }, + { 47485338U, 558U, 3521120834U }, + { 619433479U, 610U, 3991783875U }, + { 704096520U, 518U, 4139493852U }, + { 1712224984U, 606U, 2393312003U }, + { 1318233152U, 922U, 3880361134U }, + { 855572992U, 761U, 1472974787U }, + { 64721421U, 703U, 683860550U }, + { 678931758U, 840U, 380616043U }, + { 692711973U, 778U, 1382361947U }, + { 677703619U, 530U, 2826914161U }, + { 92393223U, 586U, 1522128471U }, + { 1222592920U, 743U, 3466726667U }, + { 358288986U, 695U, 1091956998U }, + { 1935056945U, 958U, 514864477U }, + { 735675993U, 990U, 1294239989U }, + { 1560089402U, 897U, 2238551287U }, + { 70616361U, 829U, 22483098U }, + { 368234700U, 731U, 2913875084U }, + { 20221190U, 879U, 1564152970U }, + { 539444654U, 682U, 1835141259U }, + { 1314987297U, 840U, 1801114136U }, + { 2019295544U, 645U, 3286438930U }, + { 469023838U, 716U, 1637918202U }, + { 1843754496U, 653U, 2562092152U }, + { 400672036U, 809U, 4264212785U }, + { 404722249U, 965U, 2704116999U }, + { 600702209U, 758U, 584979986U }, + { 519953954U, 667U, 2574436237U }, + { 1658071126U, 694U, 2214569490U }, + { 420480037U, 749U, 3430010866U }, + { 690103647U, 969U, 3700758083U }, + { 1029424799U, 937U, 3787746841U }, + { 2012608669U, 506U, 3362628973U }, + { 1535432887U, 998U, 42610943U }, + { 1330635533U, 857U, 3040806504U }, + { 1223800550U, 539U, 3954229517U }, + { 1322411537U, 680U, 3223250324U }, + { 1877847898U, 945U, 2915147143U }, + { 1646356099U, 874U, 965988280U }, + { 805687536U, 744U, 4032277920U }, + { 1948093210U, 633U, 1346597684U }, + { 392609744U, 783U, 1636083295U }, + { 690241304U, 770U, 1201031298U }, + { 1360302965U, 696U, 1665394461U }, + { 1220090946U, 780U, 1316922812U }, + { 447092251U, 500U, 3438743375U }, + { 1613868791U, 592U, 828546883U }, + { 523430951U, 548U, 2552392304U }, + { 726692899U, 810U, 1656872867U }, + { 1364340021U, 836U, 3710513486U }, + { 1986257729U, 931U, 935013962U }, + { 407983964U, 921U, 728767059U }, +}; + +static void prandom_state_test_seed(struct rnd_state *state, u32 seed) +{ +#define LCG(x) ((x) * 69069U) /* super-duper LCG */ + state->s1 = __seed(LCG(seed), 2U); + state->s2 = __seed(LCG(state->s1), 8U); + state->s3 = __seed(LCG(state->s2), 16U); + state->s4 = __seed(LCG(state->s3), 128U); +} + +static void test_prandom_seed_boundary(struct kunit *test) +{ + int i; + struct rnd_state state; + + for (i = 0; i < ARRAY_SIZE(test1); i++) { + prandom_state_test_seed(&state, test1[i].seed); + prandom_warmup(&state); + KUNIT_EXPECT_EQ(test, test1[i].result, prandom_u32_state(&state)); + } +} + +static void test_prandom_taus113(struct kunit *test) +{ + int i, j; + struct rnd_state state; + + for (i = 0; i < ARRAY_SIZE(test2); i++) { + prandom_state_test_seed(&state, test2[i].seed); + prandom_warmup(&state); + + for (j = 0; j < test2[i].iteration - 1; j++) + prandom_u32_state(&state); + + KUNIT_EXPECT_EQ(test, test2[i].result, prandom_u32_state(&state)); + } +} + +static struct kunit_case prandom_test_cases[] = { + KUNIT_CASE(test_prandom_seed_boundary), + KUNIT_CASE(test_prandom_taus113), + {} +}; + +static struct kunit_suite prandom_test_suite = { + .name = "prandom", + .test_cases = prandom_test_cases, +}; + +kunit_test_suite(prandom_test_suite); + +MODULE_DESCRIPTION("KUnit test for prandom"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/lib/tests/slub_kunit.c b/lib/tests/slub_kunit.c index fa6d31dbca16..e3b63f0338d5 100644 --- a/lib/tests/slub_kunit.c +++ b/lib/tests/slub_kunit.c @@ -8,6 +8,7 @@ #include <linux/rcupdate.h> #include <linux/delay.h> #include <linux/perf_event.h> +#include <linux/kprobes.h> #include "../mm/slab.h" static struct kunit_resource resource; @@ -161,7 +162,10 @@ static void test_kmalloc_redzone_access(struct kunit *test) } struct test_kfree_rcu_struct { - struct rcu_head rcu; + union { + struct rcu_head rcu; + struct kvfree_rcu_head kvrcu; + }; }; static void test_kfree_rcu(struct kunit *test) @@ -292,19 +296,76 @@ static void test_krealloc_redzone_zeroing(struct kunit *test) kmem_cache_destroy(s); } -#ifdef CONFIG_PERF_EVENTS +#if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP)) #define NR_ITERATIONS 1000 #define NR_OBJECTS 1000 -static void *objects[NR_OBJECTS]; +static struct test_kfree_rcu_struct *objects[NR_OBJECTS]; struct test_nolock_context { struct kunit *test; int callback_count; int alloc_ok; int alloc_fail; +#ifdef CONFIG_PERF_EVENTS struct perf_event *event; +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + struct kprobe kprobe; +#endif }; +static void test_kmalloc_and_friends(void) +{ + int i, j; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); + + for (i = 0; i < NR_ITERATIONS; i++) { + for (j = 0; j < NR_OBJECTS; j++) { + gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + + objects[j] = kmalloc_obj(*objects[j], gfp); + if (!objects[j]) { + j--; + while (j >= 0) + kfree(objects[j--]); + return; + } + } + + for (j = 0; j < NR_OBJECTS; j++) { + if (can_use_kfree_rcu && (i & 2)) + kfree_rcu(objects[j], rcu); + else + kfree(objects[j]); + } + } +} + +static void test_nolock(struct test_nolock_context *ctx) +{ + struct test_kfree_rcu_struct *objp; + gfp_t gfp; + bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); + + /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ + gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT; + objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE); + + if (objp) + ctx->alloc_ok++; + else + ctx->alloc_fail++; + + if (can_use_kfree_rcu && (ctx->callback_count & 2)) + kfree_rcu_nolock(objp, kvrcu); + else + kfree_nolock(objp); + + ctx->callback_count++; +} +#endif + +#ifdef CONFIG_PERF_EVENTS static struct perf_event_attr hw_attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES, @@ -315,67 +376,91 @@ static struct perf_event_attr hw_attr = { .sample_freq = 100000, }; -static void overflow_handler_test_kmalloc_kfree_nolock(struct perf_event *event, - struct perf_sample_data *data, - struct pt_regs *regs) +static void overflow_handler_test_nolock(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) { - void *objp; - gfp_t gfp; struct test_nolock_context *ctx = event->overflow_handler_context; - /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ - gfp = (ctx->callback_count % 2) ? 0 : __GFP_ACCOUNT; - objp = kmalloc_nolock(64, gfp, NUMA_NO_NODE); - - if (objp) - ctx->alloc_ok++; - else - ctx->alloc_fail++; - - kfree_nolock(objp); - ctx->callback_count++; + test_nolock(ctx); } -static void test_kmalloc_kfree_nolock(struct kunit *test) +static bool enable_perf_events(struct test_nolock_context *ctx) { - int i, j; - struct test_nolock_context ctx = { .test = test }; struct perf_event *event; - bool alloc_fail = false; event = perf_event_create_kernel_counter(&hw_attr, -1, current, - overflow_handler_test_kmalloc_kfree_nolock, - &ctx); + overflow_handler_test_nolock, + ctx); + if (IS_ERR(event)) - kunit_skip(test, "Failed to create perf event"); - ctx.event = event; - perf_event_enable(ctx.event); - for (i = 0; i < NR_ITERATIONS; i++) { - for (j = 0; j < NR_OBJECTS; j++) { - gfp_t gfp = (i % 2) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; + return false; - objects[j] = kmalloc(64, gfp); - if (!objects[j]) { - j--; - while (j >= 0) - kfree(objects[j--]); - alloc_fail = true; - goto cleanup; - } - } - for (j = 0; j < NR_OBJECTS; j++) - kfree(objects[j]); - } + ctx->event = event; + perf_event_enable(ctx->event); + return true; +} + +static void disable_perf_events(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); + + perf_event_disable(ctx->event); + perf_event_release_kernel(ctx->event); +} + +static void test_kmalloc_nolock_and_friends_perf(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; -cleanup: - perf_event_disable(ctx.event); - perf_event_release_kernel(ctx.event); + if (!enable_perf_events(&ctx)) + kunit_skip(test, "Failed to enable perf event, skipping"); - kunit_info(test, "callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", - ctx.callback_count, ctx.alloc_ok, ctx.alloc_fail); + test_kmalloc_and_friends(); - if (alloc_fail) - kunit_skip(test, "Allocation failed"); + disable_perf_events(&ctx); + KUNIT_EXPECT_EQ(test, 0, slab_errors); +} +#endif + +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) +static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct test_nolock_context *ctx; + + ctx = container_of(p, struct test_nolock_context, kprobe); + test_nolock(ctx); + return 0; +} + +static bool register_slab_kprobes(struct test_nolock_context *ctx) +{ + ctx->kprobe.symbol_name = "slab_attach_kprobe_locked"; + ctx->kprobe.pre_handler = slab_kprobe_pre_handler; + + if (register_kprobe(&ctx->kprobe)) + return false; + return true; +} + +static void unregister_slab_kprobes(struct test_nolock_context *ctx) +{ + kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", + ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); + unregister_kprobe(&ctx->kprobe); +} + +static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test) +{ + struct test_nolock_context ctx = { .test = test }; + + if (!register_slab_kprobes(&ctx)) + kunit_skip(test, "Failed to register kprobe, skipping"); + + test_kmalloc_and_friends(); + + unregister_slab_kprobes(&ctx); KUNIT_EXPECT_EQ(test, 0, slab_errors); } #endif @@ -405,7 +490,10 @@ static struct kunit_case test_cases[] = { KUNIT_CASE(test_leak_destroy), KUNIT_CASE(test_krealloc_redzone_zeroing), #ifdef CONFIG_PERF_EVENTS - KUNIT_CASE_SLOW(test_kmalloc_kfree_nolock), + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf), +#endif +#if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) + KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe), #endif {} }; diff --git a/lib/tests/test_ratelimit.c b/lib/tests/test_ratelimit.c index 33cea5f3d28b..e244f8cd47d7 100644 --- a/lib/tests/test_ratelimit.c +++ b/lib/tests/test_ratelimit.c @@ -68,7 +68,6 @@ static void test_ratelimit_smoke(struct kunit *test) static struct ratelimit_state stressrl = RATELIMIT_STATE_INIT_FLAGS("stressrl", HZ / 10, 3, RATELIMIT_MSG_ON_RELEASE); -static int doneflag; static const int stress_duration = 2 * HZ; struct stress_kthread { @@ -84,9 +83,8 @@ static int test_ratelimit_stress_child(void *arg) struct stress_kthread *sktp = arg; set_user_nice(current, MAX_NICE); - WARN_ON_ONCE(!sktp->tp); - while (!READ_ONCE(doneflag)) { + while (!kthread_should_stop()) { sktp->nattempts++; if (___ratelimit(&stressrl, __func__)) sktp->nunlimited++; @@ -105,26 +103,37 @@ static void test_ratelimit_stress(struct kunit *test) const int n_stress_kthread = cpumask_weight(cpu_online_mask); struct stress_kthread skt = { 0 }; struct stress_kthread *sktp = kzalloc_objs(*sktp, n_stress_kthread); + int n_started = 0; - KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "Memory allocation failure"); + KUNIT_ASSERT_NOT_NULL_MSG(test, sktp, "Memory allocation failure"); for (i = 0; i < n_stress_kthread; i++) { sktp[i].tp = kthread_run(test_ratelimit_stress_child, &sktp[i], "%s/%i", "test_ratelimit_stress_child", i); - KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "kthread creation failure"); + if (IS_ERR(sktp[i].tp)) { + KUNIT_FAIL(test, "kthread_run failed: %ld", PTR_ERR(sktp[i].tp)); + goto out_stop; + } + n_started++; pr_alert("Spawned test_ratelimit_stress_child %d\n", i); } schedule_timeout_idle(stress_duration); - WRITE_ONCE(doneflag, 1); - for (i = 0; i < n_stress_kthread; i++) { + +out_stop: + for (i = 0; i < n_started; i++) { kthread_stop(sktp[i].tp); skt.nattempts += sktp[i].nattempts; skt.nunlimited += sktp[i].nunlimited; skt.nlimited += sktp[i].nlimited; skt.nmissed += sktp[i].nmissed; } - KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts, - "Outcomes not equal to attempts"); - KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, "Misses not equal to limits"); + if (n_started == n_stress_kthread) { + KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts, + "Outcomes not equal to attempts"); + KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, + "Misses not equal to limits"); + } + + kfree(sktp); } static struct kunit_case ratelimit_test_cases[] = { diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c index dfb4f2358cab..1f7dd4eb640a 100644 --- a/lib/ucs2_string.c +++ b/lib/ucs2_string.c @@ -6,18 +6,18 @@ unsigned long ucs2_strnlen(const ucs2_char_t *s, size_t maxlength) { - unsigned long length = 0; + unsigned long length = 0; - while (*s++ != 0 && length < maxlength) - length++; - return length; + while (length < maxlength && *s++ != 0) + length++; + return length; } EXPORT_SYMBOL(ucs2_strnlen); unsigned long ucs2_strlen(const ucs2_char_t *s) { - return ucs2_strnlen(s, ~0UL); + return ucs2_strnlen(s, ~0UL); } EXPORT_SYMBOL(ucs2_strlen); @@ -28,7 +28,7 @@ EXPORT_SYMBOL(ucs2_strlen); unsigned long ucs2_strsize(const ucs2_char_t *data, unsigned long maxlength) { - return ucs2_strnlen(data, maxlength/sizeof(ucs2_char_t)) * sizeof(ucs2_char_t); + return ucs2_strnlen(data, maxlength/sizeof(ucs2_char_t)) * sizeof(ucs2_char_t); } EXPORT_SYMBOL(ucs2_strsize); @@ -87,19 +87,19 @@ EXPORT_SYMBOL(ucs2_strscpy); int ucs2_strncmp(const ucs2_char_t *a, const ucs2_char_t *b, size_t len) { - while (1) { - if (len == 0) - return 0; - if (*a < *b) - return -1; - if (*a > *b) - return 1; - if (*a == 0) /* implies *b == 0 */ - return 0; - a++; - b++; - len--; - } + while (1) { + if (len == 0) + return 0; + if (*a < *b) + return -1; + if (*a > *b) + return 1; + if (*a == 0) /* implies *b == 0 */ + return 0; + a++; + b++; + len--; + } } EXPORT_SYMBOL(ucs2_strncmp); diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index db87ba34ef19..597f5f0f9681 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -1,12 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 -config HAVE_GENERIC_VDSO +config VDSO_DATASTORE bool -if HAVE_GENERIC_VDSO - config GENERIC_GETTIMEOFDAY bool + select VDSO_DATASTORE help This is a generic implementation of gettimeofday vdso. Each architecture that enables this feature has to @@ -21,7 +20,6 @@ config GENERIC_VDSO_OVERFLOW_PROTECT config VDSO_GETRANDOM bool + select VDSO_DATASTORE help Selected by architectures that support vDSO getrandom(). - -endif diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile index 405f743253d7..ac304def42d6 100644 --- a/lib/vdso/Makefile +++ b/lib/vdso/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_HAVE_GENERIC_VDSO) += datastore.o +obj-$(CONFIG_VDSO_DATASTORE) += datastore.o diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 17d37b82ebc6..1426bf4e0c12 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -29,10 +29,11 @@ struct vdso_arch_data *vdso_k_arch_data __ro_after_init = (void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE]; #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */ +static struct page *vdso_data_pages __ro_after_init; + void __init vdso_setup_data_pages(void) { unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE); - struct page *pages; /* * Allocate the data pages dynamically. SPARC does not support mapping @@ -42,24 +43,24 @@ void __init vdso_setup_data_pages(void) * Do not use folios. In time namespaces the pages are mapped in a different order * to userspace, which is not handled by the folio optimizations in finish_fault(). */ - pages = alloc_pages(GFP_KERNEL, order); - if (!pages) + vdso_data_pages = alloc_pages(GFP_KERNEL, order); + if (!vdso_data_pages) panic("Unable to allocate VDSO storage pages"); /* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */ - split_page(pages, order); + split_page(vdso_data_pages, order); /* Move the data already written by other subsystems to the new pages */ - memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE); + memcpy(page_address(vdso_data_pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE); if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) - vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET); + vdso_k_time_data = page_address(vdso_data_pages + VDSO_TIME_PAGE_OFFSET); if (IS_ENABLED(CONFIG_VDSO_GETRANDOM)) - vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET); + vdso_k_rng_data = page_address(vdso_data_pages + VDSO_RNG_PAGE_OFFSET); if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) - vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START); + vdso_k_arch_data = page_address(vdso_data_pages + VDSO_ARCH_PAGES_START); } static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, @@ -67,27 +68,28 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, { struct page *page, *timens_page; + if (unlikely(vmf->flags & FAULT_FLAG_REMOTE)) + return VM_FAULT_SIGBUS; + + page = vdso_data_pages + vmf->pgoff; timens_page = find_timens_vvar_page(vma); switch (vmf->pgoff) { case VDSO_TIME_PAGE_OFFSET: - if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY)) - return VM_FAULT_SIGBUS; - page = virt_to_page(vdso_k_time_data); - if (timens_page) { - /* - * Fault in VVAR page too, since it will be accessed - * to get clock data anyway. - */ - unsigned long addr; - vm_fault_t err; - - addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_page(vma, addr, page); - if (unlikely(err & VM_FAULT_ERROR)) - return err; - page = timens_page; - } + if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) || !timens_page) + break; + /* + * Fault in VVAR page too, since it will be accessed + * to get clock data anyway. + */ + unsigned long addr; + vm_fault_t err; + + addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; + err = vmf_insert_page(vma, addr, page); + if (unlikely(err & VM_FAULT_ERROR)) + return err; + page = timens_page; break; case VDSO_TIMENS_PAGE_OFFSET: /* @@ -98,18 +100,11 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, * See also the comment near timens_setup_vdso_data(). */ if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page) - return VM_FAULT_SIGBUS; - page = virt_to_page(vdso_k_time_data); + break; + page = vdso_data_pages + VDSO_TIME_PAGE_OFFSET; break; case VDSO_RNG_PAGE_OFFSET: - if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM)) - return VM_FAULT_SIGBUS; - page = virt_to_page(vdso_k_rng_data); - break; case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END: - if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA)) - return VM_FAULT_SIGBUS; - page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START; break; default: return VM_FAULT_SIGBUS; @@ -128,7 +123,7 @@ const struct vm_special_mapping vdso_vvar_mapping = { struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr) { return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, - VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP | + VM_READ | VM_MAYREAD | VM_DONTDUMP | VM_MIXEDMAP | VM_SEALED_SYSMAP, &vdso_vvar_mapping); } diff --git a/lib/vdso/getrandom.c b/lib/vdso/getrandom.c index 7e29005aa208..2851afa9154f 100644 --- a/lib/vdso/getrandom.c +++ b/lib/vdso/getrandom.c @@ -8,6 +8,7 @@ #include <vdso/datapage.h> #include <vdso/getrandom.h> #include <vdso/limits.h> +#include <vdso/page.h> #include <vdso/unaligned.h> #include <asm/barrier.h> #include <asm/vdso/getrandom.h> @@ -18,11 +19,6 @@ /* Bring in default accessors */ #include <vdso/vsyscall.h> -#undef PAGE_SIZE -#undef PAGE_MASK -#define PAGE_SIZE (1UL << CONFIG_PAGE_SHIFT) -#define PAGE_MASK (~(PAGE_SIZE - 1)) - #define MEMCPY_AND_ZERO_SRC(type, dst, src, len) do { \ while (len >= sizeof(type)) { \ __put_unaligned_t(type, __get_unaligned_t(type, src), dst); \ diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index e0f289d3d110..f7a591aba59f 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -12,6 +12,8 @@ #include <vdso/time32.h> #include <vdso/time64.h> +#include <uapi/linux/unistd.h> + /* * The generic vDSO implementation requires that gettimeofday.h * provides: @@ -23,6 +25,8 @@ */ #include <asm/vdso/gettimeofday.h> +#include <linux/build_bug.h> + /* Bring in default accessors */ #include <vdso/vsyscall.h> @@ -323,6 +327,8 @@ __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec ts; bool ok; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + ok = __cvdso_clock_gettime_common(vd, clock, &ts); if (unlikely(!ok)) @@ -348,6 +354,12 @@ __cvdso_gettimeofday_data(const struct vdso_time_data *vd, { const struct vdso_clock *vc = vd->clock_data; +#ifndef __NR_gettimeofday + BUILD_BUG(); +#endif + + BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + if (likely(tv != NULL)) { struct __kernel_timespec ts; @@ -382,6 +394,12 @@ __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) const struct vdso_clock *vc = vd->clock_data; __kernel_old_time_t t; +#ifndef __NR_time + BUILD_BUG(); +#endif + + BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + if (vdso_is_timens_clock(vc)) { vd = vdso_timens_data(vd); vc = vd->clock_data; @@ -471,6 +489,8 @@ __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t cloc struct __kernel_timespec ts; bool ok; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); + ok = __cvdso_clock_getres_common(vd, clock, &ts); if (unlikely(!ok)) diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 2bc6ef483576..1d95ab9314bf 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -86,7 +86,7 @@ static unsigned long long simple_strntoull(const char *startp, char **endp, unsi cp = _parse_integer_fixup_radix(startp, &base); prefix_chars = cp - startp; if (prefix_chars < max_chars) { - rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars); + rv = _parse_integer(cp, base, &result, max_chars - prefix_chars); /* FIXME */ cp += (rv & ~KSTRTOX_OVERFLOW); } else { @@ -2360,6 +2360,9 @@ void __init hash_pointers_finalize(bool slub_debug) static int __init hash_pointers_mode_parse(char *str) { + /* Avoid stale no_hash_pointers state when hash_pointers overrides it */ + no_hash_pointers = false; + if (!str) { pr_warn("Hash pointers mode empty; falling back to auto.\n"); hash_pointers_mode = HASH_PTR_AUTO; @@ -2369,6 +2372,7 @@ static int __init hash_pointers_mode_parse(char *str) } else if (strcmp(str, "never") == 0) { pr_info("Hash pointers mode set to never.\n"); hash_pointers_mode = HASH_PTR_NEVER; + no_hash_pointers = true; } else if (strcmp(str, "always") == 0) { pr_info("Hash pointers mode set to always.\n"); hash_pointers_mode = HASH_PTR_ALWAYS; diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540c..bfe7bef80f34 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1053,6 +1053,9 @@ void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, if (xas->xa_shift + XA_CHUNK_SHIFT > order) return; + if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) + gfp |= __GFP_ACCOUNT; + do { struct xa_node *node; diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index cc49a300a5b2..88922323f96e 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -466,7 +466,7 @@ static void bcj_flush(struct xz_dec_bcj *s, struct xz_buf *b) { size_t copy_size; - copy_size = min_t(size_t, s->temp.filtered, b->out_size - b->out_pos); + copy_size = min(s->temp.filtered, b->out_size - b->out_pos); memcpy(b->out + b->out_pos, s->temp.buf, copy_size); b->out_pos += copy_size; diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index 4b783ac94e71..46c2df6ad6f5 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -135,14 +135,16 @@ struct lzma_dec { uint32_t rep2; uint32_t rep3; - /* Types of the most recently seen LZMA symbols */ - enum lzma_state state; - /* * Length of a match. This is updated so that dict_repeat can - * be called again to finish repeating the whole match. + * be called again to finish repeating the whole match. This is + * size_t because a pointer to this is passed to dict_repeat, + * and there it's nicer to have size_t instead of uint32_t. */ - uint32_t len; + size_t len; + + /* Types of the most recently seen LZMA symbols */ + enum lzma_state state; /* * LZMA properties or related bit masks (number of literal @@ -228,13 +230,13 @@ struct lzma2_dec { enum lzma2_seq next_sequence; /* Uncompressed size of LZMA chunk (2 MiB at maximum) */ - uint32_t uncompressed; + size_t uncompressed; /* * Compressed size of LZMA chunk or compressed/uncompressed * size of uncompressed chunk (64 KiB at maximum) */ - uint32_t compressed; + size_t compressed; /* * True if dictionary reset is needed. This is false before @@ -269,11 +271,11 @@ struct xz_dec_lzma2 { struct lzma_dec lzma; /* - * Temporary buffer which holds small number of input bytes between + * Temporary buffer which holds a small number of input bytes between * decoder calls. See lzma2_lzma() for details. */ struct { - uint32_t size; + size_t size; uint8_t buf[3 * LZMA_IN_REQUIRED]; } temp; }; @@ -320,7 +322,7 @@ static inline bool dict_has_space(const struct dictionary *dict) * still empty. This special case is needed for single-call decoding to * avoid writing a '\0' to the end of the destination buffer. */ -static inline uint32_t dict_get(const struct dictionary *dict, uint32_t dist) +static inline uint32_t dict_get(const struct dictionary *dict, size_t dist) { size_t offset = dict->pos - dist - 1; @@ -346,15 +348,15 @@ static inline void dict_put(struct dictionary *dict, uint8_t byte) * invalid, false is returned. On success, true is returned and *len is * updated to indicate how many bytes were left to be repeated. */ -static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist) +static bool dict_repeat(struct dictionary *dict, size_t *len, size_t dist) { size_t back; - uint32_t left; + size_t left; if (dist >= dict->full || dist >= dict->size) return false; - left = min_t(size_t, dict->limit - dict->pos, *len); + left = min(dict->limit - dict->pos, *len); *len -= left; back = dict->pos - dist - 1; @@ -375,7 +377,7 @@ static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist) /* Copy uncompressed data as is from input to dictionary and output buffers. */ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, - uint32_t *left) + size_t *left) { size_t copy_size; @@ -433,7 +435,7 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, * enough space in b->out. This is guaranteed because caller uses dict_limit() * before decoding data into the dictionary. */ -static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b) +static size_t dict_flush(struct dictionary *dict, struct xz_buf *b) { size_t copy_size = dict->pos - dict->start; @@ -755,8 +757,8 @@ static bool lzma_main(struct xz_dec_lzma2 *s) uint32_t pos_state; /* - * If the dictionary was reached during the previous call, try to - * finish the possibly pending repeat in the dictionary. + * If the dictionary write limit was reached during the previous call, + * try to finish the possibly pending repeat in the dictionary. */ if (dict_has_space(&s->dict) && s->lzma.len > 0) dict_repeat(&s->dict, &s->lzma.len, s->lzma.rep0); @@ -878,7 +880,7 @@ static bool lzma_props(struct xz_dec_lzma2 *s, uint8_t props) static bool lzma2_lzma(struct xz_dec_lzma2 *s, struct xz_buf *b) { size_t in_avail; - uint32_t tmp; + size_t tmp; in_avail = b->in_size - b->in_pos; if (s->temp.size > 0 || s->lzma2.compressed == 0) { @@ -976,7 +978,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) * an uncompressed chunk * 0x02 Uncompressed chunk (no dictionary reset) * - * Highest three bits (s->control & 0xE0): + * Highest three bits (tmp & 0xE0): * 0xE0 Dictionary reset, new properties and state * reset, followed by LZMA compressed chunk * 0xC0 New properties and state reset, followed @@ -988,7 +990,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) * 0x80 LZMA chunk (no dictionary or state reset) * * For LZMA compressed chunks, the lowest five bits - * (s->control & 1F) are the highest bits of the + * (tmp & 1F) are the highest bits of the * uncompressed size (bits 16-20). * * A new LZMA2 stream must begin with a dictionary @@ -1046,25 +1048,23 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) case SEQ_UNCOMPRESSED_1: s->lzma2.uncompressed - += (uint32_t)b->in[b->in_pos++] << 8; + += (size_t)b->in[b->in_pos++] << 8; s->lzma2.sequence = SEQ_UNCOMPRESSED_2; break; case SEQ_UNCOMPRESSED_2: s->lzma2.uncompressed - += (uint32_t)b->in[b->in_pos++] + 1; + += (size_t)b->in[b->in_pos++] + 1; s->lzma2.sequence = SEQ_COMPRESSED_0; break; case SEQ_COMPRESSED_0: - s->lzma2.compressed - = (uint32_t)b->in[b->in_pos++] << 8; + s->lzma2.compressed = (size_t)b->in[b->in_pos++] << 8; s->lzma2.sequence = SEQ_COMPRESSED_1; break; case SEQ_COMPRESSED_1: - s->lzma2.compressed - += (uint32_t)b->in[b->in_pos++] + 1; + s->lzma2.compressed += (size_t)b->in[b->in_pos++] + 1; s->lzma2.sequence = s->lzma2.next_sequence; break; @@ -1091,16 +1091,15 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b) case SEQ_LZMA_RUN: /* * Set dictionary limit to indicate how much we want - * to be encoded at maximum. Decode new data into the + * to be decoded at maximum. Decode new data into the * dictionary. Flush the new data from dictionary to * b->out. Check if we finished decoding this chunk. * In case the dictionary got full but we didn't fill * the output buffer yet, we may run this loop * multiple times without changing s->lzma2.sequence. */ - dict_limit(&s->dict, min_t(size_t, - b->out_size - b->out_pos, - s->lzma2.uncompressed)); + dict_limit(&s->dict, min(b->out_size - b->out_pos, + s->lzma2.uncompressed)); if (!lzma2_lzma(s, b)) return XZ_DATA_ERROR; @@ -1260,8 +1259,8 @@ enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s_ptr, s->dict.end = b->out_size - b->out_pos; while (true) { - dict_limit(&s->dict, min_t(size_t, b->out_size - b->out_pos, - s->lzma2.uncompressed)); + dict_limit(&s->dict, min(b->out_size - b->out_pos, + s->lzma2.uncompressed)); if (!lzma2_lzma(s, b)) return XZ_DATA_ERROR; diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index 59bfd54ffee7..0bed6daefac2 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -155,8 +155,8 @@ static const uint8_t check_sizes[16] = { */ static bool fill_temp(struct xz_dec *s, struct xz_buf *b) { - size_t copy_size = min_t(size_t, - b->in_size - b->in_pos, s->temp.size - s->temp.pos); + size_t copy_size = min(b->in_size - b->in_pos, + s->temp.size - s->temp.pos); memcpy(s->temp.buf + s->temp.pos, b->in + b->in_pos, copy_size); b->in_pos += copy_size; diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h index d2632b7dfb9c..a612ce4fd450 100644 --- a/lib/xz/xz_lzma2.h +++ b/lib/xz/xz_lzma2.h @@ -143,7 +143,7 @@ static inline bool lzma_state_is_literal(enum lzma_state state) * Get the index of the appropriate probability array for decoding * the distance slot. */ -static inline uint32_t lzma_get_dist_state(uint32_t len) +static inline size_t lzma_get_dist_state(size_t len) { return len < DIST_STATES + MATCH_LEN_MIN ? len - MATCH_LEN_MIN : DIST_STATES - 1; |