/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * CRC64 using ARM PMULL instructions
 */

#include <asm/simd.h>

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);

u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);

#define crc64_be_arch crc64_be_generic

static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
{
	if (len >= 128 && static_branch_likely(&have_pmull) &&
	    likely(may_use_simd())) {
		do {
			size_t chunk = min_t(size_t, len & ~15, SZ_4K);

			scoped_ksimd()
				crc = crc64_nvme_neon(crc, p, chunk);

			p += chunk;
			len -= chunk;
		} while (len >= 128);
	}
	return crc64_nvme_generic(crc, p, len);
}

#define crc64_mod_init_arch crc64_mod_init_arch
static void crc64_mod_init_arch(void)
{
	if (elf_hwcap2 & HWCAP2_PMULL)
		static_branch_enable(&have_pmull);
}