jemalloc: Import 5.3.0 54eaed1d8b56b1aa528be3bdd1877e59c56fa90cvendor/jemalloc/5.3.0vendor/jemalloc

Import jemalloc 5.3.0.

This import changes how manage the jemalloc vendor branch (which was
just started anyway). Starting with 5.3.0, we import a clean tree from
the upstream github, removing all the old files that are no longer
upstream, or that we've kept around for some reason. We do this because
we merge from this raw version of jemalloc into the FreeBSD
contrib/jemalloc, then we run autogen stuff, generate all the generated
.h files with gmake, then finally remove much of the generated files in
contrib/jemalloc using an update script.

Sponsored by:		Netflix

1 files changed, 122 insertions, 9 deletions

diff --git a/src/jemalloc_cpp.cpp b/src/jemalloc_cpp.cpp
index da0441a7c978..451655f1b5a5 100644
--- a/src/jemalloc_cpp.cpp
+++ b/src/jemalloc_cpp.cpp
@@ -39,9 +39,29 @@ void	operator delete(void *ptr, std::size_t size) noexcept;
 void	operator delete[](void *ptr, std::size_t size) noexcept;
 #endif
 
+#if __cpp_aligned_new >= 201606
+/* C++17's over-aligned operators. */
+void	*operator new(std::size_t size, std::align_val_t);
+void	*operator new(std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
+void	*operator new[](std::size_t size, std::align_val_t);
+void	*operator new[](std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept;
+void	operator delete(void* ptr, std::align_val_t) noexcept;
+void	operator delete(void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
+void	operator delete(void* ptr, std::size_t size, std::align_val_t al) noexcept;
+void	operator delete[](void* ptr, std::align_val_t) noexcept;
+void	operator delete[](void* ptr, std::align_val_t, const std::nothrow_t &) noexcept;
+void	operator delete[](void* ptr, std::size_t size, std::align_val_t al) noexcept;
+#endif
+
 JEMALLOC_NOINLINE
 static void *
 handleOOM(std::size_t size, bool nothrow) {
+	if (opt_experimental_infallible_new) {
+		safety_check_fail("<jemalloc>: Allocation failed and "
+		    "opt.experimental_infallible_new is true. Aborting.\n");
+		return nullptr;
+	}
+
 	void *ptr = nullptr;
 
 	while (ptr == nullptr) {
@@ -72,14 +92,21 @@ handleOOM(std::size_t size, bool nothrow) {
 }
 
 template <bool IsNoExcept>
+JEMALLOC_NOINLINE
+static void *
+fallback_impl(std::size_t size) noexcept(IsNoExcept) {
+	void *ptr = malloc_default(size);
+	if (likely(ptr != nullptr)) {
+		return ptr;
+	}
+	return handleOOM(size, IsNoExcept);
+}
+
+template <bool IsNoExcept>
 JEMALLOC_ALWAYS_INLINE
 void *
 newImpl(std::size_t size) noexcept(IsNoExcept) {
-	void *ptr = je_malloc(size);
-	if (likely(ptr != nullptr))
-		return ptr;
-
-	return handleOOM(size, IsNoExcept);
+	return imalloc_fastpath(size, &fallback_impl<IsNoExcept>);
 }
 
 void *
@@ -102,6 +129,42 @@ operator new[](std::size_t size, const std::nothrow_t &) noexcept {
 	return newImpl<true>(size);
 }
 
+#if __cpp_aligned_new >= 201606
+
+template <bool IsNoExcept>
+JEMALLOC_ALWAYS_INLINE
+void *
+alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept(IsNoExcept) {
+	void *ptr = je_aligned_alloc(static_cast<std::size_t>(alignment), size);
+	if (likely(ptr != nullptr)) {
+		return ptr;
+	}
+
+	return handleOOM(size, IsNoExcept);
+}
+
+void *
+operator new(std::size_t size, std::align_val_t alignment) {
+	return alignedNewImpl<false>(size, alignment);
+}
+
+void *
+operator new[](std::size_t size, std::align_val_t alignment) {
+	return alignedNewImpl<false>(size, alignment);
+}
+
+void *
+operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
+	return alignedNewImpl<true>(size, alignment);
+}
+
+void *
+operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept {
+	return alignedNewImpl<true>(size, alignment);
+}
+
+#endif  // __cpp_aligned_new
+
 void
 operator delete(void *ptr) noexcept {
 	je_free(ptr);
@@ -123,19 +186,69 @@ void operator delete[](void *ptr, const std::nothrow_t &) noexcept {
 
 #if __cpp_sized_deallocation >= 201309
 
+JEMALLOC_ALWAYS_INLINE
 void
-operator delete(void *ptr, std::size_t size) noexcept {
+sizedDeleteImpl(void* ptr, std::size_t size) noexcept {
 	if (unlikely(ptr == nullptr)) {
 		return;
 	}
 	je_sdallocx_noflags(ptr, size);
 }
 
-void operator delete[](void *ptr, std::size_t size) noexcept {
+void
+operator delete(void *ptr, std::size_t size) noexcept {
+	sizedDeleteImpl(ptr, size);
+}
+
+void
+operator delete[](void *ptr, std::size_t size) noexcept {
+	sizedDeleteImpl(ptr, size);
+}
+
+#endif  // __cpp_sized_deallocation
+
+#if __cpp_aligned_new >= 201606
+
+JEMALLOC_ALWAYS_INLINE
+void
+alignedSizedDeleteImpl(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
+	if (config_debug) {
+		assert(((size_t)alignment & ((size_t)alignment - 1)) == 0);
+	}
 	if (unlikely(ptr == nullptr)) {
 		return;
 	}
-	je_sdallocx_noflags(ptr, size);
+	je_sdallocx(ptr, size, MALLOCX_ALIGN(alignment));
 }
 
-#endif  // __cpp_sized_deallocation
+void
+operator delete(void* ptr, std::align_val_t) noexcept {
+	je_free(ptr);
+}
+
+void
+operator delete[](void* ptr, std::align_val_t) noexcept {
+	je_free(ptr);
+}
+
+void
+operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
+	je_free(ptr);
+}
+
+void
+operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept {
+	je_free(ptr);
+}
+
+void
+operator delete(void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
+	alignedSizedDeleteImpl(ptr, size, alignment);
+}
+
+void
+operator delete[](void* ptr, std::size_t size, std::align_val_t alignment) noexcept {
+	alignedSizedDeleteImpl(ptr, size, alignment);
+}
+
+#endif  // __cpp_aligned_new