jemalloc: Import 5.3.0 54eaed1d8b56b1aa528be3bdd1877e59c56fa90cvendor/jemalloc/5.3.0vendor/jemalloc

Import jemalloc 5.3.0.

This import changes how manage the jemalloc vendor branch (which was
just started anyway). Starting with 5.3.0, we import a clean tree from
the upstream github, removing all the old files that are no longer
upstream, or that we've kept around for some reason. We do this because
we merge from this raw version of jemalloc into the FreeBSD
contrib/jemalloc, then we run autogen stuff, generate all the generated
.h files with gmake, then finally remove much of the generated files in
contrib/jemalloc using an update script.

Sponsored by:		Netflix

1 files changed, 104 insertions, 89 deletions

diff --git a/src/base.c b/src/base.c
index f3c61661a20a..7f4d67564b74 100644
--- a/src/base.c
+++ b/src/base.c
@@ -1,4 +1,3 @@
-#define JEMALLOC_BASE_C_
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/jemalloc_internal_includes.h"
 
@@ -7,6 +6,15 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sz.h"
 
+/*
+ * In auto mode, arenas switch to huge pages for the base allocator on the
+ * second base block.  a0 switches to thp on the 5th block (after 20 megabytes
+ * of metadata), since more metadata (e.g. rtree nodes) come from a0's base.
+ */
+
+#define BASE_AUTO_THP_THRESHOLD    2
+#define BASE_AUTO_THP_THRESHOLD_A0 5
+
 /******************************************************************************/
 /* Data. */
 
@@ -29,7 +37,7 @@ metadata_thp_madvise(void) {
 }
 
 static void *
-base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
+base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
 	void *addr;
 	bool zero = true;
 	bool commit = true;
@@ -37,22 +45,21 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
 	/* Use huge page sizes and alignment regardless of opt_metadata_thp. */
 	assert(size == HUGEPAGE_CEILING(size));
 	size_t alignment = HUGEPAGE;
-	if (extent_hooks == &extent_hooks_default) {
+	if (ehooks_are_default(ehooks)) {
 		addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
+		if (have_madvise_huge && addr) {
+			pages_set_thp_state(addr, size);
+		}
 	} else {
-		/* No arena context as we are creating new arenas. */
-		tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
-		pre_reentrancy(tsd, NULL);
-		addr = extent_hooks->alloc(extent_hooks, NULL, size, alignment,
-		    &zero, &commit, ind);
-		post_reentrancy(tsd);
+		addr = ehooks_alloc(tsdn, ehooks, NULL, size, alignment, &zero,
+		    &commit);
 	}
 
 	return addr;
 }
 
 static void
-base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
+base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr,
     size_t size) {
 	/*
 	 * Cascade through dalloc, decommit, purge_forced, and purge_lazy,
@@ -64,7 +71,7 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
 	 * may in fact want the end state of all associated virtual memory to be
 	 * in some consistent-but-allocated state.
 	 */
-	if (extent_hooks == &extent_hooks_default) {
+	if (ehooks_are_default(ehooks)) {
 		if (!extent_dalloc_mmap(addr, size)) {
 			goto label_done;
 		}
@@ -80,31 +87,19 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
 		/* Nothing worked.  This should never happen. */
 		not_reached();
 	} else {
-		tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
-		pre_reentrancy(tsd, NULL);
-		if (extent_hooks->dalloc != NULL &&
-		    !extent_hooks->dalloc(extent_hooks, addr, size, true,
-		    ind)) {
-			goto label_post_reentrancy;
+		if (!ehooks_dalloc(tsdn, ehooks, addr, size, true)) {
+			goto label_done;
 		}
-		if (extent_hooks->decommit != NULL &&
-		    !extent_hooks->decommit(extent_hooks, addr, size, 0, size,
-		    ind)) {
-			goto label_post_reentrancy;
+		if (!ehooks_decommit(tsdn, ehooks, addr, size, 0, size)) {
+			goto label_done;
 		}
-		if (extent_hooks->purge_forced != NULL &&
-		    !extent_hooks->purge_forced(extent_hooks, addr, size, 0,
-		    size, ind)) {
-			goto label_post_reentrancy;
+		if (!ehooks_purge_forced(tsdn, ehooks, addr, size, 0, size)) {
+			goto label_done;
 		}
-		if (extent_hooks->purge_lazy != NULL &&
-		    !extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
-		    ind)) {
-			goto label_post_reentrancy;
+		if (!ehooks_purge_lazy(tsdn, ehooks, addr, size, 0, size)) {
+			goto label_done;
 		}
 		/* Nothing worked.  That's the application's problem. */
-	label_post_reentrancy:
-		post_reentrancy(tsd);
 	}
 label_done:
 	if (metadata_thp_madvise()) {
@@ -116,14 +111,14 @@ label_done:
 }
 
 static void
-base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr,
+base_edata_init(size_t *extent_sn_next, edata_t *edata, void *addr,
     size_t size) {
 	size_t sn;
 
 	sn = *extent_sn_next;
 	(*extent_sn_next)++;
 
-	extent_binit(extent, addr, size, sn);
+	edata_binit(edata, addr, size, sn);
 }
 
 static size_t
@@ -169,7 +164,7 @@ base_auto_thp_switch(tsdn_t *tsdn, base_t *base) {
 		pages_huge(block, block->size);
 		if (config_stats) {
 			base->n_thp += HUGEPAGE_CEILING(block->size -
-			    extent_bsize_get(&block->extent)) >> LG_HUGEPAGE;
+			    edata_bsize_get(&block->edata)) >> LG_HUGEPAGE;
 		}
 		block = block->next;
 		assert(block == NULL || (base_ind_get(base) == 0));
@@ -177,34 +172,34 @@ base_auto_thp_switch(tsdn_t *tsdn, base_t *base) {
 }
 
 static void *
-base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size,
+base_extent_bump_alloc_helper(edata_t *edata, size_t *gap_size, size_t size,
     size_t alignment) {
 	void *ret;
 
 	assert(alignment == ALIGNMENT_CEILING(alignment, QUANTUM));
 	assert(size == ALIGNMENT_CEILING(size, alignment));
 
-	*gap_size = ALIGNMENT_CEILING((uintptr_t)extent_addr_get(extent),
-	    alignment) - (uintptr_t)extent_addr_get(extent);
-	ret = (void *)((uintptr_t)extent_addr_get(extent) + *gap_size);
-	assert(extent_bsize_get(extent) >= *gap_size + size);
-	extent_binit(extent, (void *)((uintptr_t)extent_addr_get(extent) +
-	    *gap_size + size), extent_bsize_get(extent) - *gap_size - size,
-	    extent_sn_get(extent));
+	*gap_size = ALIGNMENT_CEILING((uintptr_t)edata_addr_get(edata),
+	    alignment) - (uintptr_t)edata_addr_get(edata);
+	ret = (void *)((uintptr_t)edata_addr_get(edata) + *gap_size);
+	assert(edata_bsize_get(edata) >= *gap_size + size);
+	edata_binit(edata, (void *)((uintptr_t)edata_addr_get(edata) +
+	    *gap_size + size), edata_bsize_get(edata) - *gap_size - size,
+	    edata_sn_get(edata));
 	return ret;
 }
 
 static void
-base_extent_bump_alloc_post(base_t *base, extent_t *extent, size_t gap_size,
+base_extent_bump_alloc_post(base_t *base, edata_t *edata, size_t gap_size,
     void *addr, size_t size) {
-	if (extent_bsize_get(extent) > 0) {
+	if (edata_bsize_get(edata) > 0) {
 		/*
 		 * Compute the index for the largest size class that does not
 		 * exceed extent's size.
 		 */
 		szind_t index_floor =
-		    sz_size2index(extent_bsize_get(extent) + 1) - 1;
-		extent_heap_insert(&base->avail[index_floor], extent);
+		    sz_size2index(edata_bsize_get(edata) + 1) - 1;
+		edata_heap_insert(&base->avail[index_floor], edata);
 	}
 
 	if (config_stats) {
@@ -229,13 +224,13 @@ base_extent_bump_alloc_post(base_t *base, extent_t *extent, size_t gap_size,
 }
 
 static void *
-base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size,
+base_extent_bump_alloc(base_t *base, edata_t *edata, size_t size,
     size_t alignment) {
 	void *ret;
 	size_t gap_size;
 
-	ret = base_extent_bump_alloc_helper(extent, &gap_size, size, alignment);
-	base_extent_bump_alloc_post(base, extent, gap_size, ret, size);
+	ret = base_extent_bump_alloc_helper(edata, &gap_size, size, alignment);
+	base_extent_bump_alloc_post(base, edata, gap_size, ret, size);
 	return ret;
 }
 
@@ -245,8 +240,8 @@ base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size,
  * On success a pointer to the initialized base_block_t header is returned.
  */
 static base_block_t *
-base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
-    unsigned ind, pszind_t *pind_last, size_t *extent_sn_next, size_t size,
+base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
+    pszind_t *pind_last, size_t *extent_sn_next, size_t size,
     size_t alignment) {
 	alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
 	size_t usize = ALIGNMENT_CEILING(size, alignment);
@@ -267,7 +262,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
 	size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
 	size_t block_size = (min_block_size > next_block_size) ? min_block_size
 	    : next_block_size;
-	base_block_t *block = (base_block_t *)base_map(tsdn, extent_hooks, ind,
+	base_block_t *block = (base_block_t *)base_map(tsdn, ehooks, ind,
 	    block_size);
 	if (block == NULL) {
 		return NULL;
@@ -295,7 +290,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
 	block->size = block_size;
 	block->next = NULL;
 	assert(block_size >= header_size);
-	base_extent_init(extent_sn_next, &block->extent,
+	base_edata_init(extent_sn_next, &block->edata,
 	    (void *)((uintptr_t)block + header_size), block_size - header_size);
 	return block;
 }
@@ -304,17 +299,17 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
  * Allocate an extent that is at least as large as specified size, with
  * specified alignment.
  */
-static extent_t *
+static edata_t *
 base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
 	malloc_mutex_assert_owner(tsdn, &base->mtx);
 
-	extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
+	ehooks_t *ehooks = base_ehooks_get_for_metadata(base);
 	/*
 	 * Drop mutex during base_block_alloc(), because an extent hook will be
 	 * called.
 	 */
 	malloc_mutex_unlock(tsdn, &base->mtx);
-	base_block_t *block = base_block_alloc(tsdn, base, extent_hooks,
+	base_block_t *block = base_block_alloc(tsdn, base, ehooks,
 	    base_ind_get(base), &base->pind_last, &base->extent_sn_next, size,
 	    alignment);
 	malloc_mutex_lock(tsdn, &base->mtx);
@@ -338,7 +333,7 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
 		assert(base->resident <= base->mapped);
 		assert(base->n_thp << LG_HUGEPAGE <= base->mapped);
 	}
-	return &block->extent;
+	return &block->edata;
 }
 
 base_t *
@@ -347,10 +342,22 @@ b0get(void) {
 }
 
 base_t *
-base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
+base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
+    bool metadata_use_hooks) {
 	pszind_t pind_last = 0;
 	size_t extent_sn_next = 0;
-	base_block_t *block = base_block_alloc(tsdn, NULL, extent_hooks, ind,
+
+	/*
+	 * The base will contain the ehooks eventually, but it itself is
+	 * allocated using them.  So we use some stack ehooks to bootstrap its
+	 * memory, and then initialize the ehooks within the base_t.
+	 */
+	ehooks_t fake_ehooks;
+	ehooks_init(&fake_ehooks, metadata_use_hooks ?
+	    (extent_hooks_t *)extent_hooks :
+	    (extent_hooks_t *)&ehooks_default_extent_hooks, ind);
+
+	base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind,
 	    &pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
 	if (block == NULL) {
 		return NULL;
@@ -359,13 +366,15 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 	size_t gap_size;
 	size_t base_alignment = CACHELINE;
 	size_t base_size = ALIGNMENT_CEILING(sizeof(base_t), base_alignment);
-	base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->extent,
+	base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->edata,
 	    &gap_size, base_size, base_alignment);
-	base->ind = ind;
-	atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELAXED);
+	ehooks_init(&base->ehooks, (extent_hooks_t *)extent_hooks, ind);
+	ehooks_init(&base->ehooks_base, metadata_use_hooks ?
+	    (extent_hooks_t *)extent_hooks :
+	    (extent_hooks_t *)&ehooks_default_extent_hooks, ind);
 	if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE,
 	    malloc_mutex_rank_exclusive)) {
-		base_unmap(tsdn, extent_hooks, ind, block, block->size);
+		base_unmap(tsdn, &fake_ehooks, ind, block, block->size);
 		return NULL;
 	}
 	base->pind_last = pind_last;
@@ -373,7 +382,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 	base->blocks = block;
 	base->auto_thp_switched = false;
 	for (szind_t i = 0; i < SC_NSIZES; i++) {
-		extent_heap_new(&base->avail[i]);
+		edata_heap_new(&base->avail[i]);
 	}
 	if (config_stats) {
 		base->allocated = sizeof(base_block_t);
@@ -386,7 +395,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 		assert(base->resident <= base->mapped);
 		assert(base->n_thp << LG_HUGEPAGE <= base->mapped);
 	}
-	base_extent_bump_alloc_post(base, &block->extent, gap_size, base,
+	base_extent_bump_alloc_post(base, &block->edata, gap_size, base,
 	    base_size);
 
 	return base;
@@ -394,26 +403,31 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 
 void
 base_delete(tsdn_t *tsdn, base_t *base) {
-	extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
+	ehooks_t *ehooks = base_ehooks_get_for_metadata(base);
 	base_block_t *next = base->blocks;
 	do {
 		base_block_t *block = next;
 		next = block->next;
-		base_unmap(tsdn, extent_hooks, base_ind_get(base), block,
+		base_unmap(tsdn, ehooks, base_ind_get(base), block,
 		    block->size);
 	} while (next != NULL);
 }
 
-extent_hooks_t *
-base_extent_hooks_get(base_t *base) {
-	return (extent_hooks_t *)atomic_load_p(&base->extent_hooks,
-	    ATOMIC_ACQUIRE);
+ehooks_t *
+base_ehooks_get(base_t *base) {
+	return &base->ehooks;
+}
+
+ehooks_t *
+base_ehooks_get_for_metadata(base_t *base) {
+	return &base->ehooks_base;
 }
 
 extent_hooks_t *
 base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) {
-	extent_hooks_t *old_extent_hooks = base_extent_hooks_get(base);
-	atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELEASE);
+	extent_hooks_t *old_extent_hooks =
+	    ehooks_get_extent_hooks_ptr(&base->ehooks);
+	ehooks_init(&base->ehooks, extent_hooks, ehooks_ind_get(&base->ehooks));
 	return old_extent_hooks;
 }
 
@@ -424,28 +438,28 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment,
 	size_t usize = ALIGNMENT_CEILING(size, alignment);
 	size_t asize = usize + alignment - QUANTUM;
 
-	extent_t *extent = NULL;
+	edata_t *edata = NULL;
 	malloc_mutex_lock(tsdn, &base->mtx);
 	for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) {
-		extent = extent_heap_remove_first(&base->avail[i]);
-		if (extent != NULL) {
+		edata = edata_heap_remove_first(&base->avail[i]);
+		if (edata != NULL) {
 			/* Use existing space. */
 			break;
 		}
 	}
-	if (extent == NULL) {
+	if (edata == NULL) {
 		/* Try to allocate more space. */
-		extent = base_extent_alloc(tsdn, base, usize, alignment);
+		edata = base_extent_alloc(tsdn, base, usize, alignment);
 	}
 	void *ret;
-	if (extent == NULL) {
+	if (edata == NULL) {
 		ret = NULL;
 		goto label_return;
 	}
 
-	ret = base_extent_bump_alloc(base, extent, usize, alignment);
+	ret = base_extent_bump_alloc(base, edata, usize, alignment);
 	if (esn != NULL) {
-		*esn = extent_sn_get(extent);
+		*esn = (size_t)edata_sn_get(edata);
 	}
 label_return:
 	malloc_mutex_unlock(tsdn, &base->mtx);
@@ -465,16 +479,16 @@ base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
 	return base_alloc_impl(tsdn, base, size, alignment, NULL);
 }
 
-extent_t *
-base_alloc_extent(tsdn_t *tsdn, base_t *base) {
+edata_t *
+base_alloc_edata(tsdn_t *tsdn, base_t *base) {
 	size_t esn;
-	extent_t *extent = base_alloc_impl(tsdn, base, sizeof(extent_t),
-	    CACHELINE, &esn);
-	if (extent == NULL) {
+	edata_t *edata = base_alloc_impl(tsdn, base, sizeof(edata_t),
+	    EDATA_ALIGNMENT, &esn);
+	if (edata == NULL) {
 		return NULL;
 	}
-	extent_esn_set(extent, esn);
-	return extent;
+	edata_esn_set(edata, esn);
+	return edata;
 }
 
 void
@@ -509,6 +523,7 @@ base_postfork_child(tsdn_t *tsdn, base_t *base) {
 
 bool
 base_boot(tsdn_t *tsdn) {
-	b0 = base_new(tsdn, 0, (extent_hooks_t *)&extent_hooks_default);
+	b0 = base_new(tsdn, 0, (extent_hooks_t *)&ehooks_default_extent_hooks,
+	    /* metadata_use_hooks */ true);
 	return (b0 == NULL);
 }