1 files changed, 128 insertions, 155 deletions

diff --git a/dep/jemalloc/src/chunk_mmap.c b/dep/jemalloc/src/chunk_mmap.c
index bc367559774..8a42e75915f 100644
--- a/dep/jemalloc/src/chunk_mmap.c
+++ b/dep/jemalloc/src/chunk_mmap.c
@@ -2,53 +2,36 @@
 #include "jemalloc/internal/jemalloc_internal.h"
 
 /******************************************************************************/
-/* Data. */
-
-/*
- * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
- * potentially avoid some system calls.
- */
-#ifndef NO_TLS
-static __thread bool	mmap_unaligned_tls
-    JEMALLOC_ATTR(tls_model("initial-exec"));
-#define	MMAP_UNALIGNED_GET()	mmap_unaligned_tls
-#define	MMAP_UNALIGNED_SET(v)	do {					\
-	mmap_unaligned_tls = (v);					\
-} while (0)
-#else
-static pthread_key_t	mmap_unaligned_tsd;
-#define	MMAP_UNALIGNED_GET()	((bool)pthread_getspecific(mmap_unaligned_tsd))
-#define	MMAP_UNALIGNED_SET(v)	do {					\
-	pthread_setspecific(mmap_unaligned_tsd, (void *)(v));		\
-} while (0)
-#endif
-
-/******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static void	*pages_map(void *addr, size_t size, bool noreserve);
+static void	*pages_map(void *addr, size_t size);
 static void	pages_unmap(void *addr, size_t size);
-static void	*chunk_alloc_mmap_slow(size_t size, bool unaligned,
-    bool noreserve);
-static void	*chunk_alloc_mmap_internal(size_t size, bool noreserve);
+static void	*chunk_alloc_mmap_slow(size_t size, size_t alignment,
+    bool *zero);
 
 /******************************************************************************/
 
 static void *
-pages_map(void *addr, size_t size, bool noreserve)
+pages_map(void *addr, size_t size)
 {
 	void *ret;
 
+	assert(size != 0);
+
+#ifdef _WIN32
+	/*
+	 * If VirtualAlloc can't allocate at the given address when one is
+	 * given, it fails and returns NULL.
+	 */
+	ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
+	    PAGE_READWRITE);
+#else
 	/*
 	 * We don't use MAP_FIXED here, because it can cause the *replacement*
 	 * of existing mappings, and we only want to create new mappings.
 	 */
-	int flags = MAP_PRIVATE | MAP_ANON;
-#ifdef MAP_NORESERVE
-	if (noreserve)
-		flags |= MAP_NORESERVE;
-#endif
-	ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0);
+	ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
+	    -1, 0);
 	assert(ret != NULL);
 
 	if (ret == MAP_FAILED)
@@ -60,16 +43,15 @@ pages_map(void *addr, size_t size, bool noreserve)
 		if (munmap(ret, size) == -1) {
 			char buf[BUFERROR_BUF];
 
-			buferror(errno, buf, sizeof(buf));
-			malloc_write("<jemalloc>: Error in munmap(): ");
-			malloc_write(buf);
-			malloc_write("\n");
+			buferror(buf, sizeof(buf));
+			malloc_printf("<jemalloc: Error in munmap(): %s\n",
+			    buf);
 			if (opt_abort)
 				abort();
 		}
 		ret = NULL;
 	}
-
+#endif
 	assert(ret == NULL || (addr == NULL && ret != addr)
 	    || (addr != NULL && ret == addr));
 	return (ret);
@@ -79,159 +61,150 @@ static void
 pages_unmap(void *addr, size_t size)
 {
 
-	if (munmap(addr, size) == -1) {
+#ifdef _WIN32
+	if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
+#else
+	if (munmap(addr, size) == -1)
+#endif
+	{
 		char buf[BUFERROR_BUF];
 
-		buferror(errno, buf, sizeof(buf));
-		malloc_write("<jemalloc>: Error in munmap(): ");
-		malloc_write(buf);
-		malloc_write("\n");
+		buferror(buf, sizeof(buf));
+		malloc_printf("<jemalloc>: Error in "
+#ifdef _WIN32
+		              "VirtualFree"
+#else
+		              "munmap"
+#endif
+		              "(): %s\n", buf);
 		if (opt_abort)
 			abort();
 	}
 }
 
 static void *
-chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve)
+pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
 {
-	void *ret;
-	size_t offset;
-
-	/* Beware size_t wrap-around. */
-	if (size + chunksize <= size)
+	void *ret = (void *)((uintptr_t)addr + leadsize);
+
+	assert(alloc_size >= leadsize + size);
+#ifdef _WIN32
+	{
+		void *new_addr;
+
+		pages_unmap(addr, alloc_size);
+		new_addr = pages_map(ret, size);
+		if (new_addr == ret)
+			return (ret);
+		if (new_addr)
+			pages_unmap(new_addr, size);
 		return (NULL);
+	}
+#else
+	{
+		size_t trailsize = alloc_size - leadsize - size;
+
+		if (leadsize != 0)
+			pages_unmap(addr, leadsize);
+		if (trailsize != 0)
+			pages_unmap((void *)((uintptr_t)ret + size), trailsize);
+		return (ret);
+	}
+#endif
+}
 
-	ret = pages_map(NULL, size + chunksize, noreserve);
-	if (ret == NULL)
-		return (NULL);
+bool
+pages_purge(void *addr, size_t length)
+{
+	bool unzeroed;
 
-	/* Clean up unneeded leading/trailing space. */
-	offset = CHUNK_ADDR2OFFSET(ret);
-	if (offset != 0) {
-		/* Note that mmap() returned an unaligned mapping. */
-		unaligned = true;
-
-		/* Leading space. */
-		pages_unmap(ret, chunksize - offset);
-
-		ret = (void *)((uintptr_t)ret +
-		    (chunksize - offset));
-
-		/* Trailing space. */
-		pages_unmap((void *)((uintptr_t)ret + size),
-		    offset);
-	} else {
-		/* Trailing space only. */
-		pages_unmap((void *)((uintptr_t)ret + size),
-		    chunksize);
-	}
+#ifdef _WIN32
+	VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
+	unzeroed = true;
+#else
+#  ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
+#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
+#    define JEMALLOC_MADV_ZEROS true
+#  elif defined(JEMALLOC_PURGE_MADVISE_FREE)
+#    define JEMALLOC_MADV_PURGE MADV_FREE
+#    define JEMALLOC_MADV_ZEROS false
+#  else
+#    error "No method defined for purging unused dirty pages."
+#  endif
+	int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
+	unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0);
+#  undef JEMALLOC_MADV_PURGE
+#  undef JEMALLOC_MADV_ZEROS
+#endif
+	return (unzeroed);
+}
 
-	/*
-	 * If mmap() returned an aligned mapping, reset mmap_unaligned so that
-	 * the next chunk_alloc_mmap() execution tries the fast allocation
-	 * method.
-	 */
-	if (unaligned == false)
-		MMAP_UNALIGNED_SET(false);
+static void *
+chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero)
+{
+	void *ret, *pages;
+	size_t alloc_size, leadsize;
 
+	alloc_size = size + alignment - PAGE;
+	/* Beware size_t wrap-around. */
+	if (alloc_size < size)
+		return (NULL);
+	do {
+		pages = pages_map(NULL, alloc_size);
+		if (pages == NULL)
+			return (NULL);
+		leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
+		    (uintptr_t)pages;
+		ret = pages_trim(pages, alloc_size, leadsize, size);
+	} while (ret == NULL);
+
+	assert(ret != NULL);
+	*zero = true;
 	return (ret);
 }
 
-static void *
-chunk_alloc_mmap_internal(size_t size, bool noreserve)
+void *
+chunk_alloc_mmap(size_t size, size_t alignment, bool *zero)
 {
 	void *ret;
+	size_t offset;
 
 	/*
 	 * Ideally, there would be a way to specify alignment to mmap() (like
 	 * NetBSD has), but in the absence of such a feature, we have to work
 	 * hard to efficiently create aligned mappings.  The reliable, but
 	 * slow method is to create a mapping that is over-sized, then trim the
-	 * excess.  However, that always results in at least one call to
+	 * excess.  However, that always results in one or two calls to
 	 * pages_unmap().
 	 *
-	 * A more optimistic approach is to try mapping precisely the right
-	 * amount, then try to append another mapping if alignment is off.  In
-	 * practice, this works out well as long as the application is not
-	 * interleaving mappings via direct mmap() calls.  If we do run into a
-	 * situation where there is an interleaved mapping and we are unable to
-	 * extend an unaligned mapping, our best option is to switch to the
-	 * slow method until mmap() returns another aligned mapping.  This will
-	 * tend to leave a gap in the memory map that is too small to cause
-	 * later problems for the optimistic method.
-	 *
-	 * Another possible confounding factor is address space layout
-	 * randomization (ASLR), which causes mmap(2) to disregard the
-	 * requested address.  mmap_unaligned tracks whether the previous
-	 * chunk_alloc_mmap() execution received any unaligned or relocated
-	 * mappings, and if so, the current execution will immediately fall
-	 * back to the slow method.  However, we keep track of whether the fast
-	 * method would have succeeded, and if so, we make a note to try the
-	 * fast method next time.
+	 * Optimistically try mapping precisely the right amount before falling
+	 * back to the slow method, with the expectation that the optimistic
+	 * approach works most of the time.
 	 */
 
-	if (MMAP_UNALIGNED_GET() == false) {
-		size_t offset;
+	assert(alignment != 0);
+	assert((alignment & chunksize_mask) == 0);
 
-		ret = pages_map(NULL, size, noreserve);
-		if (ret == NULL)
-			return (NULL);
-
-		offset = CHUNK_ADDR2OFFSET(ret);
-		if (offset != 0) {
-			MMAP_UNALIGNED_SET(true);
-			/* Try to extend chunk boundary. */
-			if (pages_map((void *)((uintptr_t)ret + size),
-			    chunksize - offset, noreserve) == NULL) {
-				/*
-				 * Extension failed.  Clean up, then revert to
-				 * the reliable-but-expensive method.
-				 */
-				pages_unmap(ret, size);
-				ret = chunk_alloc_mmap_slow(size, true,
-				    noreserve);
-			} else {
-				/* Clean up unneeded leading space. */
-				pages_unmap(ret, chunksize - offset);
-				ret = (void *)((uintptr_t)ret + (chunksize -
-				    offset));
-			}
-		}
-	} else
-		ret = chunk_alloc_mmap_slow(size, false, noreserve);
+	ret = pages_map(NULL, size);
+	if (ret == NULL)
+		return (NULL);
+	offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
+	if (offset != 0) {
+		pages_unmap(ret, size);
+		return (chunk_alloc_mmap_slow(size, alignment, zero));
+	}
 
+	assert(ret != NULL);
+	*zero = true;
 	return (ret);
 }
 
-void *
-chunk_alloc_mmap(size_t size)
-{
-	return chunk_alloc_mmap_internal(size, false);
-}
-
-void *
-chunk_alloc_mmap_noreserve(size_t size)
-{
-	return chunk_alloc_mmap_internal(size, true);
-}
-
-void
-chunk_dealloc_mmap(void *chunk, size_t size)
-{
-
-	pages_unmap(chunk, size);
-}
-
 bool
-chunk_mmap_boot(void)
+chunk_dealloc_mmap(void *chunk, size_t size)
 {
 
-#ifdef NO_TLS
-	if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
-		malloc_write("<jemalloc>: Error in pthread_key_create()\n");
-		return (true);
-	}
-#endif
+	if (config_munmap)
+		pages_unmap(chunk, size);
 
-	return (false);
+	return (config_munmap == false);
 }