diff options
88 files changed, 6067 insertions, 3827 deletions
diff --git a/dep/PackageList.txt b/dep/PackageList.txt index 13164e4eba3..06d44535619 100644 --- a/dep/PackageList.txt +++ b/dep/PackageList.txt @@ -26,7 +26,7 @@ G3D (a commercial-grade C++ 3D engine available as Open Source (BSD License) jemalloc (a general-purpose scalable concurrent malloc-implementation) http://www.canonware.com/jemalloc/ - Version: 5.1.0 + Version: 5.2.0 libreadline (command line editing library) https://cnswww.cns.cwru.edu/php/chet/readline/rltop.html diff --git a/dep/jemalloc/CMakeLists.txt b/dep/jemalloc/CMakeLists.txt index c2a627e2694..1776fda2cfd 100644 --- a/dep/jemalloc/CMakeLists.txt +++ b/dep/jemalloc/CMakeLists.txt @@ -49,7 +49,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT NOJEM) ${CMAKE_CURRENT_SOURCE_DIR}/src/extent_dss.c ${CMAKE_CURRENT_SOURCE_DIR}/src/extent_mmap.c ${CMAKE_CURRENT_SOURCE_DIR}/src/hash.c - ${CMAKE_CURRENT_SOURCE_DIR}/src/hooks.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/hook.c ${CMAKE_CURRENT_SOURCE_DIR}/src/jemalloc.c ${CMAKE_CURRENT_SOURCE_DIR}/src/jemalloc_cpp.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/large.c @@ -63,8 +63,10 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT NOJEM) ${CMAKE_CURRENT_SOURCE_DIR}/src/prof.c ${CMAKE_CURRENT_SOURCE_DIR}/src/rtree.c ${CMAKE_CURRENT_SOURCE_DIR}/src/stats.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/sc.c ${CMAKE_CURRENT_SOURCE_DIR}/src/sz.c ${CMAKE_CURRENT_SOURCE_DIR}/src/tcache.c + ${CMAKE_CURRENT_SOURCE_DIR}/src/test_hooks.c ${CMAKE_CURRENT_SOURCE_DIR}/src/ticker.c ${CMAKE_CURRENT_SOURCE_DIR}/src/tsd.c ${CMAKE_CURRENT_SOURCE_DIR}/src/witness.c diff --git a/dep/jemalloc/COPYING b/dep/jemalloc/COPYING index 98458d971ac..3b7fd3585d2 100644 --- a/dep/jemalloc/COPYING +++ b/dep/jemalloc/COPYING @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2018 Jason Evans <jasone@canonware.com>. +Copyright (C) 2002-present Jason Evans <jasone@canonware.com>. All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2018 Facebook, Inc. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/dep/jemalloc/ChangeLog b/dep/jemalloc/ChangeLog index 29a00fb78e5..7c73a8f2376 100644 --- a/dep/jemalloc/ChangeLog +++ b/dep/jemalloc/ChangeLog @@ -4,7 +4,110 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc -* 5.1.0 (May 4th, 2018) +* 5.2.0 (April 2, 2019) + + This release includes a few notable improvements, which are summarized below: + 1) improved fast-path performance from the optimizations by @djwatson; 2) + reduced virtual memory fragmentation and metadata usage; and 3) bug fixes on + setting the number of background threads. In addition, peak / spike memory + usage is improved with certain allocation patterns. As usual, the release and + prior dev versions have gone through large-scale production testing. + + New features: + - Implement oversize_threshold, which uses a dedicated arena for allocations + crossing the specified threshold to reduce fragmentation. (@interwq) + - Add extents usage information to stats. (@tyleretzel) + - Log time information for sampled allocations. (@tyleretzel) + - Support 0 size in sdallocx. (@djwatson) + - Output rate for certain counters in malloc_stats. (@zinoale) + - Add configure option --enable-readlinkat, which allows the use of readlinkat + over readlink. (@davidtgoldblatt) + - Add configure options --{enable,disable}-{static,shared} to allow not + building unwanted libraries. (@Ericson2314) + - Add configure option --disable-libdl to enable fully static builds. + (@interwq) + - Add mallctl interfaces: + + opt.oversize_threshold (@interwq) + + stats.arenas.<i>.extent_avail (@tyleretzel) + + stats.arenas.<i>.extents.<j>.n{dirty,muzzy,retained} (@tyleretzel) + + stats.arenas.<i>.extents.<j>.{dirty,muzzy,retained}_bytes + (@tyleretzel) + + Portability improvements: + - Update MSVC builds. (@maksqwe, @rustyx) + - Workaround a compiler optimizer bug on s390x. (@rkmisra) + - Make use of pthread_set_name_np(3) on FreeBSD. (@trasz) + - Implement malloc_getcpu() to enable percpu_arena for windows. (@santagada) + - Link against -pthread instead of -lpthread. (@paravoid) + - Make background_thread not dependent on libdl. (@interwq) + - Add stringify to fix a linker directive issue on MSVC. (@daverigby) + - Detect and fall back when 8-bit atomics are unavailable. (@interwq) + - Fall back to the default pthread_create if dlsym(3) fails. (@interwq) + + Optimizations and refactors: + - Refactor the TSD module. (@davidtgoldblatt) + - Avoid taking extents_muzzy mutex when muzzy is disabled. (@interwq) + - Avoid taking large_mtx for auto arenas on the tcache flush path. (@interwq) + - Optimize ixalloc by avoiding a size lookup. (@interwq) + - Implement opt.oversize_threshold which uses a dedicated arena for requests + crossing the threshold, also eagerly purges the oversize extents. Default + the threshold to 8 MiB. (@interwq) + - Clean compilation with -Wextra. (@gnzlbg, @jasone) + - Refactor the size class module. (@davidtgoldblatt) + - Refactor the stats emitter. (@tyleretzel) + - Optimize pow2_ceil. (@rkmisra) + - Avoid runtime detection of lazy purging on FreeBSD. (@trasz) + - Optimize mmap(2) alignment handling on FreeBSD. (@trasz) + - Improve error handling for THP state initialization. (@jsteemann) + - Rework the malloc() fast path. (@djwatson) + - Rework the free() fast path. (@djwatson) + - Refactor and optimize the tcache fill / flush paths. (@djwatson) + - Optimize sync / lwsync on PowerPC. (@chmeeedalf) + - Bypass extent_dalloc() when retain is enabled. (@interwq) + - Optimize the locking on large deallocation. (@interwq) + - Reduce the number of pages committed from sanity checking in debug build. + (@trasz, @interwq) + - Deprecate OSSpinLock. (@interwq) + - Lower the default number of background threads to 4 (when the feature + is enabled). (@interwq) + - Optimize the trylock spin wait. (@djwatson) + - Use arena index for arena-matching checks. (@interwq) + - Avoid forced decay on thread termination when using background threads. + (@interwq) + - Disable muzzy decay by default. (@djwatson, @interwq) + - Only initialize libgcc unwinder when profiling is enabled. (@paravoid, + @interwq) + + Bug fixes (all only relevant to jemalloc 5.x): + - Fix background thread index issues with max_background_threads. (@djwatson, + @interwq) + - Fix stats output for opt.lg_extent_max_active_fit. (@interwq) + - Fix opt.prof_prefix initialization. (@davidtgoldblatt) + - Properly trigger decay on tcache destroy. (@interwq, @amosbird) + - Fix tcache.flush. (@interwq) + - Detect whether explicit extent zero out is necessary with huge pages or + custom extent hooks, which may change the purge semantics. (@interwq) + - Fix a side effect caused by extent_max_active_fit combined with decay-based + purging, where freed extents can accumulate and not be reused for an + extended period of time. (@interwq, @mpghf) + - Fix a missing unlock on extent register error handling. (@zoulasc) + + Testing: + - Simplify the Travis script output. (@gnzlbg) + - Update the test scripts for FreeBSD. (@devnexen) + - Add unit tests for the producer-consumer pattern. (@interwq) + - Add Cirrus-CI config for FreeBSD builds. (@jasone) + - Add size-matching sanity checks on tcache flush. (@davidtgoldblatt, + @interwq) + + Incompatible changes: + - Remove --with-lg-page-sizes. (@davidtgoldblatt) + + Documentation: + - Attempt to build docs by default, however skip doc building when xsltproc + is missing. (@interwq, @cmuellner) + +* 5.1.0 (May 4, 2018) This release is primarily about fine-tuning, ranging from several new features to numerous notable performance and portability enhancements. The release and diff --git a/dep/jemalloc/include/jemalloc/internal/arena_externs.h b/dep/jemalloc/include/jemalloc/internal/arena_externs.h index 4b3732b41da..2bdddb77a04 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_externs.h @@ -3,8 +3,8 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/extent_dss.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" extern ssize_t opt_dirty_decay_ms; @@ -16,13 +16,17 @@ extern const char *percpu_arena_mode_names[]; extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS]; extern malloc_mutex_t arenas_lock; +extern size_t opt_oversize_threshold; +extern size_t oversize_threshold; + void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats); + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats); void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); #ifdef JEMALLOC_JET @@ -59,13 +63,14 @@ void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize); void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); -void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, - extent_t *extent, void *ptr); +void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr); bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); + size_t extra, bool zero, size_t *newsize); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache); + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_dirty_decay_ms_default_get(void); @@ -79,7 +84,12 @@ void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -void arena_boot(void); +bool arena_init_huge(void); +bool arena_is_huge(unsigned arena_ind); +arena_t *arena_choose_huge(tsd_t *tsd); +bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard); +void arena_boot(sc_data_t *sc_data); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h b/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h index 2b7e77e7216..614deddd204 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_inlines_b.h @@ -4,10 +4,36 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" +JEMALLOC_ALWAYS_INLINE bool +arena_has_default_hooks(arena_t *arena) { + return (extent_hooks_get(arena) == &extent_hooks_default); +} + +JEMALLOC_ALWAYS_INLINE arena_t * +arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { + if (arena != NULL) { + return arena; + } + + /* + * For huge allocations, use the dedicated huge arena if both are true: + * 1) is using auto arena selection (i.e. arena == NULL), and 2) the + * thread is not assigned to a manual arena. + */ + if (unlikely(size >= oversize_threshold)) { + arena_t *tsd_arena = tsd_arena_get(tsd); + if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { + return arena_choose_huge(tsd); + } + } + + return arena_choose(tsd, NULL); +} + JEMALLOC_ALWAYS_INLINE prof_tctx_t * arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { cassert(config_prof); @@ -28,7 +54,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { } JEMALLOC_ALWAYS_INLINE void -arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize, +arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -47,7 +73,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize, } static inline void -arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) { +arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -57,6 +83,32 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) { large_prof_tctx_reset(tsdn, extent); } +JEMALLOC_ALWAYS_INLINE nstime_t +arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, + alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + /* + * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're + * sure we have a sampled allocation. + */ + assert(!extent_slab_get(extent)); + return large_prof_alloc_time_get(extent); +} + +JEMALLOC_ALWAYS_INLINE void +arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + assert(!extent_slab_get(extent)); + large_prof_alloc_time_set(extent, t); +} + JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { tsd_t *tsd; @@ -83,14 +135,33 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { arena_decay_ticks(tsdn, arena, 1); } +/* Purge a single extent to retained / unmapped directly. */ +JEMALLOC_ALWAYS_INLINE void +arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks, + extent_t *extent) { + size_t extent_size = extent_size_get(extent); + extent_dalloc_wrapper(tsdn, arena, + r_extent_hooks, extent); + if (config_stats) { + /* Update stats accordingly. */ + arena_stats_lock(tsdn, &arena->stats); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->nmadvise, 1); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); + arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, + extent_size); + arena_stats_unlock(tsdn, &arena->stats); + } +} + JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); - assert(size != 0); if (likely(tcache != NULL)) { - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); } @@ -119,7 +190,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) { szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } @@ -152,7 +223,7 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { /* Only slab members should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } @@ -173,7 +244,7 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -203,7 +274,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, if (alloc_ctx != NULL) { szind = alloc_ctx->szind; slab = alloc_ctx->slab; - assert(szind != NSIZES); + assert(szind != SC_NSIZES); } else { rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, @@ -215,7 +286,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -225,7 +296,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, slow_path); } else { if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { + if (config_prof && unlikely(szind < SC_NBINS)) { arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); } else { @@ -242,7 +313,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, static inline void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); szind_t szind; bool slab; @@ -252,7 +323,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if ((config_prof && opt_prof) || config_debug) { @@ -264,7 +335,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { (uintptr_t)ptr, true, &szind, &slab); assert(szind == sz_size2index(size)); - assert((config_prof && opt_prof) || slab == (szind < NBINS)); + assert((config_prof && opt_prof) || slab == (szind < SC_NBINS)); if (config_debug) { extent_t *extent = rtree_extent_read(tsdn, @@ -288,7 +359,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, alloc_ctx_t *alloc_ctx, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); if (unlikely(tcache == NULL)) { arena_sdalloc_no_tcache(tsdn, ptr, size); @@ -297,7 +368,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, szind_t szind; bool slab; - UNUSED alloc_ctx_t local_ctx; + alloc_ctx_t local_ctx; if (config_prof && opt_prof) { if (alloc_ctx == NULL) { /* Uncommon case and should be a static check. */ @@ -318,7 +389,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if (config_debug) { @@ -337,7 +408,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, slow_path); } else { if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { + if (config_prof && unlikely(szind < SC_NBINS)) { arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); } else { diff --git a/dep/jemalloc/include/jemalloc/internal/arena_stats.h b/dep/jemalloc/include/jemalloc/internal/arena_stats.h index 5f3dca8b155..ef1e25b31f3 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_stats.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_stats.h @@ -4,7 +4,9 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex_prof.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" + +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS /* * In those architectures that support 64-bit atomics, we use atomic updates for @@ -48,6 +50,22 @@ struct arena_stats_decay_s { arena_stats_u64_t purged; }; +typedef struct arena_stats_extents_s arena_stats_extents_t; +struct arena_stats_extents_s { + /* + * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t. + * We track both bytes and # of extents: two extents in the same bucket + * may have different sizes if adjacent size classes differ by more than + * a page, so bytes cannot always be derived from # of extents. + */ + atomic_zu_t ndirty; + atomic_zu_t dirty_bytes; + atomic_zu_t nmuzzy; + atomic_zu_t muzzy_bytes; + atomic_zu_t nretained; + atomic_zu_t retained_bytes; +}; + /* * Arena stats. Note that fields marked "derived" are not directly maintained * within the arena code; rather their values are derived during stats merge @@ -69,6 +87,9 @@ struct arena_stats_s { */ atomic_zu_t retained; /* Derived. */ + /* Number of extent_t structs allocated by base, but not being used. */ + atomic_zu_t extent_avail; + arena_stats_decay_t decay_dirty; arena_stats_decay_t decay_muzzy; @@ -88,14 +109,14 @@ struct arena_stats_s { mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; /* One element for each large size class. */ - arena_stats_large_t lstats[NSIZES - NBINS]; + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; /* Arena uptime. */ nstime_t uptime; }; static inline bool -arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) { +arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { if (config_debug) { for (size_t i = 0; i < sizeof(arena_stats_t); i++) { assert(((char *)arena_stats)[i] == 0); @@ -147,11 +168,11 @@ arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, #endif } -UNUSED static inline void +static inline void arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, arena_stats_u64_t *p, uint64_t x) { #ifdef JEMALLOC_ATOMIC_U64 - UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); + uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); assert(r - x <= r); #else malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); @@ -176,7 +197,8 @@ arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) { } static inline size_t -arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { +arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p) { #ifdef JEMALLOC_ATOMIC_U64 return atomic_load_zu(p, ATOMIC_RELAXED); #else @@ -186,8 +208,8 @@ arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { } static inline void -arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, - size_t x) { +arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p, size_t x) { #ifdef JEMALLOC_ATOMIC_U64 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED); #else @@ -198,10 +220,10 @@ arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, } static inline void -arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, - size_t x) { +arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p, size_t x) { #ifdef JEMALLOC_ATOMIC_U64 - UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); + size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); assert(r - x <= r); #else malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); @@ -222,7 +244,7 @@ arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, szind_t szind, uint64_t nrequests) { arena_stats_lock(tsdn, arena_stats); arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind - - NBINS].nrequests, nrequests); + SC_NBINS].nrequests, nrequests); arena_stats_unlock(tsdn, arena_stats); } @@ -233,5 +255,4 @@ arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { arena_stats_unlock(tsdn, arena_stats); } - #endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/arena_structs_b.h b/dep/jemalloc/include/jemalloc/internal/arena_structs_b.h index 38bc95962d8..950bd13c27a 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_structs_b.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_structs_b.h @@ -10,7 +10,7 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/ticker.h" @@ -90,6 +90,9 @@ struct arena_s { */ atomic_u_t nthreads[2]; + /* Next bin shard for binding new threads. Synchronization: atomic. */ + atomic_u_t binshard_next; + /* * When percpu_arena is enabled, to amortize the cost of reading / * updating the current CPU id, track the most recent thread accessing @@ -196,6 +199,7 @@ struct arena_s { * Synchronization: extent_avail_mtx. */ extent_tree_t extent_avail; + atomic_zu_t extent_avail_cnt; malloc_mutex_t extent_avail_mtx; /* @@ -203,7 +207,7 @@ struct arena_s { * * Synchronization: internal. */ - bin_t bins[NBINS]; + bins_t bins[SC_NBINS]; /* * Base allocator, from which arena metadata are allocated. diff --git a/dep/jemalloc/include/jemalloc/internal/arena_types.h b/dep/jemalloc/include/jemalloc/internal/arena_types.h index 70001b5f16f..624937e4f59 100644 --- a/dep/jemalloc/include/jemalloc/internal/arena_types.h +++ b/dep/jemalloc/include/jemalloc/internal/arena_types.h @@ -1,13 +1,15 @@ #ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H #define JEMALLOC_INTERNAL_ARENA_TYPES_H +#include "jemalloc/internal/sc.h" + /* Maximum number of regions in one slab. */ -#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) +#define LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN) #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) /* Default decay times in milliseconds. */ #define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) -#define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000) +#define MUZZY_DECAY_MS_DEFAULT (0) /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 @@ -40,4 +42,10 @@ typedef enum { #define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base) #define PERCPU_ARENA_DEFAULT percpu_arena_disabled +/* + * When allocation_size >= oversize_threshold, use the dedicated huge arena + * (unless have explicitly spicified arena index). 0 disables the feature. + */ +#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20) + #endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/atomic.h b/dep/jemalloc/include/jemalloc/internal/atomic.h index adadb1a3acb..a76f54cee3f 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic.h @@ -1,12 +1,19 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_H -#define ATOMIC_INLINE static inline +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE +#define JEMALLOC_U8_ATOMICS #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) # include "jemalloc/internal/atomic_gcc_atomic.h" +# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(JEMALLOC_GCC_SYNC_ATOMICS) # include "jemalloc/internal/atomic_gcc_sync.h" +# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(_MSC_VER) # include "jemalloc/internal/atomic_msvc.h" #elif defined(JEMALLOC_C11_ATOMICS) @@ -66,6 +73,8 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR) JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR) +JEMALLOC_GENERATE_INT_ATOMICS(uint8_t, u8, 0) + JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2) #ifdef JEMALLOC_ATOMIC_U64 diff --git a/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h b/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h index 30846e4d27b..e02b7cbe3c3 100644 --- a/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h +++ b/dep/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h @@ -27,8 +27,10 @@ atomic_fence(atomic_memory_order_t mo) { asm volatile("" ::: "memory"); # if defined(__i386__) || defined(__x86_64__) /* This is implicit on x86. */ -# elif defined(__ppc__) +# elif defined(__ppc64__) asm volatile("lwsync"); +# elif defined(__ppc__) + asm volatile("sync"); # elif defined(__sparc__) && defined(__arch64__) if (mo == atomic_memory_order_acquire) { asm volatile("membar #LoadLoad | #LoadStore"); @@ -113,8 +115,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \ } \ \ ATOMIC_INLINE type \ -atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ +atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ + atomic_memory_order_t mo) { \ /* \ * Because of FreeBSD, we care about gcc 4.2, which doesn't have\ * an atomic exchange builtin. We fake it with a CAS loop. \ @@ -129,8 +131,9 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ @@ -142,8 +145,9 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ } \ ATOMIC_INLINE bool \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ diff --git a/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h b/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h index 3209aa49ffa..0f997e18bee 100644 --- a/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/background_thread_externs.h @@ -8,7 +8,6 @@ extern atomic_b_t background_thread_enabled_state; extern size_t n_background_threads; extern size_t max_background_threads; extern background_thread_info_t *background_thread_info; -extern bool can_enable_background_thread; bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_threads_enable(tsd_t *tsd); diff --git a/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h b/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h index ef50231e8d7..f85e86fa375 100644 --- a/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h +++ b/dep/jemalloc/include/jemalloc/internal/background_thread_inlines.h @@ -15,7 +15,12 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) { JEMALLOC_ALWAYS_INLINE background_thread_info_t * arena_background_thread_info_get(arena_t *arena) { unsigned arena_ind = arena_ind_get(arena); - return &background_thread_info[arena_ind % ncpus]; + return &background_thread_info[arena_ind % max_background_threads]; +} + +JEMALLOC_ALWAYS_INLINE background_thread_info_t * +background_thread_info_get(size_t ind) { + return &background_thread_info[ind % max_background_threads]; } JEMALLOC_ALWAYS_INLINE uint64_t diff --git a/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h b/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h index c1107dfe9c4..c02aa434c7d 100644 --- a/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/background_thread_structs.h @@ -9,6 +9,7 @@ #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT +#define DEFAULT_NUM_BACKGROUND_THREAD 4 typedef enum { background_thread_stopped, diff --git a/dep/jemalloc/include/jemalloc/internal/base_structs.h b/dep/jemalloc/include/jemalloc/internal/base_structs.h index 2102247ac43..07f214eb2f2 100644 --- a/dep/jemalloc/include/jemalloc/internal/base_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/base_structs.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" /* Embedded at the beginning of every block of base-managed virtual memory. */ struct base_block_s { @@ -46,7 +46,7 @@ struct base_s { base_block_t *blocks; /* Heap of extents that track unused trailing space within blocks. */ - extent_heap_t avail[NSIZES]; + extent_heap_t avail[SC_NSIZES]; /* Stats, only maintained if config_stats. */ size_t allocated; diff --git a/dep/jemalloc/include/jemalloc/internal/bin.h b/dep/jemalloc/include/jemalloc/internal/bin.h index 9b416ada7ed..f542c882987 100644 --- a/dep/jemalloc/include/jemalloc/internal/bin.h +++ b/dep/jemalloc/include/jemalloc/internal/bin.h @@ -1,10 +1,12 @@ #ifndef JEMALLOC_INTERNAL_BIN_H #define JEMALLOC_INTERNAL_BIN_H +#include "jemalloc/internal/bin_stats.h" +#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/extent_types.h" #include "jemalloc/internal/extent_structs.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/bin_stats.h" +#include "jemalloc/internal/sc.h" /* * A bin contains a set of extents that are currently being used for slab @@ -41,6 +43,9 @@ struct bin_info_s { /* Total number of regions in a slab for this bin's size class. */ uint32_t nregs; + /* Number of sharded bins in each arena for this size class. */ + uint32_t n_shards; + /* * Metadata used to manipulate bitmaps for slabs associated with this * bin. @@ -48,8 +53,7 @@ struct bin_info_s { bitmap_info_t bitmap_info; }; -extern const bin_info_t bin_infos[NBINS]; - +extern bin_info_t bin_infos[SC_NBINS]; typedef struct bin_s bin_t; struct bin_s { @@ -78,6 +82,18 @@ struct bin_s { bin_stats_t stats; }; +/* A set of sharded bins of the same size class. */ +typedef struct bins_s bins_t; +struct bins_s { + /* Sharded bins. Dynamically sized. */ + bin_t *bin_shards; +}; + +void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]); +bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards); +void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]); + /* Initializes a bin to empty. Returns true on error. */ bool bin_init(bin_t *bin); @@ -90,7 +106,7 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); static inline void bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { malloc_mutex_lock(tsdn, &bin->lock); - malloc_mutex_prof_read(tsdn, &dst_bin_stats->mutex_data, &bin->lock); + malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock); dst_bin_stats->nmalloc += bin->stats.nmalloc; dst_bin_stats->ndalloc += bin->stats.ndalloc; dst_bin_stats->nrequests += bin->stats.nrequests; diff --git a/dep/jemalloc/include/jemalloc/internal/bin_types.h b/dep/jemalloc/include/jemalloc/internal/bin_types.h new file mode 100644 index 00000000000..3533606b90c --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/bin_types.h @@ -0,0 +1,17 @@ +#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H +#define JEMALLOC_INTERNAL_BIN_TYPES_H + +#include "jemalloc/internal/sc.h" + +#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH) +#define N_BIN_SHARDS_DEFAULT 1 + +/* Used in TSD static initializer only. Real init in arena_bind(). */ +#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}} + +typedef struct tsd_binshards_s tsd_binshards_t; +struct tsd_binshards_s { + uint8_t binshard[SC_NBINS]; +}; + +#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/bit_util.h b/dep/jemalloc/include/jemalloc/internal/bit_util.h index 8d078a8a35e..c045eb86878 100644 --- a/dep/jemalloc/include/jemalloc/internal/bit_util.h +++ b/dep/jemalloc/include/jemalloc/internal/bit_util.h @@ -27,6 +27,25 @@ ffs_u(unsigned bitmap) { return JEMALLOC_INTERNAL_FFS(bitmap); } +#ifdef JEMALLOC_INTERNAL_POPCOUNTL +BIT_UTIL_INLINE unsigned +popcount_lu(unsigned long bitmap) { + return JEMALLOC_INTERNAL_POPCOUNTL(bitmap); +} +#endif + +/* + * Clears first unset bit in bitmap, and returns + * place of bit. bitmap *must not* be 0. + */ + +BIT_UTIL_INLINE size_t +cfs_lu(unsigned long* bitmap) { + size_t bit = ffs_lu(*bitmap) - 1; + *bitmap ^= ZU(1) << bit; + return bit; +} + BIT_UTIL_INLINE unsigned ffs_zu(size_t bitmap) { #if LG_SIZEOF_PTR == LG_SIZEOF_INT @@ -63,6 +82,22 @@ ffs_u32(uint32_t bitmap) { BIT_UTIL_INLINE uint64_t pow2_ceil_u64(uint64_t x) { +#if (defined(__amd64__) || defined(__x86_64__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + if(unlikely(x <= 1)) { + return x; + } + size_t msb_on_index; +#if (defined(__amd64__) || defined(__x86_64__)) + asm ("bsrq %1, %0" + : "=r"(msb_on_index) // Outputs. + : "r"(x-1) // Inputs. + ); +#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + msb_on_index = (63 ^ __builtin_clzll(x - 1)); +#endif + assert(msb_on_index < 63); + return 1ULL << (msb_on_index + 1); +#else x--; x |= x >> 1; x |= x >> 2; @@ -72,10 +107,27 @@ pow2_ceil_u64(uint64_t x) { x |= x >> 32; x++; return x; +#endif } BIT_UTIL_INLINE uint32_t pow2_ceil_u32(uint32_t x) { +#if ((defined(__i386__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) && (!defined(__s390__))) + if(unlikely(x <= 1)) { + return x; + } + size_t msb_on_index; +#if (defined(__i386__)) + asm ("bsr %1, %0" + : "=r"(msb_on_index) // Outputs. + : "r"(x-1) // Inputs. + ); +#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + msb_on_index = (31 ^ __builtin_clz(x - 1)); +#endif + assert(msb_on_index < 31); + return 1U << (msb_on_index + 1); +#else x--; x |= x >> 1; x |= x >> 2; @@ -84,6 +136,7 @@ pow2_ceil_u32(uint32_t x) { x |= x >> 16; x++; return x; +#endif } /* Compute the smallest power of 2 that is >= x. */ @@ -160,6 +213,27 @@ lg_floor(size_t x) { } #endif +BIT_UTIL_INLINE unsigned +lg_ceil(size_t x) { + return lg_floor(x) + ((x & (x - 1)) == 0 ? 0 : 1); +} + #undef BIT_UTIL_INLINE +/* A compile-time version of lg_floor and lg_ceil. */ +#define LG_FLOOR_1(x) 0 +#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1)) +#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2)) +#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4)) +#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) +#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) +#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) +#if LG_SIZEOF_PTR == 2 +# define LG_FLOOR(x) LG_FLOOR_32((x)) +#else +# define LG_FLOOR(x) LG_FLOOR_64((x)) +#endif + +#define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1)) + #endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/bitmap.h b/dep/jemalloc/include/jemalloc/internal/bitmap.h index ac990290a5b..c3f9cb490f6 100644 --- a/dep/jemalloc/include/jemalloc/internal/bitmap.h +++ b/dep/jemalloc/include/jemalloc/internal/bitmap.h @@ -3,18 +3,18 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/bit_util.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef unsigned long bitmap_t; #define LG_SIZEOF_BITMAP LG_SIZEOF_LONG /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES +#if LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES) /* Maximum bitmap bit count is determined by maximum regions per slab. */ # define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS #else /* Maximum bitmap bit count is determined by number of extent size classes. */ -# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES +# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES) #endif #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) diff --git a/dep/jemalloc/include/jemalloc/internal/cache_bin.h b/dep/jemalloc/include/jemalloc/internal/cache_bin.h index 12f3ef2dd0a..d14556a3da8 100644 --- a/dep/jemalloc/include/jemalloc/internal/cache_bin.h +++ b/dep/jemalloc/include/jemalloc/internal/cache_bin.h @@ -88,11 +88,21 @@ JEMALLOC_ALWAYS_INLINE void * cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { void *ret; - if (unlikely(bin->ncached == 0)) { - bin->low_water = -1; - *success = false; - return NULL; + bin->ncached--; + + /* + * Check for both bin->ncached == 0 and ncached < low_water + * in a single branch. + */ + if (unlikely(bin->ncached <= bin->low_water)) { + bin->low_water = bin->ncached; + if (bin->ncached == -1) { + bin->ncached = 0; + *success = false; + return NULL; + } } + /* * success (instead of ret) should be checked upon the return of this * function. We avoid checking (ret == NULL) because there is never a @@ -101,14 +111,21 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { * cacheline). */ *success = true; - ret = *(bin->avail - bin->ncached); - bin->ncached--; + ret = *(bin->avail - (bin->ncached + 1)); - if (unlikely(bin->ncached < bin->low_water)) { - bin->low_water = bin->ncached; + return ret; +} + +JEMALLOC_ALWAYS_INLINE bool +cache_bin_dalloc_easy(cache_bin_t *bin, cache_bin_info_t *bin_info, void *ptr) { + if (unlikely(bin->ncached == bin_info->ncached_max)) { + return false; } + assert(bin->ncached < bin_info->ncached_max); + bin->ncached++; + *(bin->avail - bin->ncached) = ptr; - return ret; + return true; } #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/ctl.h b/dep/jemalloc/include/jemalloc/internal/ctl.h index d927d94801e..775fdec04e2 100644 --- a/dep/jemalloc/include/jemalloc/internal/ctl.h +++ b/dep/jemalloc/include/jemalloc/internal/ctl.h @@ -5,7 +5,7 @@ #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/stats.h" /* Maximum ctl tree depth. */ @@ -40,8 +40,9 @@ typedef struct ctl_arena_stats_s { uint64_t ndalloc_small; uint64_t nrequests_small; - bin_stats_t bstats[NBINS]; - arena_stats_large_t lstats[NSIZES - NBINS]; + bin_stats_t bstats[SC_NBINS]; + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; + arena_stats_extents_t estats[SC_NPSIZES]; } ctl_arena_stats_t; typedef struct ctl_stats_s { diff --git a/dep/jemalloc/include/jemalloc/internal/emitter.h b/dep/jemalloc/include/jemalloc/internal/emitter.h index 3a2b2f7f2e4..0a8bc2c06b0 100644 --- a/dep/jemalloc/include/jemalloc/internal/emitter.h +++ b/dep/jemalloc/include/jemalloc/internal/emitter.h @@ -45,7 +45,9 @@ struct emitter_col_s { int int_val; unsigned unsigned_val; uint32_t uint32_val; + uint32_t uint32_t_val; uint64_t uint64_val; + uint64_t uint64_t_val; size_t size_val; ssize_t ssize_val; const char *str_val; @@ -60,17 +62,6 @@ struct emitter_row_s { ql_head(emitter_col_t) cols; }; -static inline void -emitter_row_init(emitter_row_t *row) { - ql_new(&row->cols); -} - -static inline void -emitter_col_init(emitter_col_t *col, emitter_row_t *row) { - ql_elm_new(col, link); - ql_tail_insert(&row->cols, col, link); -} - typedef struct emitter_s emitter_t; struct emitter_s { emitter_output_t output; @@ -80,18 +71,10 @@ struct emitter_s { int nesting_depth; /* True if we've already emitted a value at the given depth. */ bool item_at_depth; + /* True if we emitted a key and will emit corresponding value next. */ + bool emitted_key; }; -static inline void -emitter_init(emitter_t *emitter, emitter_output_t emitter_output, - void (*write_cb)(void *, const char *), void *cbopaque) { - emitter->output = emitter_output; - emitter->write_cb = write_cb; - emitter->cbopaque = cbopaque; - emitter->item_at_depth = false; - emitter->nesting_depth = 0; -} - /* Internal convenience function. Write to the emitter the given string. */ JEMALLOC_FORMAT_PRINTF(2, 3) static inline void @@ -103,18 +86,6 @@ emitter_printf(emitter_t *emitter, const char *format, ...) { va_end(ap); } -/* Write to the emitter the given string, but only in table mode. */ -JEMALLOC_FORMAT_PRINTF(2, 3) -static inline void -emitter_table_printf(emitter_t *emitter, const char *format, ...) { - if (emitter->output == emitter_output_table) { - va_list ap; - va_start(ap, format); - malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); - va_end(ap); - } -} - static inline void emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, emitter_justify_t justify, int width) { @@ -235,201 +206,278 @@ emitter_indent(emitter_t *emitter) { static inline void emitter_json_key_prefix(emitter_t *emitter) { + if (emitter->emitted_key) { + emitter->emitted_key = false; + return; + } emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : ""); emitter_indent(emitter); } +/******************************************************************************/ +/* Public functions for emitter_t. */ + static inline void -emitter_begin(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth == 0); - emitter_printf(emitter, "{"); - emitter_nest_inc(emitter); - } else { - // tabular init - emitter_printf(emitter, "%s", ""); - } +emitter_init(emitter_t *emitter, emitter_output_t emitter_output, + void (*write_cb)(void *, const char *), void *cbopaque) { + emitter->output = emitter_output; + emitter->write_cb = write_cb; + emitter->cbopaque = cbopaque; + emitter->item_at_depth = false; + emitter->emitted_key = false; + emitter->nesting_depth = 0; } +/******************************************************************************/ +/* JSON public API. */ + +/* + * Emits a key (e.g. as appears in an object). The next json entity emitted will + * be the corresponding value. + */ static inline void -emitter_end(emitter_t *emitter) { +emitter_json_key(emitter_t *emitter, const char *json_key) { if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth == 1); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n}\n"); + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "\"%s\": ", json_key); + emitter->emitted_key = true; } } -/* - * Note emits a different kv pair as well, but only in table mode. Omits the - * note if table_note_key is NULL. - */ static inline void -emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, - emitter_type_t value_type, const void *value, - const char *table_note_key, emitter_type_t table_note_value_type, - const void *table_note_value) { +emitter_json_value(emitter_t *emitter, emitter_type_t value_type, + const void *value) { if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": ", json_key); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); - } else { - emitter_indent(emitter); - emitter_printf(emitter, "%s: ", table_key); emitter_print_value(emitter, emitter_justify_none, -1, value_type, value); - if (table_note_key != NULL) { - emitter_printf(emitter, " (%s: ", table_note_key); - emitter_print_value(emitter, emitter_justify_none, -1, - table_note_value_type, table_note_value); - emitter_printf(emitter, ")"); - } - emitter_printf(emitter, "\n"); + emitter->item_at_depth = true; } - emitter->item_at_depth = true; } +/* Shorthand for calling emitter_json_key and then emitter_json_value. */ static inline void -emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, +emitter_json_kv(emitter_t *emitter, const char *json_key, emitter_type_t value_type, const void *value) { - emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, - emitter_type_bool, NULL); + emitter_json_key(emitter, json_key); + emitter_json_value(emitter, value_type, value); } static inline void -emitter_json_kv(emitter_t *emitter, const char *json_key, - emitter_type_t value_type, const void *value) { +emitter_json_array_begin(emitter_t *emitter) { if (emitter->output == emitter_output_json) { - emitter_kv(emitter, json_key, NULL, value_type, value); + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "["); + emitter_nest_inc(emitter); } } +/* Shorthand for calling emitter_json_key and then emitter_json_array_begin. */ static inline void -emitter_table_kv(emitter_t *emitter, const char *table_key, - emitter_type_t value_type, const void *value) { - if (emitter->output == emitter_output_table) { - emitter_kv(emitter, NULL, table_key, value_type, value); +emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) { + emitter_json_key(emitter, json_key); + emitter_json_array_begin(emitter); +} + +static inline void +emitter_json_array_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "]"); } } static inline void -emitter_dict_begin(emitter_t *emitter, const char *json_key, - const char *table_header) { +emitter_json_object_begin(emitter_t *emitter) { if (emitter->output == emitter_output_json) { emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": {", json_key); - emitter_nest_inc(emitter); - } else { - emitter_indent(emitter); - emitter_printf(emitter, "%s\n", table_header); + emitter_printf(emitter, "{"); emitter_nest_inc(emitter); } } +/* Shorthand for calling emitter_json_key and then emitter_json_object_begin. */ static inline void -emitter_dict_end(emitter_t *emitter) { +emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) { + emitter_json_key(emitter, json_key); + emitter_json_object_begin(emitter); +} + +static inline void +emitter_json_object_end(emitter_t *emitter) { if (emitter->output == emitter_output_json) { assert(emitter->nesting_depth > 0); emitter_nest_dec(emitter); emitter_printf(emitter, "\n"); emitter_indent(emitter); emitter_printf(emitter, "}"); - } else { - emitter_nest_dec(emitter); } } + +/******************************************************************************/ +/* Table public API. */ + static inline void -emitter_json_dict_begin(emitter_t *emitter, const char *json_key) { - if (emitter->output == emitter_output_json) { - emitter_dict_begin(emitter, json_key, NULL); +emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { + if (emitter->output == emitter_output_table) { + emitter_indent(emitter); + emitter_printf(emitter, "%s\n", table_key); + emitter_nest_inc(emitter); } } static inline void -emitter_json_dict_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - emitter_dict_end(emitter); +emitter_table_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_table) { + emitter_nest_dec(emitter); } } static inline void -emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { +emitter_table_kv_note(emitter_t *emitter, const char *table_key, + emitter_type_t value_type, const void *value, + const char *table_note_key, emitter_type_t table_note_value_type, + const void *table_note_value) { if (emitter->output == emitter_output_table) { - emitter_dict_begin(emitter, NULL, table_key); + emitter_indent(emitter); + emitter_printf(emitter, "%s: ", table_key); + emitter_print_value(emitter, emitter_justify_none, -1, + value_type, value); + if (table_note_key != NULL) { + emitter_printf(emitter, " (%s: ", table_note_key); + emitter_print_value(emitter, emitter_justify_none, -1, + table_note_value_type, table_note_value); + emitter_printf(emitter, ")"); + } + emitter_printf(emitter, "\n"); } + emitter->item_at_depth = true; } static inline void -emitter_table_dict_end(emitter_t *emitter) { +emitter_table_kv(emitter_t *emitter, const char *table_key, + emitter_type_t value_type, const void *value) { + emitter_table_kv_note(emitter, table_key, value_type, value, NULL, + emitter_type_bool, NULL); +} + + +/* Write to the emitter the given string, but only in table mode. */ +JEMALLOC_FORMAT_PRINTF(2, 3) +static inline void +emitter_table_printf(emitter_t *emitter, const char *format, ...) { if (emitter->output == emitter_output_table) { - emitter_dict_end(emitter); + va_list ap; + va_start(ap, format); + malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); + va_end(ap); } } static inline void -emitter_json_arr_begin(emitter_t *emitter, const char *json_key) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": [", json_key); - emitter_nest_inc(emitter); +emitter_table_row(emitter_t *emitter, emitter_row_t *row) { + if (emitter->output != emitter_output_table) { + return; + } + emitter_col_t *col; + ql_foreach(col, &row->cols, link) { + emitter_print_value(emitter, col->justify, col->width, + col->type, (const void *)&col->bool_val); } + emitter_table_printf(emitter, "\n"); +} + +static inline void +emitter_row_init(emitter_row_t *row) { + ql_new(&row->cols); } static inline void -emitter_json_arr_end(emitter_t *emitter) { +emitter_col_init(emitter_col_t *col, emitter_row_t *row) { + ql_elm_new(col, link); + ql_tail_insert(&row->cols, col, link); +} + + +/******************************************************************************/ +/* + * Generalized public API. Emits using either JSON or table, according to + * settings in the emitter_t. */ + +/* + * Note emits a different kv pair as well, but only in table mode. Omits the + * note if table_note_key is NULL. + */ +static inline void +emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, + emitter_type_t value_type, const void *value, + const char *table_note_key, emitter_type_t table_note_value_type, + const void *table_note_value) { if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "]"); + emitter_json_key(emitter, json_key); + emitter_json_value(emitter, value_type, value); + } else { + emitter_table_kv_note(emitter, table_key, value_type, value, + table_note_key, table_note_value_type, table_note_value); } + emitter->item_at_depth = true; } static inline void -emitter_json_arr_obj_begin(emitter_t *emitter) { +emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, + emitter_type_t value_type, const void *value) { + emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, + emitter_type_bool, NULL); +} + +static inline void +emitter_dict_begin(emitter_t *emitter, const char *json_key, + const char *table_header) { if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "{"); - emitter_nest_inc(emitter); + emitter_json_key(emitter, json_key); + emitter_json_object_begin(emitter); + } else { + emitter_table_dict_begin(emitter, table_header); } } static inline void -emitter_json_arr_obj_end(emitter_t *emitter) { +emitter_dict_end(emitter_t *emitter) { if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "}"); + emitter_json_object_end(emitter); + } else { + emitter_table_dict_end(emitter); } } static inline void -emitter_json_arr_value(emitter_t *emitter, emitter_type_t value_type, - const void *value) { +emitter_begin(emitter_t *emitter) { if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); + assert(emitter->nesting_depth == 0); + emitter_printf(emitter, "{"); + emitter_nest_inc(emitter); + } else { + /* + * This guarantees that we always call write_cb at least once. + * This is useful if some invariant is established by each call + * to write_cb, but doesn't hold initially: e.g., some buffer + * holds a null-terminated string. + */ + emitter_printf(emitter, "%s", ""); } } static inline void -emitter_table_row(emitter_t *emitter, emitter_row_t *row) { - if (emitter->output != emitter_output_table) { - return; - } - emitter_col_t *col; - ql_foreach(col, &row->cols, link) { - emitter_print_value(emitter, col->justify, col->width, - col->type, (const void *)&col->bool_val); +emitter_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth == 1); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n}\n"); } - emitter_table_printf(emitter, "\n"); } #endif /* JEMALLOC_INTERNAL_EMITTER_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/extent_externs.h b/dep/jemalloc/include/jemalloc/internal/extent_externs.h index b8a4d026ceb..8680251abab 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/extent_externs.h @@ -31,6 +31,10 @@ bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool delay_coalesce); extent_state_t extents_state_get(const extents_t *extents); size_t extents_npages_get(extents_t *extents); +/* Get the number of extents in the given page size index. */ +size_t extents_nextents_get(extents_t *extents, pszind_t ind); +/* Get the sum total bytes of the extents in the given page size index. */ +size_t extents_nbytes_get(extents_t *extents, pszind_t ind); extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, diff --git a/dep/jemalloc/include/jemalloc/internal/extent_inlines.h b/dep/jemalloc/include/jemalloc/internal/extent_inlines.h index 77181df8d24..63b710dcf0c 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent_inlines.h +++ b/dep/jemalloc/include/jemalloc/internal/extent_inlines.h @@ -6,6 +6,7 @@ #include "jemalloc/internal/pages.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ql.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" static inline void @@ -34,18 +35,19 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { (uintptr_t)extent2); } -static inline arena_t * -extent_arena_get(const extent_t *extent) { +static inline unsigned +extent_arena_ind_get(const extent_t *extent) { unsigned arena_ind = (unsigned)((extent->e_bits & EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); - /* - * The following check is omitted because we should never actually read - * a NULL arena pointer. - */ - if (false && arena_ind >= MALLOCX_ARENA_LIMIT) { - return NULL; - } assert(arena_ind < MALLOCX_ARENA_LIMIT); + + return arena_ind; +} + +static inline arena_t * +extent_arena_get(const extent_t *extent) { + unsigned arena_ind = extent_arena_ind_get(extent); + return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); } @@ -53,14 +55,14 @@ static inline szind_t extent_szind_get_maybe_invalid(const extent_t *extent) { szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> EXTENT_BITS_SZIND_SHIFT); - assert(szind <= NSIZES); + assert(szind <= SC_NSIZES); return szind; } static inline szind_t extent_szind_get(const extent_t *extent) { szind_t szind = extent_szind_get_maybe_invalid(extent); - assert(szind < NSIZES); /* Never call when "invalid". */ + assert(szind < SC_NSIZES); /* Never call when "invalid". */ return szind; } @@ -69,6 +71,14 @@ extent_usize_get(const extent_t *extent) { return sz_index2size(extent_szind_get(extent)); } +static inline unsigned +extent_binshard_get(const extent_t *extent) { + unsigned binshard = (unsigned)((extent->e_bits & + EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT); + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + return binshard; +} + static inline size_t extent_sn_get(const extent_t *extent) { return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> @@ -176,6 +186,11 @@ extent_prof_tctx_get(const extent_t *extent) { ATOMIC_ACQUIRE); } +static inline nstime_t +extent_prof_alloc_time_get(const extent_t *extent) { + return extent->e_alloc_time; +} + static inline void extent_arena_set(extent_t *extent, arena_t *arena) { unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << @@ -185,12 +200,20 @@ extent_arena_set(extent_t *extent, arena_t *arena) { } static inline void +extent_binshard_set(extent_t *extent, unsigned binshard) { + /* The assertion assumes szind is set already. */ + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) | + ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT); +} + +static inline void extent_addr_set(extent_t *extent, void *addr) { extent->e_addr = addr; } static inline void -extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) { +extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { assert(extent_base_get(extent) == extent_addr_get(extent)); if (alignment < PAGE) { @@ -234,7 +257,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) { static inline void extent_szind_set(extent_t *extent, szind_t szind) { - assert(szind <= NSIZES); /* NSIZES means "invalid". */ + assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); } @@ -247,6 +270,16 @@ extent_nfree_set(extent_t *extent, unsigned nfree) { } static inline void +extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) { + /* The assertion assumes szind is set already. */ + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + extent->e_bits = (extent->e_bits & + (~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) | + ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) | + ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); +} + +static inline void extent_nfree_inc(extent_t *extent) { assert(extent_slab_get(extent)); extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); @@ -259,6 +292,12 @@ extent_nfree_dec(extent_t *extent) { } static inline void +extent_nfree_sub(extent_t *extent, uint64_t n) { + assert(extent_slab_get(extent)); + extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT); +} + +static inline void extent_sn_set(extent_t *extent, size_t sn) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | ((uint64_t)sn << EXTENT_BITS_SN_SHIFT); @@ -300,6 +339,11 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { } static inline void +extent_prof_alloc_time_set(extent_t *extent, nstime_t t) { + nstime_copy(&extent->e_alloc_time, &t); +} + +static inline void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, bool committed, bool dumpable) { @@ -327,7 +371,7 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { extent_addr_set(extent, addr); extent_bsize_set(extent, bsize); extent_slab_set(extent, false); - extent_szind_set(extent, NSIZES); + extent_szind_set(extent, SC_NSIZES); extent_sn_set(extent, sn); extent_state_set(extent, extent_state_active); extent_zeroed_set(extent, true); diff --git a/dep/jemalloc/include/jemalloc/internal/extent_structs.h b/dep/jemalloc/include/jemalloc/internal/extent_structs.h index 4873b9e9e49..ceb18979f1c 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/extent_structs.h @@ -2,11 +2,12 @@ #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/ph.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef enum { extent_state_active = 0, @@ -28,9 +29,10 @@ struct extent_s { * t: state * i: szind * f: nfree + * s: bin_shard * n: sn * - * nnnnnnnn ... nnnnffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa + * nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa * * arena_ind: Arena from which this extent came, or all 1 bits if * unassociated. @@ -75,6 +77,8 @@ struct extent_s { * * nfree: Number of free regions in slab. * + * bin_shard: the shard of the bin from which this extent came. + * * sn: Serial number (potentially non-unique). * * Serial numbers may wrap around if !opt_retain, but as long as @@ -112,7 +116,7 @@ struct extent_s { #define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT) #define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT) -#define EXTENT_BITS_SZIND_WIDTH LG_CEIL_NSIZES +#define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) #define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT) #define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT) @@ -120,7 +124,11 @@ struct extent_s { #define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT) #define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT) -#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) +#define EXTENT_BITS_BINSHARD_WIDTH 6 +#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) +#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT) + +#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT) #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) /* Pointer to the extent that this structure is responsible for. */ @@ -160,11 +168,13 @@ struct extent_s { /* Small region slab metadata. */ arena_slab_data_t e_slab_data; - /* - * Profile counters, used for large objects. Points to a - * prof_tctx_t. - */ - atomic_p_t e_prof_tctx; + /* Profiling data, used for large objects. */ + struct { + /* Time when this was allocated. */ + nstime_t e_alloc_time; + /* Points to a prof_tctx_t. */ + atomic_p_t e_prof_tctx; + }; }; }; typedef ql_head(extent_t) extent_list_t; @@ -180,14 +190,16 @@ struct extents_s { * * Synchronization: mtx. */ - extent_heap_t heaps[NPSIZES+1]; + extent_heap_t heaps[SC_NPSIZES + 1]; + atomic_zu_t nextents[SC_NPSIZES + 1]; + atomic_zu_t nbytes[SC_NPSIZES + 1]; /* * Bitmap for which set bits correspond to non-empty heaps. * * Synchronization: mtx. */ - bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)]; + bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; /* * LRU of all extents in heaps. diff --git a/dep/jemalloc/include/jemalloc/internal/extent_types.h b/dep/jemalloc/include/jemalloc/internal/extent_types.h index c0561d99f8f..acbcf27b51d 100644 --- a/dep/jemalloc/include/jemalloc/internal/extent_types.h +++ b/dep/jemalloc/include/jemalloc/internal/extent_types.h @@ -6,8 +6,6 @@ typedef struct extents_s extents_t; #define EXTENT_HOOKS_INITIALIZER NULL -#define EXTENT_GROW_MAX_PIND (NPSIZES - 1) - /* * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit) * is the max ratio between the size of the active extent and the new extent. diff --git a/dep/jemalloc/include/jemalloc/internal/hash.h b/dep/jemalloc/include/jemalloc/internal/hash.h index dcfc992df37..0270034e876 100644 --- a/dep/jemalloc/include/jemalloc/internal/hash.h +++ b/dep/jemalloc/include/jemalloc/internal/hash.h @@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) { uint32_t k1 = 0; switch (len & 3) { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; + case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; } @@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) { return h1; } -UNUSED static inline void +static inline void hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) { const uint8_t * data = (const uint8_t *) key; @@ -177,28 +177,29 @@ hash_x86_128(const void *key, const int len, uint32_t seed, uint32_t k4 = 0; switch (len & 15) { - case 15: k4 ^= tail[14] << 16; - case 14: k4 ^= tail[13] << 8; + case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH + case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH case 13: k4 ^= tail[12] << 0; k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; - - case 12: k3 ^= tail[11] << 24; - case 11: k3 ^= tail[10] << 16; - case 10: k3 ^= tail[ 9] << 8; + JEMALLOC_FALLTHROUGH + case 12: k3 ^= tail[11] << 24; JEMALLOC_FALLTHROUGH + case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH + case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH case 9: k3 ^= tail[ 8] << 0; k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; - - case 8: k2 ^= tail[ 7] << 24; - case 7: k2 ^= tail[ 6] << 16; - case 6: k2 ^= tail[ 5] << 8; + JEMALLOC_FALLTHROUGH + case 8: k2 ^= tail[ 7] << 24; JEMALLOC_FALLTHROUGH + case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH + case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH case 5: k2 ^= tail[ 4] << 0; k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; - - case 4: k1 ^= tail[ 3] << 24; - case 3: k1 ^= tail[ 2] << 16; - case 2: k1 ^= tail[ 1] << 8; + JEMALLOC_FALLTHROUGH + case 4: k1 ^= tail[ 3] << 24; JEMALLOC_FALLTHROUGH + case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= tail[ 0] << 0; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + JEMALLOC_FALLTHROUGH } } @@ -220,7 +221,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed, r_out[1] = (((uint64_t) h4) << 32) | h3; } -UNUSED static inline void +static inline void hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) { const uint8_t *data = (const uint8_t *) key; @@ -260,22 +261,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t k2 = 0; switch (len & 15) { - case 15: k2 ^= ((uint64_t)(tail[14])) << 48; /* falls through */ - case 14: k2 ^= ((uint64_t)(tail[13])) << 40; /* falls through */ - case 13: k2 ^= ((uint64_t)(tail[12])) << 32; /* falls through */ - case 12: k2 ^= ((uint64_t)(tail[11])) << 24; /* falls through */ - case 11: k2 ^= ((uint64_t)(tail[10])) << 16; /* falls through */ - case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; /* falls through */ + case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH + case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH + case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH + case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH + case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH + case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; - /* falls through */ - case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; /* falls through */ - case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; /* falls through */ - case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; /* falls through */ - case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; /* falls through */ - case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; /* falls through */ - case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; /* falls through */ - case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; /* falls through */ + JEMALLOC_FALLTHROUGH + case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH + case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH + case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH + case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH + case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH + case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; } diff --git a/dep/jemalloc/include/jemalloc/internal/hook.h b/dep/jemalloc/include/jemalloc/internal/hook.h new file mode 100644 index 00000000000..ee246b1e0bb --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/hook.h @@ -0,0 +1,163 @@ +#ifndef JEMALLOC_INTERNAL_HOOK_H +#define JEMALLOC_INTERNAL_HOOK_H + +#include "jemalloc/internal/tsd.h" + +/* + * This API is *extremely* experimental, and may get ripped out, changed in API- + * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc. + * + * It allows hooking the stateful parts of the API to see changes as they + * happen. + * + * Allocation hooks are called after the allocation is done, free hooks are + * called before the free is done, and expand hooks are called after the + * allocation is expanded. + * + * For realloc and rallocx, if the expansion happens in place, the expansion + * hook is called. If it is moved, then the alloc hook is called on the new + * location, and then the free hook is called on the old location (i.e. both + * hooks are invoked in between the alloc and the dalloc). + * + * If we return NULL from OOM, then usize might not be trustworthy. Calling + * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0) + * only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0), + * and only calls the alloc hook). + * + * Reentrancy: + * Reentrancy is guarded against from within the hook implementation. If you + * call allocator functions from within a hook, the hooks will not be invoked + * again. + * Threading: + * The installation of a hook synchronizes with all its uses. If you can + * prove the installation of a hook happens-before a jemalloc entry point, + * then the hook will get invoked (unless there's a racing removal). + * + * Hook insertion appears to be atomic at a per-thread level (i.e. if a thread + * allocates and has the alloc hook invoked, then a subsequent free on the + * same thread will also have the free hook invoked). + * + * The *removal* of a hook does *not* block until all threads are done with + * the hook. Hook authors have to be resilient to this, and need some + * out-of-band mechanism for cleaning up any dynamically allocated memory + * associated with their hook. + * Ordering: + * Order of hook execution is unspecified, and may be different than insertion + * order. + */ + +#define HOOK_MAX 4 + +enum hook_alloc_e { + hook_alloc_malloc, + hook_alloc_posix_memalign, + hook_alloc_aligned_alloc, + hook_alloc_calloc, + hook_alloc_memalign, + hook_alloc_valloc, + hook_alloc_mallocx, + + /* The reallocating functions have both alloc and dalloc variants */ + hook_alloc_realloc, + hook_alloc_rallocx, +}; +/* + * We put the enum typedef after the enum, since this file may get included by + * jemalloc_cpp.cpp, and C++ disallows enum forward declarations. + */ +typedef enum hook_alloc_e hook_alloc_t; + +enum hook_dalloc_e { + hook_dalloc_free, + hook_dalloc_dallocx, + hook_dalloc_sdallocx, + + /* + * The dalloc halves of reallocation (not called if in-place expansion + * happens). + */ + hook_dalloc_realloc, + hook_dalloc_rallocx, +}; +typedef enum hook_dalloc_e hook_dalloc_t; + + +enum hook_expand_e { + hook_expand_realloc, + hook_expand_rallocx, + hook_expand_xallocx, +}; +typedef enum hook_expand_e hook_expand_t; + +typedef void (*hook_alloc)( + void *extra, hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]); + +typedef void (*hook_dalloc)( + void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]); + +typedef void (*hook_expand)( + void *extra, hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); + +typedef struct hooks_s hooks_t; +struct hooks_s { + hook_alloc alloc_hook; + hook_dalloc dalloc_hook; + hook_expand expand_hook; + void *extra; +}; + +/* + * Begin implementation details; everything above this point might one day live + * in a public API. Everything below this point never will. + */ + +/* + * The realloc pathways haven't gotten any refactoring love in a while, and it's + * fairly difficult to pass information from the entry point to the hooks. We + * put the informaiton the hooks will need into a struct to encapsulate + * everything. + * + * Much of these pathways are force-inlined, so that the compiler can avoid + * materializing this struct until we hit an extern arena function. For fairly + * goofy reasons, *many* of the realloc paths hit an extern arena function. + * These paths are cold enough that it doesn't matter; eventually, we should + * rewrite the realloc code to make the expand-in-place and the + * free-then-realloc paths more orthogonal, at which point we don't need to + * spread the hook logic all over the place. + */ +typedef struct hook_ralloc_args_s hook_ralloc_args_t; +struct hook_ralloc_args_s { + /* I.e. as opposed to rallocx. */ + bool is_realloc; + /* + * The expand hook takes 4 arguments, even if only 3 are actually used; + * we add an extra one in case the user decides to memcpy without + * looking too closely at the hooked function. + */ + uintptr_t args[4]; +}; + +/* + * Returns an opaque handle to be used when removing the hook. NULL means that + * we couldn't install the hook. + */ +bool hook_boot(); + +void *hook_install(tsdn_t *tsdn, hooks_t *hooks); +/* Uninstalls the hook with the handle previously returned from hook_install. */ +void hook_remove(tsdn_t *tsdn, void *opaque); + +/* Hooks */ + +void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]); + +void hook_invoke_dalloc(hook_dalloc_t type, void *address, + uintptr_t args_raw[3]); + +void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); + +#endif /* JEMALLOC_INTERNAL_HOOK_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/hooks.h b/dep/jemalloc/include/jemalloc/internal/hooks.h deleted file mode 100644 index cd49afcb094..00000000000 --- a/dep/jemalloc/include/jemalloc/internal/hooks.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_HOOKS_H -#define JEMALLOC_INTERNAL_HOOKS_H - -extern JEMALLOC_EXPORT void (*hooks_arena_new_hook)(); -extern JEMALLOC_EXPORT void (*hooks_libc_hook)(); - -#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) - -#define open JEMALLOC_HOOK(open, hooks_libc_hook) -#define read JEMALLOC_HOOK(read, hooks_libc_hook) -#define write JEMALLOC_HOOK(write, hooks_libc_hook) -#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook) -#define close JEMALLOC_HOOK(close, hooks_libc_hook) -#define creat JEMALLOC_HOOK(creat, hooks_libc_hook) -#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook) -/* Note that this is undef'd and re-define'd in src/prof.c. */ -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) - -#endif /* JEMALLOC_INTERNAL_HOOKS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h index be70df510e3..7d6053e219e 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h @@ -31,6 +31,9 @@ # include <sys/uio.h> # endif # include <pthread.h> +# ifdef __FreeBSD__ +# include <pthread_np.h> +# endif # include <signal.h> # ifdef JEMALLOC_OS_UNFAIR_LOCK # include <os/lock.h> diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h index e10fb275d40..b784362338b 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h @@ -2,7 +2,6 @@ #define JEMALLOC_INTERNAL_EXTERNS_H #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/tsd_types.h" /* TSD checks this to set thread local slow state accordingly. */ @@ -25,6 +24,9 @@ extern unsigned ncpus; /* Number of arenas used for automatic multiplexing of threads and arenas. */ extern unsigned narenas_auto; +/* Base index for manual arenas. */ +extern unsigned manual_arena_base; + /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h index c6a1f7eb2ca..ddde9b4e63e 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -4,13 +4,15 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/ticker.h" JEMALLOC_ALWAYS_INLINE malloc_cpuid_t malloc_getcpu(void) { assert(have_percpu_arena); -#if defined(JEMALLOC_HAVE_SCHED_GETCPU) +#if defined(_WIN32) + return GetCurrentProcessorNumber(); +#elif defined(JEMALLOC_HAVE_SCHED_GETCPU) return (malloc_cpuid_t)sched_getcpu(); #else not_reached(); @@ -108,14 +110,14 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_small_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind < NBINS); + assert(binind < SC_NBINS); return &tcache->bins_small[binind]; } JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_large_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind >= NBINS &&binind < nhbins); - return &tcache->bins_large[binind - NBINS]; + assert(binind >= SC_NBINS &&binind < nhbins); + return &tcache->bins_large[binind - SC_NBINS]; } JEMALLOC_ALWAYS_INLINE bool @@ -156,7 +158,7 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) { if (fast) { /* Prepare slow path for reentrancy. */ tsd_slow_update(tsd); - assert(tsd->state == tsd_state_nominal_slow); + assert(tsd_state_get(tsd) == tsd_state_nominal_slow); } } diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h index 2e76e5d8f7c..70d6e578857 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -71,7 +71,8 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) { static inline bool arena_is_auto(arena_t *arena) { assert(narenas_auto > 0); - return (arena_ind_get(arena) < narenas_auto); + + return (arena_ind_get(arena) < manual_arena_base); } JEMALLOC_ALWAYS_INLINE extent_t * diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index c829ac60cbe..cdb10eb21f7 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -1,6 +1,7 @@ #ifndef JEMALLOC_INTERNAL_INLINES_C_H #define JEMALLOC_INTERNAL_INLINES_C_H +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/witness.h" @@ -42,7 +43,6 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path) { void *ret; - assert(size != 0); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { @@ -133,31 +133,20 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache, - arena_t *arena) { + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, + hook_ralloc_args_t *hook_args) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); void *p; size_t usize, copysize; - usize = sz_sa2u(size + extra, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + usize = sz_sa2u(size, alignment); + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); if (p == NULL) { - if (extra == 0) { - return NULL; - } - /* Try again, without extra this time. */ - usize = sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { - return NULL; - } - p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); - if (p == NULL) { - return NULL; - } + return NULL; } /* * Copy at most size bytes (not size+extra), since the caller has no @@ -165,13 +154,26 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return p; } +/* + * is_realloc threads through the knowledge of whether or not this call comes + * from je_realloc (as opposed to je_rallocx); this ensures that we pass the + * correct entry point into any hooks. + * Note that these functions are all force-inlined, so no actual bool gets + * passed-around anywhere. + */ JEMALLOC_ALWAYS_INLINE void * iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, tcache_t *tcache, arena_t *arena) { + bool zero, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args) +{ assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -183,24 +185,24 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, * Existing object alignment is inadequate; allocate new space * and copy. */ - return iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, - zero, tcache, arena); + return iralloct_realign(tsdn, ptr, oldsize, size, alignment, + zero, tcache, arena, hook_args); } return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, - tcache); + tcache, hook_args); } JEMALLOC_ALWAYS_INLINE void * iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero) { + bool zero, hook_ralloc_args_t *hook_args) { return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd), NULL); + tcache_get(tsd), NULL, hook_args); } JEMALLOC_ALWAYS_INLINE bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) { + size_t alignment, bool zero, size_t *newsize) { assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -209,10 +211,12 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { /* Existing object alignment is inadequate. */ + *newsize = oldsize; return true; } - return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); + return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero, + newsize); } #endif /* JEMALLOC_INTERNAL_INLINES_C_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h index ed75d3768e5..d8ea06f6d06 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -30,7 +30,7 @@ # define restrict #endif -/* Various function pointers are statick and immutable except during testing. */ +/* Various function pointers are static and immutable except during testing. */ #ifdef JEMALLOC_JET # define JET_MUTABLE #else @@ -40,4 +40,75 @@ #define JEMALLOC_VA_ARGS_HEAD(head, ...) head #define JEMALLOC_VA_ARGS_TAIL(head, ...) __VA_ARGS__ +#if (defined(__GNUC__) || defined(__GNUG__)) && !defined(__clang__) \ + && defined(JEMALLOC_HAVE_ATTR) && (__GNUC__ >= 7) +#define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough); +#else +#define JEMALLOC_FALLTHROUGH /* falls through */ +#endif + +/* Diagnostic suppression macros */ +#if defined(_MSC_VER) && !defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) +# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W)) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +/* #pragma GCC diagnostic first appeared in gcc 4.6. */ +#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \ + (__GNUC_MINOR__ > 5)))) || defined(__clang__) +/* + * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang + * diagnostic suppression macros and should not be used anywhere else. + */ +# define JEMALLOC_PRAGMA__(X) _Pragma(#X) +# define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push) +# define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) \ + JEMALLOC_PRAGMA__(GCC diagnostic ignored W) + +/* + * The -Wmissing-field-initializers warning is buggy in GCC versions < 5.1 and + * all clang versions up to version 7 (currently trunk, unreleased). This macro + * suppresses the warning for the affected compiler versions only. + */ +# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \ + defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# endif + +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits") +# define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter") +# if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# endif +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \ + JEMALLOC_DIAGNOSTIC_PUSH \ + JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER +#else +# define JEMALLOC_DIAGNOSTIC_PUSH +# define JEMALLOC_DIAGNOSTIC_POP +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +#endif + +/* + * Disables spurious diagnostics for all headers. Since these headers are not + * included by users directly, it does not affect their diagnostic settings. + */ +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + #endif /* JEMALLOC_INTERNAL_MACROS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h index 1b750b122c9..e296c5a7e84 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_TYPES_H #define JEMALLOC_INTERNAL_TYPES_H +#include "jemalloc/internal/quantum.h" + /* Page size index type. */ typedef unsigned pszind_t; @@ -50,79 +52,6 @@ typedef int malloc_cpuid_t; /* Smallest size class to support. */ #define TINY_MIN (1U << LG_TINY_MIN) -/* - * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size - * classes). - */ -#ifndef LG_QUANTUM -# if (defined(__i386__) || defined(_M_IX86)) -# define LG_QUANTUM 4 -# endif -# ifdef __ia64__ -# define LG_QUANTUM 4 -# endif -# ifdef __alpha__ -# define LG_QUANTUM 4 -# endif -# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) -# define LG_QUANTUM 4 -# endif -# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -# define LG_QUANTUM 4 -# endif -# ifdef __arm__ -# define LG_QUANTUM 3 -# endif -# ifdef __aarch64__ -# define LG_QUANTUM 4 -# endif -# ifdef __hppa__ -# define LG_QUANTUM 4 -# endif -# ifdef __m68k__ -# define LG_QUANTUM 3 -# endif -# ifdef __mips__ -# define LG_QUANTUM 3 -# endif -# ifdef __nios2__ -# define LG_QUANTUM 3 -# endif -# ifdef __or1k__ -# define LG_QUANTUM 3 -# endif -# ifdef __powerpc__ -# define LG_QUANTUM 4 -# endif -# if defined(__riscv) || defined(__riscv__) -# define LG_QUANTUM 4 -# endif -# ifdef __s390__ -# define LG_QUANTUM 4 -# endif -# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ - defined(__SH4_SINGLE_ONLY__)) -# define LG_QUANTUM 4 -# endif -# ifdef __tile__ -# define LG_QUANTUM 4 -# endif -# ifdef __le32__ -# define LG_QUANTUM 4 -# endif -# ifndef LG_QUANTUM -# error "Unknown minimum alignment for architecture; specify via " - "--with-lg-quantum" -# endif -#endif - -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) - -/* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) - #define LONG ((size_t)(1U << LG_SIZEOF_LONG)) #define LONG_MASK (LONG - 1) diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h b/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h index a373a8b2d3e..2e0e741ddec 100644 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h +++ b/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h @@ -21,7 +21,7 @@ # include "../jemalloc.h" #endif -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#if defined(JEMALLOC_OSATOMIC) #include <libkern/OSAtomic.h> #endif @@ -45,7 +45,7 @@ # include "jemalloc/internal/private_namespace_jet.h" # endif #endif -#include "jemalloc/internal/hooks.h" +#include "jemalloc/internal/test_hooks.h" #ifdef JEMALLOC_DEFINE_MADVISE_FREE # define JEMALLOC_MADV_FREE 8 @@ -161,7 +161,7 @@ static const bool config_log = false #endif ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU +#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in b/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in deleted file mode 100644 index e621fbc8599..00000000000 --- a/dep/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in +++ /dev/null @@ -1,194 +0,0 @@ -#ifndef JEMALLOC_PREAMBLE_H -#define JEMALLOC_PREAMBLE_H - -#include "jemalloc_internal_defs.h" -#include "jemalloc/internal/jemalloc_internal_decls.h" - -#ifdef JEMALLOC_UTRACE -#include <sys/ktrace.h> -#endif - -#define JEMALLOC_NO_DEMANGLE -#ifdef JEMALLOC_JET -# undef JEMALLOC_IS_MALLOC -# define JEMALLOC_N(n) jet_##n -# include "jemalloc/internal/public_namespace.h" -# define JEMALLOC_NO_RENAME -# include "../jemalloc@install_suffix@.h" -# undef JEMALLOC_NO_RENAME -#else -# define JEMALLOC_N(n) @private_namespace@##n -# include "../jemalloc@install_suffix@.h" -#endif - -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) -#include <libkern/OSAtomic.h> -#endif - -#ifdef JEMALLOC_ZONE -#include <mach/mach_error.h> -#include <mach/mach_init.h> -#include <mach/vm_map.h> -#endif - -#include "jemalloc/internal/jemalloc_internal_macros.h" - -/* - * Note that the ordering matters here; the hook itself is name-mangled. We - * want the inclusion of hooks to happen early, so that we hook as much as - * possible. - */ -#ifndef JEMALLOC_NO_PRIVATE_NAMESPACE -# ifndef JEMALLOC_JET -# include "jemalloc/internal/private_namespace.h" -# else -# include "jemalloc/internal/private_namespace_jet.h" -# endif -#endif -#include "jemalloc/internal/hooks.h" - -#ifdef JEMALLOC_DEFINE_MADVISE_FREE -# define JEMALLOC_MADV_FREE 8 -#endif - -static const bool config_debug = -#ifdef JEMALLOC_DEBUG - true -#else - false -#endif - ; -static const bool have_dss = -#ifdef JEMALLOC_DSS - true -#else - false -#endif - ; -static const bool have_madvise_huge = -#ifdef JEMALLOC_HAVE_MADVISE_HUGE - true -#else - false -#endif - ; -static const bool config_fill = -#ifdef JEMALLOC_FILL - true -#else - false -#endif - ; -static const bool config_lazy_lock = -#ifdef JEMALLOC_LAZY_LOCK - true -#else - false -#endif - ; -static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; -static const bool config_prof = -#ifdef JEMALLOC_PROF - true -#else - false -#endif - ; -static const bool config_prof_libgcc = -#ifdef JEMALLOC_PROF_LIBGCC - true -#else - false -#endif - ; -static const bool config_prof_libunwind = -#ifdef JEMALLOC_PROF_LIBUNWIND - true -#else - false -#endif - ; -static const bool maps_coalesce = -#ifdef JEMALLOC_MAPS_COALESCE - true -#else - false -#endif - ; -static const bool config_stats = -#ifdef JEMALLOC_STATS - true -#else - false -#endif - ; -static const bool config_tls = -#ifdef JEMALLOC_TLS - true -#else - false -#endif - ; -static const bool config_utrace = -#ifdef JEMALLOC_UTRACE - true -#else - false -#endif - ; -static const bool config_xmalloc = -#ifdef JEMALLOC_XMALLOC - true -#else - false -#endif - ; -static const bool config_cache_oblivious = -#ifdef JEMALLOC_CACHE_OBLIVIOUS - true -#else - false -#endif - ; -/* - * Undocumented, for jemalloc development use only at the moment. See the note - * in jemalloc/internal/log.h. - */ -static const bool config_log = -#ifdef JEMALLOC_LOG - true -#else - false -#endif - ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU -/* Currently percpu_arena depends on sched_getcpu. */ -#define JEMALLOC_PERCPU_ARENA -#endif -static const bool have_percpu_arena = -#ifdef JEMALLOC_PERCPU_ARENA - true -#else - false -#endif - ; -/* - * Undocumented, and not recommended; the application should take full - * responsibility for tracking provenance. - */ -static const bool force_ivsalloc = -#ifdef JEMALLOC_FORCE_IVSALLOC - true -#else - false -#endif - ; -static const bool have_background_thread = -#ifdef JEMALLOC_BACKGROUND_THREAD - true -#else - false -#endif - ; - -#endif /* JEMALLOC_PREAMBLE_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/large_externs.h b/dep/jemalloc/include/jemalloc/internal/large_externs.h index 3f36282cd40..a05019e8a54 100644 --- a/dep/jemalloc/include/jemalloc/internal/large_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/large_externs.h @@ -1,13 +1,16 @@ #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H +#include "jemalloc/internal/hook.h" + void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t usize_max, bool zero); -void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, - size_t alignment, bool zero, tcache_t *tcache); +void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, + size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); typedef void (large_dalloc_junk_t)(void *, size_t); extern large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk; @@ -23,4 +26,7 @@ prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent); void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx); void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent); +nstime_t large_prof_alloc_time_get(const extent_t *extent); +void large_prof_alloc_time_set(extent_t *extent, nstime_t time); + #endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/mutex.h b/dep/jemalloc/include/jemalloc/internal/mutex.h index 6520c2512d3..7c24f0725ee 100644 --- a/dep/jemalloc/include/jemalloc/internal/mutex.h +++ b/dep/jemalloc/include/jemalloc/internal/mutex.h @@ -37,14 +37,17 @@ struct malloc_mutex_s { # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLock lock; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) pthread_mutex_t lock; malloc_mutex_t *postponed_next; #else pthread_mutex_t lock; #endif + /* + * Hint flag to avoid exclusive cache line contention + * during spin waiting + */ + atomic_b_t locked; }; /* * We only touch witness when configured w/ debug. However we @@ -84,10 +87,6 @@ struct malloc_mutex_s { # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) -#elif (defined(JEMALLOC_OSSPIN)) -# define MALLOC_MUTEX_LOCK(m) OSSpinLockLock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) OSSpinLockUnlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!OSSpinLockTry(&(m)->lock)) #else # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) @@ -101,22 +100,37 @@ struct malloc_mutex_s { #ifdef _WIN32 # define MALLOC_MUTEX_INITIALIZER #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -#elif (defined(JEMALLOC_OSSPIN)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, 0}}, \ +# if defined(JEMALLOC_DEBUG) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif #elif (defined(JEMALLOC_MUTEX_INIT_CB)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# if (defined(JEMALLOC_DEBUG)) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif + #else # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT +# if defined(JEMALLOC_DEBUG) # define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif #endif #ifdef JEMALLOC_LAZY_LOCK @@ -139,6 +153,7 @@ void malloc_mutex_lock_slow(malloc_mutex_t *mutex); static inline void malloc_mutex_lock_final(malloc_mutex_t *mutex) { MALLOC_MUTEX_LOCK(mutex); + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } static inline bool @@ -164,6 +179,7 @@ malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); return true; } mutex_owner_stats_update(tsdn, mutex); @@ -203,6 +219,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { malloc_mutex_lock_slow(mutex); + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } mutex_owner_stats_update(tsdn, mutex); } @@ -211,6 +228,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { + atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { MALLOC_MUTEX_UNLOCK(mutex); @@ -245,4 +263,26 @@ malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); } +static inline void +malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, + malloc_mutex_t *mutex) { + mutex_prof_data_t *source = &mutex->prof_data; + /* Can only read holding the mutex. */ + malloc_mutex_assert_owner(tsdn, mutex); + + nstime_add(&data->tot_wait_time, &source->tot_wait_time); + if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { + nstime_copy(&data->max_wait_time, &source->max_wait_time); + } + data->n_wait_times += source->n_wait_times; + data->n_spin_acquired += source->n_spin_acquired; + if (data->max_n_thds < source->max_n_thds) { + data->max_n_thds = source->max_n_thds; + } + /* n_wait_thds is not reported. */ + atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); + data->n_owner_switches += source->n_owner_switches; + data->n_lock_ops += source->n_lock_ops; +} + #endif /* JEMALLOC_INTERNAL_MUTEX_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/mutex_prof.h b/dep/jemalloc/include/jemalloc/internal/mutex_prof.h index ce183d33529..2cb8fb0cbf7 100644 --- a/dep/jemalloc/include/jemalloc/internal/mutex_prof.h +++ b/dep/jemalloc/include/jemalloc/internal/mutex_prof.h @@ -35,22 +35,31 @@ typedef enum { mutex_prof_num_arena_mutexes } mutex_prof_arena_ind_t; +/* + * The forth parameter is a boolean value that is true for derived rate counters + * and false for real ones. + */ #define MUTEX_PROF_UINT64_COUNTERS \ - OP(num_ops, uint64_t, "n_lock_ops") \ - OP(num_wait, uint64_t, "n_waiting") \ - OP(num_spin_acq, uint64_t, "n_spin_acq") \ - OP(num_owner_switch, uint64_t, "n_owner_switch") \ - OP(total_wait_time, uint64_t, "total_wait_ns") \ - OP(max_wait_time, uint64_t, "max_wait_ns") + OP(num_ops, uint64_t, "n_lock_ops", false, num_ops) \ + OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops) \ + OP(num_wait, uint64_t, "n_waiting", false, num_wait) \ + OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ + OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ + OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ + OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch) \ + OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ + OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ + OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ + OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time) #define MUTEX_PROF_UINT32_COUNTERS \ - OP(max_num_thds, uint32_t, "max_n_thds") + OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds) #define MUTEX_PROF_COUNTERS \ MUTEX_PROF_UINT64_COUNTERS \ MUTEX_PROF_UINT32_COUNTERS -#define OP(counter, type, human) mutex_counter_##counter, +#define OP(counter, type, human, derived, base_counter) mutex_counter_##counter, #define COUNTER_ENUM(counter_list, t) \ typedef enum { \ diff --git a/dep/jemalloc/include/jemalloc/internal/private_namespace.h b/dep/jemalloc/include/jemalloc/internal/private_namespace.h index 7ebeeba8269..77282e72f83 100644 --- a/dep/jemalloc/include/jemalloc/internal/private_namespace.h +++ b/dep/jemalloc/include/jemalloc/internal/private_namespace.h @@ -1,639 +1,416 @@ -#define a0dalloc JEMALLOC_N(a0dalloc) -#define a0get JEMALLOC_N(a0get) -#define a0malloc JEMALLOC_N(a0malloc) -#define arena_aalloc JEMALLOC_N(arena_aalloc) -#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small) -#define arena_basic_stats_merge JEMALLOC_N(arena_basic_stats_merge) -#define arena_bin_index JEMALLOC_N(arena_bin_index) -#define arena_bin_info JEMALLOC_N(arena_bin_info) -#define arena_bitselm_get_const JEMALLOC_N(arena_bitselm_get_const) -#define arena_bitselm_get_mutable JEMALLOC_N(arena_bitselm_get_mutable) -#define arena_boot JEMALLOC_N(arena_boot) -#define arena_choose JEMALLOC_N(arena_choose) -#define arena_choose_hard JEMALLOC_N(arena_choose_hard) -#define arena_choose_impl JEMALLOC_N(arena_choose_impl) -#define arena_chunk_alloc_huge JEMALLOC_N(arena_chunk_alloc_huge) -#define arena_chunk_cache_maybe_insert JEMALLOC_N(arena_chunk_cache_maybe_insert) -#define arena_chunk_cache_maybe_remove JEMALLOC_N(arena_chunk_cache_maybe_remove) -#define arena_chunk_dalloc_huge JEMALLOC_N(arena_chunk_dalloc_huge) -#define arena_chunk_ralloc_huge_expand JEMALLOC_N(arena_chunk_ralloc_huge_expand) -#define arena_chunk_ralloc_huge_shrink JEMALLOC_N(arena_chunk_ralloc_huge_shrink) -#define arena_chunk_ralloc_huge_similar JEMALLOC_N(arena_chunk_ralloc_huge_similar) -#define arena_cleanup JEMALLOC_N(arena_cleanup) -#define arena_dalloc JEMALLOC_N(arena_dalloc) -#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) -#define arena_dalloc_bin_junked_locked JEMALLOC_N(arena_dalloc_bin_junked_locked) -#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) -#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) -#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) -#define arena_dalloc_large_junked_locked JEMALLOC_N(arena_dalloc_large_junked_locked) -#define arena_dalloc_small JEMALLOC_N(arena_dalloc_small) -#define arena_decay_tick JEMALLOC_N(arena_decay_tick) -#define arena_decay_ticks JEMALLOC_N(arena_decay_ticks) -#define arena_decay_time_default_get JEMALLOC_N(arena_decay_time_default_get) -#define arena_decay_time_default_set JEMALLOC_N(arena_decay_time_default_set) -#define arena_decay_time_get JEMALLOC_N(arena_decay_time_get) -#define arena_decay_time_set JEMALLOC_N(arena_decay_time_set) -#define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) -#define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) -#define arena_extent_sn_next JEMALLOC_N(arena_extent_sn_next) -#define arena_get JEMALLOC_N(arena_get) -#define arena_ichoose JEMALLOC_N(arena_ichoose) -#define arena_init JEMALLOC_N(arena_init) -#define arena_lg_dirty_mult_default_get JEMALLOC_N(arena_lg_dirty_mult_default_get) -#define arena_lg_dirty_mult_default_set JEMALLOC_N(arena_lg_dirty_mult_default_set) -#define arena_lg_dirty_mult_get JEMALLOC_N(arena_lg_dirty_mult_get) -#define arena_lg_dirty_mult_set JEMALLOC_N(arena_lg_dirty_mult_set) -#define arena_malloc JEMALLOC_N(arena_malloc) -#define arena_malloc_hard JEMALLOC_N(arena_malloc_hard) -#define arena_malloc_large JEMALLOC_N(arena_malloc_large) -#define arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get) -#define arena_mapbits_binind_get JEMALLOC_N(arena_mapbits_binind_get) -#define arena_mapbits_decommitted_get JEMALLOC_N(arena_mapbits_decommitted_get) -#define arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get) -#define arena_mapbits_get JEMALLOC_N(arena_mapbits_get) -#define arena_mapbits_internal_set JEMALLOC_N(arena_mapbits_internal_set) -#define arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set) -#define arena_mapbits_large_get JEMALLOC_N(arena_mapbits_large_get) -#define arena_mapbits_large_set JEMALLOC_N(arena_mapbits_large_set) -#define arena_mapbits_large_size_get JEMALLOC_N(arena_mapbits_large_size_get) -#define arena_mapbits_size_decode JEMALLOC_N(arena_mapbits_size_decode) -#define arena_mapbits_size_encode JEMALLOC_N(arena_mapbits_size_encode) -#define arena_mapbits_small_runind_get JEMALLOC_N(arena_mapbits_small_runind_get) -#define arena_mapbits_small_set JEMALLOC_N(arena_mapbits_small_set) -#define arena_mapbits_unallocated_set JEMALLOC_N(arena_mapbits_unallocated_set) -#define arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get) -#define arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set) -#define arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get) -#define arena_mapbitsp_get_const JEMALLOC_N(arena_mapbitsp_get_const) -#define arena_mapbitsp_get_mutable JEMALLOC_N(arena_mapbitsp_get_mutable) -#define arena_mapbitsp_read JEMALLOC_N(arena_mapbitsp_read) -#define arena_mapbitsp_write JEMALLOC_N(arena_mapbitsp_write) -#define arena_maxrun JEMALLOC_N(arena_maxrun) -#define arena_maybe_purge JEMALLOC_N(arena_maybe_purge) -#define arena_metadata_allocated_add JEMALLOC_N(arena_metadata_allocated_add) -#define arena_metadata_allocated_get JEMALLOC_N(arena_metadata_allocated_get) -#define arena_metadata_allocated_sub JEMALLOC_N(arena_metadata_allocated_sub) -#define arena_migrate JEMALLOC_N(arena_migrate) -#define arena_miscelm_get_const JEMALLOC_N(arena_miscelm_get_const) -#define arena_miscelm_get_mutable JEMALLOC_N(arena_miscelm_get_mutable) -#define arena_miscelm_to_pageind JEMALLOC_N(arena_miscelm_to_pageind) -#define arena_miscelm_to_rpages JEMALLOC_N(arena_miscelm_to_rpages) -#define arena_new JEMALLOC_N(arena_new) -#define arena_node_alloc JEMALLOC_N(arena_node_alloc) -#define arena_node_dalloc JEMALLOC_N(arena_node_dalloc) -#define arena_nthreads_dec JEMALLOC_N(arena_nthreads_dec) -#define arena_nthreads_get JEMALLOC_N(arena_nthreads_get) -#define arena_nthreads_inc JEMALLOC_N(arena_nthreads_inc) -#define arena_palloc JEMALLOC_N(arena_palloc) -#define arena_postfork_child JEMALLOC_N(arena_postfork_child) -#define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) -#define arena_prefork0 JEMALLOC_N(arena_prefork0) -#define arena_prefork1 JEMALLOC_N(arena_prefork1) -#define arena_prefork2 JEMALLOC_N(arena_prefork2) -#define arena_prefork3 JEMALLOC_N(arena_prefork3) -#define arena_prof_accum JEMALLOC_N(arena_prof_accum) -#define arena_prof_accum_impl JEMALLOC_N(arena_prof_accum_impl) -#define arena_prof_accum_locked JEMALLOC_N(arena_prof_accum_locked) -#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) -#define arena_prof_tctx_get JEMALLOC_N(arena_prof_tctx_get) -#define arena_prof_tctx_reset JEMALLOC_N(arena_prof_tctx_reset) -#define arena_prof_tctx_set JEMALLOC_N(arena_prof_tctx_set) -#define arena_ptr_small_binind_get JEMALLOC_N(arena_ptr_small_binind_get) -#define arena_purge JEMALLOC_N(arena_purge) -#define arena_quarantine_junk_small JEMALLOC_N(arena_quarantine_junk_small) -#define arena_ralloc JEMALLOC_N(arena_ralloc) -#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) -#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) -#define arena_rd_to_miscelm JEMALLOC_N(arena_rd_to_miscelm) -#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) -#define arena_reset JEMALLOC_N(arena_reset) -#define arena_run_regind JEMALLOC_N(arena_run_regind) -#define arena_run_to_miscelm JEMALLOC_N(arena_run_to_miscelm) -#define arena_salloc JEMALLOC_N(arena_salloc) -#define arena_sdalloc JEMALLOC_N(arena_sdalloc) -#define arena_stats_merge JEMALLOC_N(arena_stats_merge) -#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) -#define arena_tdata_get JEMALLOC_N(arena_tdata_get) -#define arena_tdata_get_hard JEMALLOC_N(arena_tdata_get_hard) -#define arenas JEMALLOC_N(arenas) -#define arenas_tdata_bypass_cleanup JEMALLOC_N(arenas_tdata_bypass_cleanup) -#define arenas_tdata_cleanup JEMALLOC_N(arenas_tdata_cleanup) -#define atomic_add_p JEMALLOC_N(atomic_add_p) -#define atomic_add_u JEMALLOC_N(atomic_add_u) -#define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32) -#define atomic_add_uint64 JEMALLOC_N(atomic_add_uint64) -#define atomic_add_z JEMALLOC_N(atomic_add_z) -#define atomic_cas_p JEMALLOC_N(atomic_cas_p) -#define atomic_cas_u JEMALLOC_N(atomic_cas_u) -#define atomic_cas_uint32 JEMALLOC_N(atomic_cas_uint32) -#define atomic_cas_uint64 JEMALLOC_N(atomic_cas_uint64) -#define atomic_cas_z JEMALLOC_N(atomic_cas_z) -#define atomic_sub_p JEMALLOC_N(atomic_sub_p) -#define atomic_sub_u JEMALLOC_N(atomic_sub_u) -#define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) -#define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) -#define atomic_sub_z JEMALLOC_N(atomic_sub_z) -#define atomic_write_p JEMALLOC_N(atomic_write_p) -#define atomic_write_u JEMALLOC_N(atomic_write_u) -#define atomic_write_uint32 JEMALLOC_N(atomic_write_uint32) -#define atomic_write_uint64 JEMALLOC_N(atomic_write_uint64) -#define atomic_write_z JEMALLOC_N(atomic_write_z) -#define base_alloc JEMALLOC_N(base_alloc) -#define base_boot JEMALLOC_N(base_boot) -#define base_postfork_child JEMALLOC_N(base_postfork_child) -#define base_postfork_parent JEMALLOC_N(base_postfork_parent) -#define base_prefork JEMALLOC_N(base_prefork) -#define base_stats_get JEMALLOC_N(base_stats_get) -#define bitmap_full JEMALLOC_N(bitmap_full) -#define bitmap_get JEMALLOC_N(bitmap_get) -#define bitmap_info_init JEMALLOC_N(bitmap_info_init) -#define bitmap_init JEMALLOC_N(bitmap_init) -#define bitmap_set JEMALLOC_N(bitmap_set) -#define bitmap_sfu JEMALLOC_N(bitmap_sfu) -#define bitmap_size JEMALLOC_N(bitmap_size) -#define bitmap_unset JEMALLOC_N(bitmap_unset) -#define bootstrap_calloc JEMALLOC_N(bootstrap_calloc) -#define bootstrap_free JEMALLOC_N(bootstrap_free) -#define bootstrap_malloc JEMALLOC_N(bootstrap_malloc) -#define bt_init JEMALLOC_N(bt_init) -#define buferror JEMALLOC_N(buferror) -#define chunk_alloc_base JEMALLOC_N(chunk_alloc_base) -#define chunk_alloc_cache JEMALLOC_N(chunk_alloc_cache) -#define chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss) -#define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) -#define chunk_alloc_wrapper JEMALLOC_N(chunk_alloc_wrapper) -#define chunk_boot JEMALLOC_N(chunk_boot) -#define chunk_dalloc_cache JEMALLOC_N(chunk_dalloc_cache) -#define chunk_dalloc_mmap JEMALLOC_N(chunk_dalloc_mmap) -#define chunk_dalloc_wrapper JEMALLOC_N(chunk_dalloc_wrapper) -#define chunk_deregister JEMALLOC_N(chunk_deregister) -#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) -#define chunk_dss_mergeable JEMALLOC_N(chunk_dss_mergeable) -#define chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get) -#define chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set) -#define chunk_hooks_default JEMALLOC_N(chunk_hooks_default) -#define chunk_hooks_get JEMALLOC_N(chunk_hooks_get) -#define chunk_hooks_set JEMALLOC_N(chunk_hooks_set) -#define chunk_in_dss JEMALLOC_N(chunk_in_dss) -#define chunk_lookup JEMALLOC_N(chunk_lookup) -#define chunk_npages JEMALLOC_N(chunk_npages) -#define chunk_purge_wrapper JEMALLOC_N(chunk_purge_wrapper) -#define chunk_register JEMALLOC_N(chunk_register) -#define chunks_rtree JEMALLOC_N(chunks_rtree) -#define chunksize JEMALLOC_N(chunksize) -#define chunksize_mask JEMALLOC_N(chunksize_mask) -#define ckh_count JEMALLOC_N(ckh_count) -#define ckh_delete JEMALLOC_N(ckh_delete) -#define ckh_insert JEMALLOC_N(ckh_insert) -#define ckh_iter JEMALLOC_N(ckh_iter) -#define ckh_new JEMALLOC_N(ckh_new) -#define ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash) -#define ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp) -#define ckh_remove JEMALLOC_N(ckh_remove) -#define ckh_search JEMALLOC_N(ckh_search) -#define ckh_string_hash JEMALLOC_N(ckh_string_hash) -#define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) -#define ctl_boot JEMALLOC_N(ctl_boot) -#define ctl_bymib JEMALLOC_N(ctl_bymib) -#define ctl_byname JEMALLOC_N(ctl_byname) -#define ctl_nametomib JEMALLOC_N(ctl_nametomib) -#define ctl_postfork_child JEMALLOC_N(ctl_postfork_child) -#define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent) -#define ctl_prefork JEMALLOC_N(ctl_prefork) -#define decay_ticker_get JEMALLOC_N(decay_ticker_get) -#define dss_prec_names JEMALLOC_N(dss_prec_names) -#define extent_node_achunk_get JEMALLOC_N(extent_node_achunk_get) -#define extent_node_achunk_set JEMALLOC_N(extent_node_achunk_set) -#define extent_node_addr_get JEMALLOC_N(extent_node_addr_get) -#define extent_node_addr_set JEMALLOC_N(extent_node_addr_set) -#define extent_node_arena_get JEMALLOC_N(extent_node_arena_get) -#define extent_node_arena_set JEMALLOC_N(extent_node_arena_set) -#define extent_node_committed_get JEMALLOC_N(extent_node_committed_get) -#define extent_node_committed_set JEMALLOC_N(extent_node_committed_set) -#define extent_node_dirty_insert JEMALLOC_N(extent_node_dirty_insert) -#define extent_node_dirty_linkage_init JEMALLOC_N(extent_node_dirty_linkage_init) -#define extent_node_dirty_remove JEMALLOC_N(extent_node_dirty_remove) -#define extent_node_init JEMALLOC_N(extent_node_init) -#define extent_node_prof_tctx_get JEMALLOC_N(extent_node_prof_tctx_get) -#define extent_node_prof_tctx_set JEMALLOC_N(extent_node_prof_tctx_set) -#define extent_node_size_get JEMALLOC_N(extent_node_size_get) -#define extent_node_size_set JEMALLOC_N(extent_node_size_set) -#define extent_node_sn_get JEMALLOC_N(extent_node_sn_get) -#define extent_node_sn_set JEMALLOC_N(extent_node_sn_set) -#define extent_node_zeroed_get JEMALLOC_N(extent_node_zeroed_get) -#define extent_node_zeroed_set JEMALLOC_N(extent_node_zeroed_set) -#define extent_size_quantize_ceil JEMALLOC_N(extent_size_quantize_ceil) -#define extent_size_quantize_floor JEMALLOC_N(extent_size_quantize_floor) -#define extent_tree_ad_destroy JEMALLOC_N(extent_tree_ad_destroy) -#define extent_tree_ad_destroy_recurse JEMALLOC_N(extent_tree_ad_destroy_recurse) -#define extent_tree_ad_empty JEMALLOC_N(extent_tree_ad_empty) -#define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) -#define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) -#define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) -#define extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse) -#define extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start) -#define extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last) -#define extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new) -#define extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next) -#define extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch) -#define extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev) -#define extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch) -#define extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove) -#define extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter) -#define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) -#define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) -#define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) -#define extent_tree_szsnad_destroy JEMALLOC_N(extent_tree_szsnad_destroy) -#define extent_tree_szsnad_destroy_recurse JEMALLOC_N(extent_tree_szsnad_destroy_recurse) -#define extent_tree_szsnad_empty JEMALLOC_N(extent_tree_szsnad_empty) -#define extent_tree_szsnad_first JEMALLOC_N(extent_tree_szsnad_first) -#define extent_tree_szsnad_insert JEMALLOC_N(extent_tree_szsnad_insert) -#define extent_tree_szsnad_iter JEMALLOC_N(extent_tree_szsnad_iter) -#define extent_tree_szsnad_iter_recurse JEMALLOC_N(extent_tree_szsnad_iter_recurse) -#define extent_tree_szsnad_iter_start JEMALLOC_N(extent_tree_szsnad_iter_start) -#define extent_tree_szsnad_last JEMALLOC_N(extent_tree_szsnad_last) -#define extent_tree_szsnad_new JEMALLOC_N(extent_tree_szsnad_new) -#define extent_tree_szsnad_next JEMALLOC_N(extent_tree_szsnad_next) -#define extent_tree_szsnad_nsearch JEMALLOC_N(extent_tree_szsnad_nsearch) -#define extent_tree_szsnad_prev JEMALLOC_N(extent_tree_szsnad_prev) -#define extent_tree_szsnad_psearch JEMALLOC_N(extent_tree_szsnad_psearch) -#define extent_tree_szsnad_remove JEMALLOC_N(extent_tree_szsnad_remove) -#define extent_tree_szsnad_reverse_iter JEMALLOC_N(extent_tree_szsnad_reverse_iter) -#define extent_tree_szsnad_reverse_iter_recurse JEMALLOC_N(extent_tree_szsnad_reverse_iter_recurse) -#define extent_tree_szsnad_reverse_iter_start JEMALLOC_N(extent_tree_szsnad_reverse_iter_start) -#define extent_tree_szsnad_search JEMALLOC_N(extent_tree_szsnad_search) -#define ffs_llu JEMALLOC_N(ffs_llu) -#define ffs_lu JEMALLOC_N(ffs_lu) -#define ffs_u JEMALLOC_N(ffs_u) -#define ffs_u32 JEMALLOC_N(ffs_u32) -#define ffs_u64 JEMALLOC_N(ffs_u64) -#define ffs_zu JEMALLOC_N(ffs_zu) -#define get_errno JEMALLOC_N(get_errno) -#define hash JEMALLOC_N(hash) -#define hash_fmix_32 JEMALLOC_N(hash_fmix_32) -#define hash_fmix_64 JEMALLOC_N(hash_fmix_64) -#define hash_get_block_32 JEMALLOC_N(hash_get_block_32) -#define hash_get_block_64 JEMALLOC_N(hash_get_block_64) -#define hash_rotl_32 JEMALLOC_N(hash_rotl_32) -#define hash_rotl_64 JEMALLOC_N(hash_rotl_64) -#define hash_x64_128 JEMALLOC_N(hash_x64_128) -#define hash_x86_128 JEMALLOC_N(hash_x86_128) -#define hash_x86_32 JEMALLOC_N(hash_x86_32) -#define huge_aalloc JEMALLOC_N(huge_aalloc) -#define huge_dalloc JEMALLOC_N(huge_dalloc) -#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) -#define huge_malloc JEMALLOC_N(huge_malloc) -#define huge_palloc JEMALLOC_N(huge_palloc) -#define huge_prof_tctx_get JEMALLOC_N(huge_prof_tctx_get) -#define huge_prof_tctx_reset JEMALLOC_N(huge_prof_tctx_reset) -#define huge_prof_tctx_set JEMALLOC_N(huge_prof_tctx_set) -#define huge_ralloc JEMALLOC_N(huge_ralloc) -#define huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move) -#define huge_salloc JEMALLOC_N(huge_salloc) -#define iaalloc JEMALLOC_N(iaalloc) -#define ialloc JEMALLOC_N(ialloc) -#define iallocztm JEMALLOC_N(iallocztm) -#define iarena_cleanup JEMALLOC_N(iarena_cleanup) -#define idalloc JEMALLOC_N(idalloc) -#define idalloctm JEMALLOC_N(idalloctm) -#define in_valgrind JEMALLOC_N(in_valgrind) -#define index2size JEMALLOC_N(index2size) -#define index2size_compute JEMALLOC_N(index2size_compute) -#define index2size_lookup JEMALLOC_N(index2size_lookup) -#define index2size_tab JEMALLOC_N(index2size_tab) -#define ipalloc JEMALLOC_N(ipalloc) -#define ipalloct JEMALLOC_N(ipalloct) -#define ipallocztm JEMALLOC_N(ipallocztm) -#define iqalloc JEMALLOC_N(iqalloc) -#define iralloc JEMALLOC_N(iralloc) -#define iralloct JEMALLOC_N(iralloct) -#define iralloct_realign JEMALLOC_N(iralloct_realign) -#define isalloc JEMALLOC_N(isalloc) -#define isdalloct JEMALLOC_N(isdalloct) -#define isqalloc JEMALLOC_N(isqalloc) -#define isthreaded JEMALLOC_N(isthreaded) -#define ivsalloc JEMALLOC_N(ivsalloc) -#define ixalloc JEMALLOC_N(ixalloc) -#define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) -#define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) -#define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) -#define large_maxclass JEMALLOC_N(large_maxclass) -#define lg_floor JEMALLOC_N(lg_floor) -#define lg_prof_sample JEMALLOC_N(lg_prof_sample) -#define malloc_cprintf JEMALLOC_N(malloc_cprintf) -#define malloc_mutex_assert_not_owner JEMALLOC_N(malloc_mutex_assert_not_owner) -#define malloc_mutex_assert_owner JEMALLOC_N(malloc_mutex_assert_owner) -#define malloc_mutex_boot JEMALLOC_N(malloc_mutex_boot) -#define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) -#define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) -#define malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child) -#define malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent) -#define malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork) -#define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) -#define malloc_printf JEMALLOC_N(malloc_printf) -#define malloc_snprintf JEMALLOC_N(malloc_snprintf) -#define malloc_strtoumax JEMALLOC_N(malloc_strtoumax) -#define malloc_tsd_boot0 JEMALLOC_N(malloc_tsd_boot0) -#define malloc_tsd_boot1 JEMALLOC_N(malloc_tsd_boot1) -#define malloc_tsd_cleanup_register JEMALLOC_N(malloc_tsd_cleanup_register) -#define malloc_tsd_dalloc JEMALLOC_N(malloc_tsd_dalloc) -#define malloc_tsd_malloc JEMALLOC_N(malloc_tsd_malloc) -#define malloc_tsd_no_cleanup JEMALLOC_N(malloc_tsd_no_cleanup) -#define malloc_vcprintf JEMALLOC_N(malloc_vcprintf) -#define malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf) -#define malloc_write JEMALLOC_N(malloc_write) -#define map_bias JEMALLOC_N(map_bias) -#define map_misc_offset JEMALLOC_N(map_misc_offset) -#define mb_write JEMALLOC_N(mb_write) -#define narenas_auto JEMALLOC_N(narenas_auto) -#define narenas_tdata_cleanup JEMALLOC_N(narenas_tdata_cleanup) -#define narenas_total_get JEMALLOC_N(narenas_total_get) -#define ncpus JEMALLOC_N(ncpus) -#define nhbins JEMALLOC_N(nhbins) -#define nhclasses JEMALLOC_N(nhclasses) -#define nlclasses JEMALLOC_N(nlclasses) -#define nstime_add JEMALLOC_N(nstime_add) -#define nstime_compare JEMALLOC_N(nstime_compare) -#define nstime_copy JEMALLOC_N(nstime_copy) -#define nstime_divide JEMALLOC_N(nstime_divide) -#define nstime_idivide JEMALLOC_N(nstime_idivide) -#define nstime_imultiply JEMALLOC_N(nstime_imultiply) -#define nstime_init JEMALLOC_N(nstime_init) -#define nstime_init2 JEMALLOC_N(nstime_init2) -#define nstime_monotonic JEMALLOC_N(nstime_monotonic) -#define nstime_ns JEMALLOC_N(nstime_ns) -#define nstime_nsec JEMALLOC_N(nstime_nsec) -#define nstime_sec JEMALLOC_N(nstime_sec) -#define nstime_subtract JEMALLOC_N(nstime_subtract) -#define nstime_update JEMALLOC_N(nstime_update) -#define opt_abort JEMALLOC_N(opt_abort) -#define opt_decay_time JEMALLOC_N(opt_decay_time) -#define opt_dss JEMALLOC_N(opt_dss) -#define opt_junk JEMALLOC_N(opt_junk) -#define opt_junk_alloc JEMALLOC_N(opt_junk_alloc) -#define opt_junk_free JEMALLOC_N(opt_junk_free) -#define opt_lg_chunk JEMALLOC_N(opt_lg_chunk) -#define opt_lg_dirty_mult JEMALLOC_N(opt_lg_dirty_mult) -#define opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval) -#define opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample) -#define opt_lg_tcache_max JEMALLOC_N(opt_lg_tcache_max) -#define opt_narenas JEMALLOC_N(opt_narenas) -#define opt_prof JEMALLOC_N(opt_prof) -#define opt_prof_accum JEMALLOC_N(opt_prof_accum) -#define opt_prof_active JEMALLOC_N(opt_prof_active) -#define opt_prof_final JEMALLOC_N(opt_prof_final) -#define opt_prof_gdump JEMALLOC_N(opt_prof_gdump) -#define opt_prof_leak JEMALLOC_N(opt_prof_leak) -#define opt_prof_prefix JEMALLOC_N(opt_prof_prefix) -#define opt_prof_thread_active_init JEMALLOC_N(opt_prof_thread_active_init) -#define opt_purge JEMALLOC_N(opt_purge) -#define opt_quarantine JEMALLOC_N(opt_quarantine) -#define opt_redzone JEMALLOC_N(opt_redzone) -#define opt_stats_print JEMALLOC_N(opt_stats_print) -#define opt_tcache JEMALLOC_N(opt_tcache) -#define opt_thp JEMALLOC_N(opt_thp) -#define opt_utrace JEMALLOC_N(opt_utrace) -#define opt_xmalloc JEMALLOC_N(opt_xmalloc) -#define opt_zero JEMALLOC_N(opt_zero) -#define p2rz JEMALLOC_N(p2rz) -#define pages_boot JEMALLOC_N(pages_boot) -#define pages_commit JEMALLOC_N(pages_commit) -#define pages_decommit JEMALLOC_N(pages_decommit) -#define pages_huge JEMALLOC_N(pages_huge) -#define pages_map JEMALLOC_N(pages_map) -#define pages_nohuge JEMALLOC_N(pages_nohuge) -#define pages_purge JEMALLOC_N(pages_purge) -#define pages_trim JEMALLOC_N(pages_trim) -#define pages_unmap JEMALLOC_N(pages_unmap) -#define pind2sz JEMALLOC_N(pind2sz) -#define pind2sz_compute JEMALLOC_N(pind2sz_compute) -#define pind2sz_lookup JEMALLOC_N(pind2sz_lookup) -#define pind2sz_tab JEMALLOC_N(pind2sz_tab) -#define pow2_ceil_u32 JEMALLOC_N(pow2_ceil_u32) -#define pow2_ceil_u64 JEMALLOC_N(pow2_ceil_u64) -#define pow2_ceil_zu JEMALLOC_N(pow2_ceil_zu) -#define prng_lg_range_u32 JEMALLOC_N(prng_lg_range_u32) -#define prng_lg_range_u64 JEMALLOC_N(prng_lg_range_u64) -#define prng_lg_range_zu JEMALLOC_N(prng_lg_range_zu) -#define prng_range_u32 JEMALLOC_N(prng_range_u32) -#define prng_range_u64 JEMALLOC_N(prng_range_u64) -#define prng_range_zu JEMALLOC_N(prng_range_zu) -#define prng_state_next_u32 JEMALLOC_N(prng_state_next_u32) -#define prng_state_next_u64 JEMALLOC_N(prng_state_next_u64) -#define prng_state_next_zu JEMALLOC_N(prng_state_next_zu) -#define prof_active JEMALLOC_N(prof_active) -#define prof_active_get JEMALLOC_N(prof_active_get) -#define prof_active_get_unlocked JEMALLOC_N(prof_active_get_unlocked) -#define prof_active_set JEMALLOC_N(prof_active_set) -#define prof_alloc_prep JEMALLOC_N(prof_alloc_prep) -#define prof_alloc_rollback JEMALLOC_N(prof_alloc_rollback) -#define prof_backtrace JEMALLOC_N(prof_backtrace) -#define prof_boot0 JEMALLOC_N(prof_boot0) -#define prof_boot1 JEMALLOC_N(prof_boot1) -#define prof_boot2 JEMALLOC_N(prof_boot2) -#define prof_bt_count JEMALLOC_N(prof_bt_count) -#define prof_dump_header JEMALLOC_N(prof_dump_header) -#define prof_dump_open JEMALLOC_N(prof_dump_open) -#define prof_free JEMALLOC_N(prof_free) -#define prof_free_sampled_object JEMALLOC_N(prof_free_sampled_object) -#define prof_gdump JEMALLOC_N(prof_gdump) -#define prof_gdump_get JEMALLOC_N(prof_gdump_get) -#define prof_gdump_get_unlocked JEMALLOC_N(prof_gdump_get_unlocked) -#define prof_gdump_set JEMALLOC_N(prof_gdump_set) -#define prof_gdump_val JEMALLOC_N(prof_gdump_val) -#define prof_idump JEMALLOC_N(prof_idump) -#define prof_interval JEMALLOC_N(prof_interval) -#define prof_lookup JEMALLOC_N(prof_lookup) -#define prof_malloc JEMALLOC_N(prof_malloc) -#define prof_malloc_sample_object JEMALLOC_N(prof_malloc_sample_object) -#define prof_mdump JEMALLOC_N(prof_mdump) -#define prof_postfork_child JEMALLOC_N(prof_postfork_child) -#define prof_postfork_parent JEMALLOC_N(prof_postfork_parent) -#define prof_prefork0 JEMALLOC_N(prof_prefork0) -#define prof_prefork1 JEMALLOC_N(prof_prefork1) -#define prof_realloc JEMALLOC_N(prof_realloc) -#define prof_reset JEMALLOC_N(prof_reset) -#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) -#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) -#define prof_tctx_get JEMALLOC_N(prof_tctx_get) -#define prof_tctx_reset JEMALLOC_N(prof_tctx_reset) -#define prof_tctx_set JEMALLOC_N(prof_tctx_set) -#define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) -#define prof_tdata_count JEMALLOC_N(prof_tdata_count) -#define prof_tdata_get JEMALLOC_N(prof_tdata_get) -#define prof_tdata_init JEMALLOC_N(prof_tdata_init) -#define prof_tdata_reinit JEMALLOC_N(prof_tdata_reinit) -#define prof_thread_active_get JEMALLOC_N(prof_thread_active_get) -#define prof_thread_active_init_get JEMALLOC_N(prof_thread_active_init_get) -#define prof_thread_active_init_set JEMALLOC_N(prof_thread_active_init_set) -#define prof_thread_active_set JEMALLOC_N(prof_thread_active_set) -#define prof_thread_name_get JEMALLOC_N(prof_thread_name_get) -#define prof_thread_name_set JEMALLOC_N(prof_thread_name_set) -#define psz2ind JEMALLOC_N(psz2ind) -#define psz2u JEMALLOC_N(psz2u) -#define purge_mode_names JEMALLOC_N(purge_mode_names) -#define quarantine JEMALLOC_N(quarantine) -#define quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook) -#define quarantine_alloc_hook_work JEMALLOC_N(quarantine_alloc_hook_work) -#define quarantine_cleanup JEMALLOC_N(quarantine_cleanup) -#define rtree_child_read JEMALLOC_N(rtree_child_read) -#define rtree_child_read_hard JEMALLOC_N(rtree_child_read_hard) -#define rtree_child_tryread JEMALLOC_N(rtree_child_tryread) -#define rtree_delete JEMALLOC_N(rtree_delete) -#define rtree_get JEMALLOC_N(rtree_get) -#define rtree_new JEMALLOC_N(rtree_new) -#define rtree_node_valid JEMALLOC_N(rtree_node_valid) -#define rtree_set JEMALLOC_N(rtree_set) -#define rtree_start_level JEMALLOC_N(rtree_start_level) -#define rtree_subkey JEMALLOC_N(rtree_subkey) -#define rtree_subtree_read JEMALLOC_N(rtree_subtree_read) -#define rtree_subtree_read_hard JEMALLOC_N(rtree_subtree_read_hard) -#define rtree_subtree_tryread JEMALLOC_N(rtree_subtree_tryread) -#define rtree_val_read JEMALLOC_N(rtree_val_read) -#define rtree_val_write JEMALLOC_N(rtree_val_write) -#define run_quantize_ceil JEMALLOC_N(run_quantize_ceil) -#define run_quantize_floor JEMALLOC_N(run_quantize_floor) -#define s2u JEMALLOC_N(s2u) -#define s2u_compute JEMALLOC_N(s2u_compute) -#define s2u_lookup JEMALLOC_N(s2u_lookup) -#define sa2u JEMALLOC_N(sa2u) -#define set_errno JEMALLOC_N(set_errno) -#define size2index JEMALLOC_N(size2index) -#define size2index_compute JEMALLOC_N(size2index_compute) -#define size2index_lookup JEMALLOC_N(size2index_lookup) -#define size2index_tab JEMALLOC_N(size2index_tab) -#define spin_adaptive JEMALLOC_N(spin_adaptive) -#define spin_init JEMALLOC_N(spin_init) -#define stats_cactive JEMALLOC_N(stats_cactive) -#define stats_cactive_add JEMALLOC_N(stats_cactive_add) -#define stats_cactive_get JEMALLOC_N(stats_cactive_get) -#define stats_cactive_sub JEMALLOC_N(stats_cactive_sub) -#define stats_print JEMALLOC_N(stats_print) -#define tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy) -#define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) -#define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) -#define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) -#define tcache_arena_reassociate JEMALLOC_N(tcache_arena_reassociate) -#define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) -#define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) -#define tcache_bin_info JEMALLOC_N(tcache_bin_info) -#define tcache_boot JEMALLOC_N(tcache_boot) -#define tcache_cleanup JEMALLOC_N(tcache_cleanup) -#define tcache_create JEMALLOC_N(tcache_create) -#define tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large) -#define tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small) -#define tcache_enabled_cleanup JEMALLOC_N(tcache_enabled_cleanup) -#define tcache_enabled_get JEMALLOC_N(tcache_enabled_get) -#define tcache_enabled_set JEMALLOC_N(tcache_enabled_set) -#define tcache_event JEMALLOC_N(tcache_event) -#define tcache_event_hard JEMALLOC_N(tcache_event_hard) -#define tcache_flush JEMALLOC_N(tcache_flush) -#define tcache_get JEMALLOC_N(tcache_get) -#define tcache_get_hard JEMALLOC_N(tcache_get_hard) -#define tcache_maxclass JEMALLOC_N(tcache_maxclass) -#define tcache_postfork_child JEMALLOC_N(tcache_postfork_child) -#define tcache_postfork_parent JEMALLOC_N(tcache_postfork_parent) -#define tcache_prefork JEMALLOC_N(tcache_prefork) -#define tcache_salloc JEMALLOC_N(tcache_salloc) -#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) -#define tcaches JEMALLOC_N(tcaches) -#define tcaches_create JEMALLOC_N(tcaches_create) -#define tcaches_destroy JEMALLOC_N(tcaches_destroy) -#define tcaches_flush JEMALLOC_N(tcaches_flush) -#define tcaches_get JEMALLOC_N(tcaches_get) -#define thread_allocated_cleanup JEMALLOC_N(thread_allocated_cleanup) -#define thread_deallocated_cleanup JEMALLOC_N(thread_deallocated_cleanup) -#define ticker_copy JEMALLOC_N(ticker_copy) -#define ticker_init JEMALLOC_N(ticker_init) -#define ticker_read JEMALLOC_N(ticker_read) -#define ticker_tick JEMALLOC_N(ticker_tick) -#define ticker_ticks JEMALLOC_N(ticker_ticks) -#define tsd_arena_get JEMALLOC_N(tsd_arena_get) -#define tsd_arena_set JEMALLOC_N(tsd_arena_set) -#define tsd_arenap_get JEMALLOC_N(tsd_arenap_get) -#define tsd_arenas_tdata_bypass_get JEMALLOC_N(tsd_arenas_tdata_bypass_get) -#define tsd_arenas_tdata_bypass_set JEMALLOC_N(tsd_arenas_tdata_bypass_set) -#define tsd_arenas_tdata_bypassp_get JEMALLOC_N(tsd_arenas_tdata_bypassp_get) -#define tsd_arenas_tdata_get JEMALLOC_N(tsd_arenas_tdata_get) -#define tsd_arenas_tdata_set JEMALLOC_N(tsd_arenas_tdata_set) -#define tsd_arenas_tdatap_get JEMALLOC_N(tsd_arenas_tdatap_get) -#define tsd_boot JEMALLOC_N(tsd_boot) -#define tsd_boot0 JEMALLOC_N(tsd_boot0) -#define tsd_boot1 JEMALLOC_N(tsd_boot1) -#define tsd_booted JEMALLOC_N(tsd_booted) -#define tsd_booted_get JEMALLOC_N(tsd_booted_get) -#define tsd_cleanup JEMALLOC_N(tsd_cleanup) -#define tsd_cleanup_wrapper JEMALLOC_N(tsd_cleanup_wrapper) -#define tsd_fetch JEMALLOC_N(tsd_fetch) -#define tsd_fetch_impl JEMALLOC_N(tsd_fetch_impl) -#define tsd_get JEMALLOC_N(tsd_get) -#define tsd_get_allocates JEMALLOC_N(tsd_get_allocates) -#define tsd_iarena_get JEMALLOC_N(tsd_iarena_get) -#define tsd_iarena_set JEMALLOC_N(tsd_iarena_set) -#define tsd_iarenap_get JEMALLOC_N(tsd_iarenap_get) -#define tsd_initialized JEMALLOC_N(tsd_initialized) -#define tsd_init_check_recursion JEMALLOC_N(tsd_init_check_recursion) -#define tsd_init_finish JEMALLOC_N(tsd_init_finish) -#define tsd_init_head JEMALLOC_N(tsd_init_head) -#define tsd_narenas_tdata_get JEMALLOC_N(tsd_narenas_tdata_get) -#define tsd_narenas_tdata_set JEMALLOC_N(tsd_narenas_tdata_set) -#define tsd_narenas_tdatap_get JEMALLOC_N(tsd_narenas_tdatap_get) -#define tsd_wrapper_get JEMALLOC_N(tsd_wrapper_get) -#define tsd_wrapper_set JEMALLOC_N(tsd_wrapper_set) -#define tsd_nominal JEMALLOC_N(tsd_nominal) -#define tsd_prof_tdata_get JEMALLOC_N(tsd_prof_tdata_get) -#define tsd_prof_tdata_set JEMALLOC_N(tsd_prof_tdata_set) -#define tsd_prof_tdatap_get JEMALLOC_N(tsd_prof_tdatap_get) -#define tsd_quarantine_get JEMALLOC_N(tsd_quarantine_get) -#define tsd_quarantine_set JEMALLOC_N(tsd_quarantine_set) -#define tsd_quarantinep_get JEMALLOC_N(tsd_quarantinep_get) -#define tsd_set JEMALLOC_N(tsd_set) -#define tsd_tcache_enabled_get JEMALLOC_N(tsd_tcache_enabled_get) -#define tsd_tcache_enabled_set JEMALLOC_N(tsd_tcache_enabled_set) -#define tsd_tcache_enabledp_get JEMALLOC_N(tsd_tcache_enabledp_get) -#define tsd_tcache_get JEMALLOC_N(tsd_tcache_get) -#define tsd_tcache_set JEMALLOC_N(tsd_tcache_set) -#define tsd_tcachep_get JEMALLOC_N(tsd_tcachep_get) -#define tsd_thread_allocated_get JEMALLOC_N(tsd_thread_allocated_get) -#define tsd_thread_allocated_set JEMALLOC_N(tsd_thread_allocated_set) -#define tsd_thread_allocatedp_get JEMALLOC_N(tsd_thread_allocatedp_get) -#define tsd_thread_deallocated_get JEMALLOC_N(tsd_thread_deallocated_get) -#define tsd_thread_deallocated_set JEMALLOC_N(tsd_thread_deallocated_set) -#define tsd_thread_deallocatedp_get JEMALLOC_N(tsd_thread_deallocatedp_get) -#define tsd_tls JEMALLOC_N(tsd_tls) -#define tsd_tsd JEMALLOC_N(tsd_tsd) -#define tsd_tsdn JEMALLOC_N(tsd_tsdn) -#define tsd_witness_fork_get JEMALLOC_N(tsd_witness_fork_get) -#define tsd_witness_fork_set JEMALLOC_N(tsd_witness_fork_set) -#define tsd_witness_forkp_get JEMALLOC_N(tsd_witness_forkp_get) -#define tsd_witnesses_get JEMALLOC_N(tsd_witnesses_get) -#define tsd_witnesses_set JEMALLOC_N(tsd_witnesses_set) -#define tsd_witnessesp_get JEMALLOC_N(tsd_witnessesp_get) -#define tsdn_fetch JEMALLOC_N(tsdn_fetch) -#define tsdn_null JEMALLOC_N(tsdn_null) -#define tsdn_tsd JEMALLOC_N(tsdn_tsd) -#define u2rz JEMALLOC_N(u2rz) -#define valgrind_freelike_block JEMALLOC_N(valgrind_freelike_block) -#define valgrind_make_mem_defined JEMALLOC_N(valgrind_make_mem_defined) -#define valgrind_make_mem_noaccess JEMALLOC_N(valgrind_make_mem_noaccess) -#define valgrind_make_mem_undefined JEMALLOC_N(valgrind_make_mem_undefined) -#define witness_assert_depth JEMALLOC_N(witness_assert_depth) -#define witness_assert_depth_to_rank JEMALLOC_N(witness_assert_depth_to_rank) -#define witness_assert_lockless JEMALLOC_N(witness_assert_lockless) -#define witness_assert_not_owner JEMALLOC_N(witness_assert_not_owner) -#define witness_assert_owner JEMALLOC_N(witness_assert_owner) -#define witness_depth_error JEMALLOC_N(witness_depth_error) -#define witness_fork_cleanup JEMALLOC_N(witness_fork_cleanup) -#define witness_init JEMALLOC_N(witness_init) -#define witness_lock JEMALLOC_N(witness_lock) -#define witness_lock_error JEMALLOC_N(witness_lock_error) -#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error) -#define witness_owner JEMALLOC_N(witness_owner) -#define witness_owner_error JEMALLOC_N(witness_owner_error) -#define witness_postfork_child JEMALLOC_N(witness_postfork_child) -#define witness_postfork_parent JEMALLOC_N(witness_postfork_parent) -#define witness_prefork JEMALLOC_N(witness_prefork) -#define witness_unlock JEMALLOC_N(witness_unlock) -#define witnesses_cleanup JEMALLOC_N(witnesses_cleanup) -#define zone_register JEMALLOC_N(zone_register) +#define a0dalloc JEMALLOC_N(a0dalloc) +#define a0malloc JEMALLOC_N(a0malloc) +#define arena_choose_hard JEMALLOC_N(arena_choose_hard) +#define arena_cleanup JEMALLOC_N(arena_cleanup) +#define arena_init JEMALLOC_N(arena_init) +#define arena_migrate JEMALLOC_N(arena_migrate) +#define arenas JEMALLOC_N(arenas) +#define arena_set JEMALLOC_N(arena_set) +#define arenas_lock JEMALLOC_N(arenas_lock) +#define arenas_tdata_cleanup JEMALLOC_N(arenas_tdata_cleanup) +#define arena_tdata_get_hard JEMALLOC_N(arena_tdata_get_hard) +#define bootstrap_calloc JEMALLOC_N(bootstrap_calloc) +#define bootstrap_free JEMALLOC_N(bootstrap_free) +#define bootstrap_malloc JEMALLOC_N(bootstrap_malloc) +#define free_default JEMALLOC_N(free_default) +#define iarena_cleanup JEMALLOC_N(iarena_cleanup) +#define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) +#define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) +#define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) +#define malloc_default JEMALLOC_N(malloc_default) +#define malloc_initialized JEMALLOC_N(malloc_initialized) +#define malloc_slow JEMALLOC_N(malloc_slow) +#define manual_arena_base JEMALLOC_N(manual_arena_base) +#define narenas_auto JEMALLOC_N(narenas_auto) +#define narenas_total_get JEMALLOC_N(narenas_total_get) +#define ncpus JEMALLOC_N(ncpus) +#define opt_abort JEMALLOC_N(opt_abort) +#define opt_abort_conf JEMALLOC_N(opt_abort_conf) +#define opt_junk JEMALLOC_N(opt_junk) +#define opt_junk_alloc JEMALLOC_N(opt_junk_alloc) +#define opt_junk_free JEMALLOC_N(opt_junk_free) +#define opt_narenas JEMALLOC_N(opt_narenas) +#define opt_utrace JEMALLOC_N(opt_utrace) +#define opt_xmalloc JEMALLOC_N(opt_xmalloc) +#define opt_zero JEMALLOC_N(opt_zero) +#define sdallocx_default JEMALLOC_N(sdallocx_default) +#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small) +#define arena_basic_stats_merge JEMALLOC_N(arena_basic_stats_merge) +#define arena_bin_choose_lock JEMALLOC_N(arena_bin_choose_lock) +#define arena_boot JEMALLOC_N(arena_boot) +#define arena_choose_huge JEMALLOC_N(arena_choose_huge) +#define arena_dalloc_bin_junked_locked JEMALLOC_N(arena_dalloc_bin_junked_locked) +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) +#define arena_dalloc_promoted JEMALLOC_N(arena_dalloc_promoted) +#define arena_dalloc_small JEMALLOC_N(arena_dalloc_small) +#define arena_decay JEMALLOC_N(arena_decay) +#define arena_destroy JEMALLOC_N(arena_destroy) +#define arena_dirty_decay_ms_default_get JEMALLOC_N(arena_dirty_decay_ms_default_get) +#define arena_dirty_decay_ms_default_set JEMALLOC_N(arena_dirty_decay_ms_default_set) +#define arena_dirty_decay_ms_get JEMALLOC_N(arena_dirty_decay_ms_get) +#define arena_dirty_decay_ms_set JEMALLOC_N(arena_dirty_decay_ms_set) +#define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) +#define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) +#define arena_extent_alloc_large JEMALLOC_N(arena_extent_alloc_large) +#define arena_extent_dalloc_large_prep JEMALLOC_N(arena_extent_dalloc_large_prep) +#define arena_extent_ralloc_large_expand JEMALLOC_N(arena_extent_ralloc_large_expand) +#define arena_extent_ralloc_large_shrink JEMALLOC_N(arena_extent_ralloc_large_shrink) +#define arena_extents_dirty_dalloc JEMALLOC_N(arena_extents_dirty_dalloc) +#define arena_extent_sn_next JEMALLOC_N(arena_extent_sn_next) +#define arena_init_huge JEMALLOC_N(arena_init_huge) +#define arena_is_huge JEMALLOC_N(arena_is_huge) +#define arena_malloc_hard JEMALLOC_N(arena_malloc_hard) +#define arena_muzzy_decay_ms_default_get JEMALLOC_N(arena_muzzy_decay_ms_default_get) +#define arena_muzzy_decay_ms_default_set JEMALLOC_N(arena_muzzy_decay_ms_default_set) +#define arena_muzzy_decay_ms_get JEMALLOC_N(arena_muzzy_decay_ms_get) +#define arena_muzzy_decay_ms_set JEMALLOC_N(arena_muzzy_decay_ms_set) +#define arena_new JEMALLOC_N(arena_new) +#define arena_nthreads_dec JEMALLOC_N(arena_nthreads_dec) +#define arena_nthreads_get JEMALLOC_N(arena_nthreads_get) +#define arena_nthreads_inc JEMALLOC_N(arena_nthreads_inc) +#define arena_palloc JEMALLOC_N(arena_palloc) +#define arena_postfork_child JEMALLOC_N(arena_postfork_child) +#define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) +#define arena_prefork0 JEMALLOC_N(arena_prefork0) +#define arena_prefork1 JEMALLOC_N(arena_prefork1) +#define arena_prefork2 JEMALLOC_N(arena_prefork2) +#define arena_prefork3 JEMALLOC_N(arena_prefork3) +#define arena_prefork4 JEMALLOC_N(arena_prefork4) +#define arena_prefork5 JEMALLOC_N(arena_prefork5) +#define arena_prefork6 JEMALLOC_N(arena_prefork6) +#define arena_prefork7 JEMALLOC_N(arena_prefork7) +#define arena_prof_promote JEMALLOC_N(arena_prof_promote) +#define arena_ralloc JEMALLOC_N(arena_ralloc) +#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) +#define arena_reset JEMALLOC_N(arena_reset) +#define arena_retain_grow_limit_get_set JEMALLOC_N(arena_retain_grow_limit_get_set) +#define arena_stats_merge JEMALLOC_N(arena_stats_merge) +#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) +#define h_steps JEMALLOC_N(h_steps) +#define opt_dirty_decay_ms JEMALLOC_N(opt_dirty_decay_ms) +#define opt_muzzy_decay_ms JEMALLOC_N(opt_muzzy_decay_ms) +#define opt_oversize_threshold JEMALLOC_N(opt_oversize_threshold) +#define opt_percpu_arena JEMALLOC_N(opt_percpu_arena) +#define oversize_threshold JEMALLOC_N(oversize_threshold) +#define percpu_arena_mode_names JEMALLOC_N(percpu_arena_mode_names) +#define background_thread_boot0 JEMALLOC_N(background_thread_boot0) +#define background_thread_boot1 JEMALLOC_N(background_thread_boot1) +#define background_thread_create JEMALLOC_N(background_thread_create) +#define background_thread_ctl_init JEMALLOC_N(background_thread_ctl_init) +#define background_thread_enabled_state JEMALLOC_N(background_thread_enabled_state) +#define background_thread_info JEMALLOC_N(background_thread_info) +#define background_thread_interval_check JEMALLOC_N(background_thread_interval_check) +#define background_thread_lock JEMALLOC_N(background_thread_lock) +#define background_thread_postfork_child JEMALLOC_N(background_thread_postfork_child) +#define background_thread_postfork_parent JEMALLOC_N(background_thread_postfork_parent) +#define background_thread_prefork0 JEMALLOC_N(background_thread_prefork0) +#define background_thread_prefork1 JEMALLOC_N(background_thread_prefork1) +#define background_threads_disable JEMALLOC_N(background_threads_disable) +#define background_threads_enable JEMALLOC_N(background_threads_enable) +#define background_thread_stats_read JEMALLOC_N(background_thread_stats_read) +#define max_background_threads JEMALLOC_N(max_background_threads) +#define n_background_threads JEMALLOC_N(n_background_threads) +#define opt_background_thread JEMALLOC_N(opt_background_thread) +#define opt_max_background_threads JEMALLOC_N(opt_max_background_threads) +#define pthread_create_wrapper JEMALLOC_N(pthread_create_wrapper) +#define b0get JEMALLOC_N(b0get) +#define base_alloc JEMALLOC_N(base_alloc) +#define base_alloc_extent JEMALLOC_N(base_alloc_extent) +#define base_boot JEMALLOC_N(base_boot) +#define base_delete JEMALLOC_N(base_delete) +#define base_extent_hooks_get JEMALLOC_N(base_extent_hooks_get) +#define base_extent_hooks_set JEMALLOC_N(base_extent_hooks_set) +#define base_new JEMALLOC_N(base_new) +#define base_postfork_child JEMALLOC_N(base_postfork_child) +#define base_postfork_parent JEMALLOC_N(base_postfork_parent) +#define base_prefork JEMALLOC_N(base_prefork) +#define base_stats_get JEMALLOC_N(base_stats_get) +#define metadata_thp_mode_names JEMALLOC_N(metadata_thp_mode_names) +#define opt_metadata_thp JEMALLOC_N(opt_metadata_thp) +#define bin_boot JEMALLOC_N(bin_boot) +#define bin_infos JEMALLOC_N(bin_infos) +#define bin_init JEMALLOC_N(bin_init) +#define bin_postfork_child JEMALLOC_N(bin_postfork_child) +#define bin_postfork_parent JEMALLOC_N(bin_postfork_parent) +#define bin_prefork JEMALLOC_N(bin_prefork) +#define bin_shard_sizes_boot JEMALLOC_N(bin_shard_sizes_boot) +#define bin_update_shard_size JEMALLOC_N(bin_update_shard_size) +#define bitmap_info_init JEMALLOC_N(bitmap_info_init) +#define bitmap_init JEMALLOC_N(bitmap_init) +#define bitmap_size JEMALLOC_N(bitmap_size) +#define ckh_count JEMALLOC_N(ckh_count) +#define ckh_delete JEMALLOC_N(ckh_delete) +#define ckh_insert JEMALLOC_N(ckh_insert) +#define ckh_iter JEMALLOC_N(ckh_iter) +#define ckh_new JEMALLOC_N(ckh_new) +#define ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash) +#define ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp) +#define ckh_remove JEMALLOC_N(ckh_remove) +#define ckh_search JEMALLOC_N(ckh_search) +#define ckh_string_hash JEMALLOC_N(ckh_string_hash) +#define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) +#define ctl_boot JEMALLOC_N(ctl_boot) +#define ctl_bymib JEMALLOC_N(ctl_bymib) +#define ctl_byname JEMALLOC_N(ctl_byname) +#define ctl_nametomib JEMALLOC_N(ctl_nametomib) +#define ctl_postfork_child JEMALLOC_N(ctl_postfork_child) +#define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent) +#define ctl_prefork JEMALLOC_N(ctl_prefork) +#define div_init JEMALLOC_N(div_init) +#define extent_alloc JEMALLOC_N(extent_alloc) +#define extent_alloc_wrapper JEMALLOC_N(extent_alloc_wrapper) +#define extent_avail_any JEMALLOC_N(extent_avail_any) +#define extent_avail_empty JEMALLOC_N(extent_avail_empty) +#define extent_avail_first JEMALLOC_N(extent_avail_first) +#define extent_avail_insert JEMALLOC_N(extent_avail_insert) +#define extent_avail_new JEMALLOC_N(extent_avail_new) +#define extent_avail_remove JEMALLOC_N(extent_avail_remove) +#define extent_avail_remove_any JEMALLOC_N(extent_avail_remove_any) +#define extent_avail_remove_first JEMALLOC_N(extent_avail_remove_first) +#define extent_boot JEMALLOC_N(extent_boot) +#define extent_commit_wrapper JEMALLOC_N(extent_commit_wrapper) +#define extent_dalloc JEMALLOC_N(extent_dalloc) +#define extent_dalloc_gap JEMALLOC_N(extent_dalloc_gap) +#define extent_dalloc_wrapper JEMALLOC_N(extent_dalloc_wrapper) +#define extent_decommit_wrapper JEMALLOC_N(extent_decommit_wrapper) +#define extent_destroy_wrapper JEMALLOC_N(extent_destroy_wrapper) +#define extent_heap_any JEMALLOC_N(extent_heap_any) +#define extent_heap_empty JEMALLOC_N(extent_heap_empty) +#define extent_heap_first JEMALLOC_N(extent_heap_first) +#define extent_heap_insert JEMALLOC_N(extent_heap_insert) +#define extent_heap_new JEMALLOC_N(extent_heap_new) +#define extent_heap_remove JEMALLOC_N(extent_heap_remove) +#define extent_heap_remove_any JEMALLOC_N(extent_heap_remove_any) +#define extent_heap_remove_first JEMALLOC_N(extent_heap_remove_first) +#define extent_hooks_default JEMALLOC_N(extent_hooks_default) +#define extent_hooks_get JEMALLOC_N(extent_hooks_get) +#define extent_hooks_set JEMALLOC_N(extent_hooks_set) +#define extent_merge_wrapper JEMALLOC_N(extent_merge_wrapper) +#define extent_mutex_pool JEMALLOC_N(extent_mutex_pool) +#define extent_purge_forced_wrapper JEMALLOC_N(extent_purge_forced_wrapper) +#define extent_purge_lazy_wrapper JEMALLOC_N(extent_purge_lazy_wrapper) +#define extents_alloc JEMALLOC_N(extents_alloc) +#define extents_dalloc JEMALLOC_N(extents_dalloc) +#define extents_evict JEMALLOC_N(extents_evict) +#define extents_init JEMALLOC_N(extents_init) +#define extents_nbytes_get JEMALLOC_N(extents_nbytes_get) +#define extents_nextents_get JEMALLOC_N(extents_nextents_get) +#define extents_npages_get JEMALLOC_N(extents_npages_get) +#define extent_split_wrapper JEMALLOC_N(extent_split_wrapper) +#define extents_postfork_child JEMALLOC_N(extents_postfork_child) +#define extents_postfork_parent JEMALLOC_N(extents_postfork_parent) +#define extents_prefork JEMALLOC_N(extents_prefork) +#define extents_rtree JEMALLOC_N(extents_rtree) +#define extents_state_get JEMALLOC_N(extents_state_get) +#define opt_lg_extent_max_active_fit JEMALLOC_N(opt_lg_extent_max_active_fit) +#define dss_prec_names JEMALLOC_N(dss_prec_names) +#define extent_alloc_dss JEMALLOC_N(extent_alloc_dss) +#define extent_dss_boot JEMALLOC_N(extent_dss_boot) +#define extent_dss_mergeable JEMALLOC_N(extent_dss_mergeable) +#define extent_dss_prec_get JEMALLOC_N(extent_dss_prec_get) +#define extent_dss_prec_set JEMALLOC_N(extent_dss_prec_set) +#define extent_in_dss JEMALLOC_N(extent_in_dss) +#define opt_dss JEMALLOC_N(opt_dss) +#define extent_alloc_mmap JEMALLOC_N(extent_alloc_mmap) +#define extent_dalloc_mmap JEMALLOC_N(extent_dalloc_mmap) +#define opt_retain JEMALLOC_N(opt_retain) +#define hook_boot JEMALLOC_N(hook_boot) +#define hook_install JEMALLOC_N(hook_install) +#define hook_invoke_alloc JEMALLOC_N(hook_invoke_alloc) +#define hook_invoke_dalloc JEMALLOC_N(hook_invoke_dalloc) +#define hook_invoke_expand JEMALLOC_N(hook_invoke_expand) +#define hook_remove JEMALLOC_N(hook_remove) +#define large_dalloc JEMALLOC_N(large_dalloc) +#define large_dalloc_finish JEMALLOC_N(large_dalloc_finish) +#define large_dalloc_junk JEMALLOC_N(large_dalloc_junk) +#define large_dalloc_maybe_junk JEMALLOC_N(large_dalloc_maybe_junk) +#define large_dalloc_prep_junked_locked JEMALLOC_N(large_dalloc_prep_junked_locked) +#define large_malloc JEMALLOC_N(large_malloc) +#define large_palloc JEMALLOC_N(large_palloc) +#define large_prof_alloc_time_get JEMALLOC_N(large_prof_alloc_time_get) +#define large_prof_alloc_time_set JEMALLOC_N(large_prof_alloc_time_set) +#define large_prof_tctx_get JEMALLOC_N(large_prof_tctx_get) +#define large_prof_tctx_reset JEMALLOC_N(large_prof_tctx_reset) +#define large_prof_tctx_set JEMALLOC_N(large_prof_tctx_set) +#define large_ralloc JEMALLOC_N(large_ralloc) +#define large_ralloc_no_move JEMALLOC_N(large_ralloc_no_move) +#define large_salloc JEMALLOC_N(large_salloc) +#define log_init_done JEMALLOC_N(log_init_done) +#define log_var_names JEMALLOC_N(log_var_names) +#define log_var_update_state JEMALLOC_N(log_var_update_state) +#define buferror JEMALLOC_N(buferror) +#define malloc_cprintf JEMALLOC_N(malloc_cprintf) +#define malloc_printf JEMALLOC_N(malloc_printf) +#define malloc_snprintf JEMALLOC_N(malloc_snprintf) +#define malloc_strtoumax JEMALLOC_N(malloc_strtoumax) +#define malloc_vcprintf JEMALLOC_N(malloc_vcprintf) +#define malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf) +#define malloc_write JEMALLOC_N(malloc_write) +#define malloc_mutex_boot JEMALLOC_N(malloc_mutex_boot) +#define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) +#define malloc_mutex_lock_slow JEMALLOC_N(malloc_mutex_lock_slow) +#define malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child) +#define malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent) +#define malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork) +#define malloc_mutex_prof_data_reset JEMALLOC_N(malloc_mutex_prof_data_reset) +#define mutex_pool_init JEMALLOC_N(mutex_pool_init) +#define nstime_add JEMALLOC_N(nstime_add) +#define nstime_compare JEMALLOC_N(nstime_compare) +#define nstime_copy JEMALLOC_N(nstime_copy) +#define nstime_divide JEMALLOC_N(nstime_divide) +#define nstime_iadd JEMALLOC_N(nstime_iadd) +#define nstime_idivide JEMALLOC_N(nstime_idivide) +#define nstime_imultiply JEMALLOC_N(nstime_imultiply) +#define nstime_init JEMALLOC_N(nstime_init) +#define nstime_init2 JEMALLOC_N(nstime_init2) +#define nstime_isubtract JEMALLOC_N(nstime_isubtract) +#define nstime_monotonic JEMALLOC_N(nstime_monotonic) +#define nstime_msec JEMALLOC_N(nstime_msec) +#define nstime_ns JEMALLOC_N(nstime_ns) +#define nstime_nsec JEMALLOC_N(nstime_nsec) +#define nstime_sec JEMALLOC_N(nstime_sec) +#define nstime_subtract JEMALLOC_N(nstime_subtract) +#define nstime_update JEMALLOC_N(nstime_update) +#define init_system_thp_mode JEMALLOC_N(init_system_thp_mode) +#define opt_thp JEMALLOC_N(opt_thp) +#define pages_boot JEMALLOC_N(pages_boot) +#define pages_commit JEMALLOC_N(pages_commit) +#define pages_decommit JEMALLOC_N(pages_decommit) +#define pages_dodump JEMALLOC_N(pages_dodump) +#define pages_dontdump JEMALLOC_N(pages_dontdump) +#define pages_huge JEMALLOC_N(pages_huge) +#define pages_map JEMALLOC_N(pages_map) +#define pages_nohuge JEMALLOC_N(pages_nohuge) +#define pages_purge_forced JEMALLOC_N(pages_purge_forced) +#define pages_purge_lazy JEMALLOC_N(pages_purge_lazy) +#define pages_set_thp_state JEMALLOC_N(pages_set_thp_state) +#define pages_unmap JEMALLOC_N(pages_unmap) +#define thp_mode_names JEMALLOC_N(thp_mode_names) +#define bt2gctx_mtx JEMALLOC_N(bt2gctx_mtx) +#define bt_init JEMALLOC_N(bt_init) +#define lg_prof_sample JEMALLOC_N(lg_prof_sample) +#define opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval) +#define opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample) +#define opt_prof JEMALLOC_N(opt_prof) +#define opt_prof_accum JEMALLOC_N(opt_prof_accum) +#define opt_prof_active JEMALLOC_N(opt_prof_active) +#define opt_prof_final JEMALLOC_N(opt_prof_final) +#define opt_prof_gdump JEMALLOC_N(opt_prof_gdump) +#define opt_prof_leak JEMALLOC_N(opt_prof_leak) +#define opt_prof_log JEMALLOC_N(opt_prof_log) +#define opt_prof_prefix JEMALLOC_N(opt_prof_prefix) +#define opt_prof_thread_active_init JEMALLOC_N(opt_prof_thread_active_init) +#define prof_accum_init JEMALLOC_N(prof_accum_init) +#define prof_active JEMALLOC_N(prof_active) +#define prof_active_get JEMALLOC_N(prof_active_get) +#define prof_active_set JEMALLOC_N(prof_active_set) +#define prof_alloc_rollback JEMALLOC_N(prof_alloc_rollback) +#define prof_backtrace JEMALLOC_N(prof_backtrace) +#define prof_boot0 JEMALLOC_N(prof_boot0) +#define prof_boot1 JEMALLOC_N(prof_boot1) +#define prof_boot2 JEMALLOC_N(prof_boot2) +#define prof_dump_header JEMALLOC_N(prof_dump_header) +#define prof_dump_open JEMALLOC_N(prof_dump_open) +#define prof_free_sampled_object JEMALLOC_N(prof_free_sampled_object) +#define prof_gdump JEMALLOC_N(prof_gdump) +#define prof_gdump_get JEMALLOC_N(prof_gdump_get) +#define prof_gdump_set JEMALLOC_N(prof_gdump_set) +#define prof_gdump_val JEMALLOC_N(prof_gdump_val) +#define prof_idump JEMALLOC_N(prof_idump) +#define prof_interval JEMALLOC_N(prof_interval) +#define prof_logging_state JEMALLOC_N(prof_logging_state) +#define prof_log_start JEMALLOC_N(prof_log_start) +#define prof_log_stop JEMALLOC_N(prof_log_stop) +#define prof_lookup JEMALLOC_N(prof_lookup) +#define prof_malloc_sample_object JEMALLOC_N(prof_malloc_sample_object) +#define prof_mdump JEMALLOC_N(prof_mdump) +#define prof_postfork_child JEMALLOC_N(prof_postfork_child) +#define prof_postfork_parent JEMALLOC_N(prof_postfork_parent) +#define prof_prefork0 JEMALLOC_N(prof_prefork0) +#define prof_prefork1 JEMALLOC_N(prof_prefork1) +#define prof_reset JEMALLOC_N(prof_reset) +#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) +#define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) +#define prof_tdata_init JEMALLOC_N(prof_tdata_init) +#define prof_tdata_reinit JEMALLOC_N(prof_tdata_reinit) +#define prof_thread_active_get JEMALLOC_N(prof_thread_active_get) +#define prof_thread_active_init_get JEMALLOC_N(prof_thread_active_init_get) +#define prof_thread_active_init_set JEMALLOC_N(prof_thread_active_init_set) +#define prof_thread_active_set JEMALLOC_N(prof_thread_active_set) +#define prof_thread_name_get JEMALLOC_N(prof_thread_name_get) +#define prof_thread_name_set JEMALLOC_N(prof_thread_name_set) +#define rtree_ctx_data_init JEMALLOC_N(rtree_ctx_data_init) +#define rtree_leaf_alloc JEMALLOC_N(rtree_leaf_alloc) +#define rtree_leaf_dalloc JEMALLOC_N(rtree_leaf_dalloc) +#define rtree_leaf_elm_lookup_hard JEMALLOC_N(rtree_leaf_elm_lookup_hard) +#define rtree_new JEMALLOC_N(rtree_new) +#define rtree_node_alloc JEMALLOC_N(rtree_node_alloc) +#define rtree_node_dalloc JEMALLOC_N(rtree_node_dalloc) +#define arena_mutex_names JEMALLOC_N(arena_mutex_names) +#define global_mutex_names JEMALLOC_N(global_mutex_names) +#define opt_stats_print JEMALLOC_N(opt_stats_print) +#define opt_stats_print_opts JEMALLOC_N(opt_stats_print_opts) +#define stats_print JEMALLOC_N(stats_print) +#define sc_boot JEMALLOC_N(sc_boot) +#define sc_data_global JEMALLOC_N(sc_data_global) +#define sc_data_init JEMALLOC_N(sc_data_init) +#define sc_data_update_slab_size JEMALLOC_N(sc_data_update_slab_size) +#define sz_boot JEMALLOC_N(sz_boot) +#define sz_index2size_tab JEMALLOC_N(sz_index2size_tab) +#define sz_pind2sz_tab JEMALLOC_N(sz_pind2sz_tab) +#define sz_size2index_tab JEMALLOC_N(sz_size2index_tab) +#define nhbins JEMALLOC_N(nhbins) +#define opt_lg_tcache_max JEMALLOC_N(opt_lg_tcache_max) +#define opt_tcache JEMALLOC_N(opt_tcache) +#define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) +#define tcache_arena_associate JEMALLOC_N(tcache_arena_associate) +#define tcache_arena_reassociate JEMALLOC_N(tcache_arena_reassociate) +#define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) +#define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) +#define tcache_bin_info JEMALLOC_N(tcache_bin_info) +#define tcache_boot JEMALLOC_N(tcache_boot) +#define tcache_cleanup JEMALLOC_N(tcache_cleanup) +#define tcache_create_explicit JEMALLOC_N(tcache_create_explicit) +#define tcache_event_hard JEMALLOC_N(tcache_event_hard) +#define tcache_flush JEMALLOC_N(tcache_flush) +#define tcache_maxclass JEMALLOC_N(tcache_maxclass) +#define tcache_postfork_child JEMALLOC_N(tcache_postfork_child) +#define tcache_postfork_parent JEMALLOC_N(tcache_postfork_parent) +#define tcache_prefork JEMALLOC_N(tcache_prefork) +#define tcaches JEMALLOC_N(tcaches) +#define tcache_salloc JEMALLOC_N(tcache_salloc) +#define tcaches_create JEMALLOC_N(tcaches_create) +#define tcaches_destroy JEMALLOC_N(tcaches_destroy) +#define tcaches_flush JEMALLOC_N(tcaches_flush) +#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) +#define tsd_tcache_data_init JEMALLOC_N(tsd_tcache_data_init) +#define tsd_tcache_enabled_data_init JEMALLOC_N(tsd_tcache_enabled_data_init) +#define test_hooks_arena_new_hook JEMALLOC_N(test_hooks_arena_new_hook) +#define test_hooks_libc_hook JEMALLOC_N(test_hooks_libc_hook) +#define malloc_tsd_boot0 JEMALLOC_N(malloc_tsd_boot0) +#define malloc_tsd_boot1 JEMALLOC_N(malloc_tsd_boot1) +#define malloc_tsd_cleanup_register JEMALLOC_N(malloc_tsd_cleanup_register) +#define malloc_tsd_dalloc JEMALLOC_N(malloc_tsd_dalloc) +#define malloc_tsd_malloc JEMALLOC_N(malloc_tsd_malloc) +#define tsd_booted JEMALLOC_N(tsd_booted) +#define tsd_cleanup JEMALLOC_N(tsd_cleanup) +#define tsd_fetch_slow JEMALLOC_N(tsd_fetch_slow) +#define tsd_global_slow JEMALLOC_N(tsd_global_slow) +#define tsd_global_slow_dec JEMALLOC_N(tsd_global_slow_dec) +#define tsd_global_slow_inc JEMALLOC_N(tsd_global_slow_inc) +#define tsd_postfork_child JEMALLOC_N(tsd_postfork_child) +#define tsd_postfork_parent JEMALLOC_N(tsd_postfork_parent) +#define tsd_prefork JEMALLOC_N(tsd_prefork) +#define tsd_slow_update JEMALLOC_N(tsd_slow_update) +#define tsd_state_set JEMALLOC_N(tsd_state_set) +#define tsd_tls JEMALLOC_N(tsd_tls) +#define tsd_tsd JEMALLOC_N(tsd_tsd) +#define witness_depth_error JEMALLOC_N(witness_depth_error) +#define witnesses_cleanup JEMALLOC_N(witnesses_cleanup) +#define witness_init JEMALLOC_N(witness_init) +#define witness_lock_error JEMALLOC_N(witness_lock_error) +#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error) +#define witness_owner_error JEMALLOC_N(witness_owner_error) +#define witness_postfork_child JEMALLOC_N(witness_postfork_child) +#define witness_postfork_parent JEMALLOC_N(witness_postfork_parent) +#define witness_prefork JEMALLOC_N(witness_prefork) diff --git a/dep/jemalloc/include/jemalloc/internal/prof_externs.h b/dep/jemalloc/include/jemalloc/internal/prof_externs.h index 04348696f58..094f3e170ae 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_externs.h @@ -14,6 +14,7 @@ extern bool opt_prof_gdump; /* High-water memory dumping. */ extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ +extern bool opt_prof_log; /* Turn logging on at boot. */ extern char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -45,7 +46,8 @@ extern size_t lg_prof_sample; void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx); -void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); +void prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx); void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); @@ -89,4 +91,15 @@ void prof_postfork_parent(tsdn_t *tsdn); void prof_postfork_child(tsdn_t *tsdn); void prof_sample_threshold_update(prof_tdata_t *tdata); +bool prof_log_start(tsdn_t *tsdn, const char *filename); +bool prof_log_stop(tsdn_t *tsdn); +#ifdef JEMALLOC_JET +size_t prof_log_bt_count(void); +size_t prof_log_alloc_count(void); +size_t prof_log_thr_count(void); +bool prof_log_is_logging(void); +bool prof_log_rep_check(void); +void prof_log_dummy_set(bool new_value); +#endif + #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/prof_inlines_a.h b/dep/jemalloc/include/jemalloc/internal/prof_inlines_a.h index a6efb4851dc..471d9853cf8 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_inlines_a.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_inlines_a.h @@ -4,7 +4,8 @@ #include "jemalloc/internal/mutex.h" static inline bool -prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { +prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, + uint64_t accumbytes) { cassert(config_prof); bool overflow; @@ -42,7 +43,8 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { } static inline void -prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { +prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, + size_t usize) { cassert(config_prof); /* @@ -55,15 +57,15 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { #ifdef JEMALLOC_ATOMIC_U64 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); do { - a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - - usize) : 0; + a1 = (a0 >= SC_LARGE_MINCLASS - usize) + ? a0 - (SC_LARGE_MINCLASS - usize) : 0; } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else malloc_mutex_lock(tsdn, &prof_accum->mtx); a0 = prof_accum->accumbytes; - a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : - 0; + a1 = (a0 >= SC_LARGE_MINCLASS - usize) + ? a0 - (SC_LARGE_MINCLASS - usize) : 0; prof_accum->accumbytes = a1; malloc_mutex_unlock(tsdn, &prof_accum->mtx); #endif diff --git a/dep/jemalloc/include/jemalloc/internal/prof_inlines_b.h b/dep/jemalloc/include/jemalloc/internal/prof_inlines_b.h index 6ff465ad7f0..8358bffb5e6 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_inlines_b.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_inlines_b.h @@ -61,13 +61,54 @@ prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { arena_prof_tctx_reset(tsdn, ptr, tctx); } +JEMALLOC_ALWAYS_INLINE nstime_t +prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + return arena_prof_alloc_time_get(tsdn, ptr, alloc_ctx); +} + +JEMALLOC_ALWAYS_INLINE void +prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + arena_prof_alloc_time_set(tsdn, ptr, alloc_ctx, t); +} + +JEMALLOC_ALWAYS_INLINE bool +prof_sample_check(tsd_t *tsd, size_t usize, bool update) { + ssize_t check = update ? 0 : usize; + + int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); + if (update) { + bytes_until_sample -= usize; + if (tsd_nominal(tsd)) { + tsd_bytes_until_sample_set(tsd, bytes_until_sample); + } + } + if (likely(bytes_until_sample >= check)) { + return true; + } + + return false; +} + JEMALLOC_ALWAYS_INLINE bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, - prof_tdata_t **tdata_out) { + prof_tdata_t **tdata_out) { prof_tdata_t *tdata; cassert(config_prof); + /* Fastpath: no need to load tdata */ + if (likely(prof_sample_check(tsd, usize, update))) { + return true; + } + + bool booted = tsd_prof_tdata_get(tsd); tdata = prof_tdata_get(tsd, true); if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) { tdata = NULL; @@ -81,21 +122,23 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, return true; } - if (likely(tdata->bytes_until_sample >= usize)) { - if (update) { - tdata->bytes_until_sample -= usize; - } + /* + * If this was the first creation of tdata, then + * prof_tdata_get() reset bytes_until_sample, so decrement and + * check it again + */ + if (!booted && prof_sample_check(tsd, usize, update)) { return true; - } else { - if (tsd_reentrancy_level_get(tsd) > 0) { - return true; - } - /* Compute new sample threshold. */ - if (update) { - prof_sample_threshold_update(tdata); - } - return !tdata->active; } + + if (tsd_reentrancy_level_get(tsd) > 0) { + return true; + } + /* Compute new sample threshold. */ + if (update) { + prof_sample_threshold_update(tdata); + } + return !tdata->active; } JEMALLOC_ALWAYS_INLINE prof_tctx_t * @@ -187,7 +230,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, * counters. */ if (unlikely(old_sampled)) { - prof_free_sampled_object(tsd, old_usize, old_tctx); + prof_free_sampled_object(tsd, ptr, old_usize, old_tctx); } } @@ -199,7 +242,7 @@ prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) { assert(usize == isalloc(tsd_tsdn(tsd), ptr)); if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { - prof_free_sampled_object(tsd, usize, tctx); + prof_free_sampled_object(tsd, ptr, usize, tctx); } } diff --git a/dep/jemalloc/include/jemalloc/internal/prof_structs.h b/dep/jemalloc/include/jemalloc/internal/prof_structs.h index 0d58ae1005b..34ed4822b67 100644 --- a/dep/jemalloc/include/jemalloc/internal/prof_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/prof_structs.h @@ -169,7 +169,6 @@ struct prof_tdata_s { /* Sampling state. */ uint64_t prng_state; - uint64_t bytes_until_sample; /* State used to avoid dumping while operating on prof internals. */ bool enq; diff --git a/dep/jemalloc/include/jemalloc/internal/quantum.h b/dep/jemalloc/include/jemalloc/internal/quantum.h new file mode 100644 index 00000000000..821086e992c --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/quantum.h @@ -0,0 +1,77 @@ +#ifndef JEMALLOC_INTERNAL_QUANTUM_H +#define JEMALLOC_INTERNAL_QUANTUM_H + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __m68k__ +# define LG_QUANTUM 3 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __nios2__ +# define LG_QUANTUM 3 +# endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# if defined(__riscv) || defined(__riscv__) +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ + defined(__SH4_SINGLE_ONLY__)) +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "Unknown minimum alignment for architecture; specify via " + "--with-lg-quantum" +# endif +#endif + +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) + +/* Return the smallest quantum multiple that is >= a. */ +#define QUANTUM_CEILING(a) \ + (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) + +#endif /* JEMALLOC_INTERNAL_QUANTUM_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/rtree.h b/dep/jemalloc/include/jemalloc/internal/rtree.h index b59d33a80bc..16ccbebee7f 100644 --- a/dep/jemalloc/include/jemalloc/internal/rtree.h +++ b/dep/jemalloc/include/jemalloc/internal/rtree.h @@ -4,7 +4,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree_tsd.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/tsd.h" /* @@ -31,7 +31,7 @@ # error Unsupported number of significant virtual address bits #endif /* Use compact leaf representation if virtual address encoding allows. */ -#if RTREE_NHIB >= LG_CEIL_NSIZES +#if RTREE_NHIB >= LG_CEIL(SC_NSIZES) # define RTREE_LEAF_COMPACT #endif @@ -170,8 +170,8 @@ rtree_subkey(uintptr_t key, unsigned level) { */ # ifdef RTREE_LEAF_COMPACT JEMALLOC_ALWAYS_INLINE uintptr_t -rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool dependent) { return (uintptr_t)atomic_load_p(&elm->le_bits, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); } @@ -208,7 +208,7 @@ rtree_leaf_elm_bits_slab_get(uintptr_t bits) { # endif JEMALLOC_ALWAYS_INLINE extent_t * -rtree_leaf_elm_extent_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -221,7 +221,7 @@ rtree_leaf_elm_extent_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE szind_t -rtree_leaf_elm_szind_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -233,7 +233,7 @@ rtree_leaf_elm_szind_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE bool -rtree_leaf_elm_slab_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -245,7 +245,7 @@ rtree_leaf_elm_slab_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_extent_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, extent_t *extent) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true); @@ -259,9 +259,9 @@ rtree_leaf_elm_extent_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_szind_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind) { - assert(szind <= NSIZES); + assert(szind <= SC_NSIZES); #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -277,7 +277,7 @@ rtree_leaf_elm_szind_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_slab_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, +rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -292,8 +292,8 @@ rtree_leaf_elm_slab_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, } static inline void -rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - extent_t *extent, szind_t szind, bool slab) { +rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, extent_t *extent, szind_t szind, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = ((uintptr_t)szind << LG_VADDR) | ((uintptr_t)extent & (((uintptr_t)0x1 << LG_VADDR) - 1)) | @@ -313,7 +313,7 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, static inline void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind, bool slab) { - assert(!slab || szind < NBINS); + assert(!slab || szind < SC_NBINS); /* * The caller implicitly assures that it is the only writer to the szind @@ -429,7 +429,7 @@ rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, dependent); if (!dependent && elm == NULL) { - return NSIZES; + return SC_NSIZES; } return rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent); } @@ -452,6 +452,42 @@ rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return false; } +/* + * Try to read szind_slab from the L1 cache. Returns true on a hit, + * and fills in r_szind and r_slab. Otherwise returns false. + * + * Key is allowed to be NULL in order to save an extra branch on the + * fastpath. returns false in this case. + */ +JEMALLOC_ALWAYS_INLINE bool +rtree_szind_slab_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, szind_t *r_szind, bool *r_slab) { + rtree_leaf_elm_t *elm; + + size_t slot = rtree_cache_direct_map(key); + uintptr_t leafkey = rtree_leafkey(key); + assert(leafkey != RTREE_LEAFKEY_INVALID); + + if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) { + rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; + assert(leaf != NULL); + uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); + elm = &leaf[subkey]; + +#ifdef RTREE_LEAF_COMPACT + uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, + elm, true); + *r_szind = rtree_leaf_elm_bits_szind_get(bits); + *r_slab = rtree_leaf_elm_bits_slab_get(bits); +#else + *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, true); + *r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, true); +#endif + return true; + } else { + return false; + } +} JEMALLOC_ALWAYS_INLINE bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) { @@ -474,7 +510,7 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, static inline void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab) { - assert(!slab || szind < NBINS); + assert(!slab || szind < SC_NBINS); rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab); @@ -486,7 +522,7 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false) != NULL); - rtree_leaf_elm_write(tsdn, rtree, elm, NULL, NSIZES, false); + rtree_leaf_elm_write(tsdn, rtree, elm, NULL, SC_NSIZES, false); } #endif /* JEMALLOC_INTERNAL_RTREE_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h b/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h index 93a75173a8d..562e29297a7 100644 --- a/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h +++ b/dep/jemalloc/include/jemalloc/internal/rtree_tsd.h @@ -26,7 +26,7 @@ * Zero initializer required for tsd initialization only. Proper initialization * done via rtree_ctx_data_init(). */ -#define RTREE_CTX_ZERO_INITIALIZER {{{0}}, {{0}}} +#define RTREE_CTX_ZERO_INITIALIZER {{{0, 0}}, {{0, 0}}} typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; diff --git a/dep/jemalloc/include/jemalloc/internal/sc.h b/dep/jemalloc/include/jemalloc/internal/sc.h new file mode 100644 index 00000000000..ef0a4512370 --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/sc.h @@ -0,0 +1,320 @@ +#ifndef JEMALLOC_INTERNAL_SC_H +#define JEMALLOC_INTERNAL_SC_H + +#include "jemalloc/internal/jemalloc_internal_types.h" + +/* + * Size class computations: + * + * These are a little tricky; we'll first start by describing how things + * generally work, and then describe some of the details. + * + * Ignore the first few size classes for a moment. We can then split all the + * remaining size classes into groups. The size classes in a group are spaced + * such that they cover allocation request sizes in a power-of-2 range. The + * power of two is called the base of the group, and the size classes in it + * satisfy allocations in the half-open range (base, base * 2]. There are + * SC_NGROUP size classes in each group, equally spaced in the range, so that + * each one covers allocations for base / SC_NGROUP possible allocation sizes. + * We call that value (base / SC_NGROUP) the delta of the group. Each size class + * is delta larger than the one before it (including the initial size class in a + * group, which is delta large than 2**base, the largest size class in the + * previous group). + * To make the math all work out nicely, we require that SC_NGROUP is a power of + * two, and define it in terms of SC_LG_NGROUP. We'll often talk in terms of + * lg_base and lg_delta. For each of these groups then, we have that + * lg_delta == lg_base - SC_LG_NGROUP. + * The size classes in a group with a given lg_base and lg_delta (which, recall, + * can be computed from lg_base for these groups) are therefore: + * base + 1 * delta + * which covers allocations in (base, base + 1 * delta] + * base + 2 * delta + * which covers allocations in (base + 1 * delta, base + 2 * delta]. + * base + 3 * delta + * which covers allocations in (base + 2 * delta, base + 3 * delta]. + * ... + * base + SC_NGROUP * delta ( == 2 * base) + * which covers allocations in (base + (SC_NGROUP - 1) * delta, 2 * base]. + * (Note that currently SC_NGROUP is always 4, so the "..." is empty in + * practice.) + * Note that the last size class in the group is the next power of two (after + * base), so that we've set up the induction correctly for the next group's + * selection of delta. + * + * Now, let's start considering the first few size classes. Two extra constants + * come into play here: LG_QUANTUM and SC_LG_TINY_MIN. LG_QUANTUM ensures + * correct platform alignment; all objects of size (1 << LG_QUANTUM) or larger + * are at least (1 << LG_QUANTUM) aligned; this can be used to ensure that we + * never return improperly aligned memory, by making (1 << LG_QUANTUM) equal the + * highest required alignment of a platform. For allocation sizes smaller than + * (1 << LG_QUANTUM) though, we can be more relaxed (since we don't support + * platforms with types with alignment larger than their size). To allow such + * allocations (without wasting space unnecessarily), we introduce tiny size + * classes; one per power of two, up until we hit the quantum size. There are + * therefore LG_QUANTUM - SC_LG_TINY_MIN such size classes. + * + * Next, we have a size class of size LG_QUANTUM. This can't be the start of a + * group in the sense we described above (covering a power of two range) since, + * if we divided into it to pick a value of delta, we'd get a delta smaller than + * (1 << LG_QUANTUM) for sizes >= (1 << LG_QUANTUM), which is against the rules. + * + * The first base we can divide by SC_NGROUP while still being at least + * (1 << LG_QUANTUM) is SC_NGROUP * (1 << LG_QUANTUM). We can get there by + * having SC_NGROUP size classes, spaced (1 << LG_QUANTUM) apart. These size + * classes are: + * 1 * (1 << LG_QUANTUM) + * 2 * (1 << LG_QUANTUM) + * 3 * (1 << LG_QUANTUM) + * ... (although, as above, this "..." is empty in practice) + * SC_NGROUP * (1 << LG_QUANTUM). + * + * There are SC_NGROUP of these size classes, so we can regard it as a sort of + * pseudo-group, even though it spans multiple powers of 2, is divided + * differently, and both starts and ends on a power of 2 (as opposed to just + * ending). SC_NGROUP is itself a power of two, so the first group after the + * pseudo-group has the power-of-two base SC_NGROUP * (1 << LG_QUANTUM), for a + * lg_base of LG_QUANTUM + SC_LG_NGROUP. We can divide this base into SC_NGROUP + * sizes without violating our LG_QUANTUM requirements, so we can safely set + * lg_delta = lg_base - SC_LG_GROUP (== LG_QUANTUM). + * + * So, in order, the size classes are: + * + * Tiny size classes: + * - Count: LG_QUANTUM - SC_LG_TINY_MIN. + * - Sizes: + * 1 << SC_LG_TINY_MIN + * 1 << (SC_LG_TINY_MIN + 1) + * 1 << (SC_LG_TINY_MIN + 2) + * ... + * 1 << (LG_QUANTUM - 1) + * + * Initial pseudo-group: + * - Count: SC_NGROUP + * - Sizes: + * 1 * (1 << LG_QUANTUM) + * 2 * (1 << LG_QUANTUM) + * 3 * (1 << LG_QUANTUM) + * ... + * SC_NGROUP * (1 << LG_QUANTUM) + * + * Regular group 0: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * Regular group 1: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + 1 and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * ... + * + * Regular group N: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + N and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * + * Representation of metadata: + * To make the math easy, we'll mostly work in lg quantities. We record lg_base, + * lg_delta, and ndelta (i.e. number of deltas above the base) on a + * per-size-class basis, and maintain the invariant that, across all size + * classes, size == (1 << lg_base) + ndelta * (1 << lg_delta). + * + * For regular groups (i.e. those with lg_base >= LG_QUANTUM + SC_LG_NGROUP), + * lg_delta is lg_base - SC_LG_NGROUP, and ndelta goes from 1 to SC_NGROUP. + * + * For the initial tiny size classes (if any), lg_base is lg(size class size). + * lg_delta is lg_base for the first size class, and lg_base - 1 for all + * subsequent ones. ndelta is always 0. + * + * For the pseudo-group, if there are no tiny size classes, then we set + * lg_base == LG_QUANTUM, lg_delta == LG_QUANTUM, and have ndelta range from 0 + * to SC_NGROUP - 1. (Note that delta == base, so base + (SC_NGROUP - 1) * delta + * is just SC_NGROUP * base, or (1 << (SC_LG_NGROUP + LG_QUANTUM)), so we do + * indeed get a power of two that way). If there *are* tiny size classes, then + * the first size class needs to have lg_delta relative to the largest tiny size + * class. We therefore set lg_base == LG_QUANTUM - 1, + * lg_delta == LG_QUANTUM - 1, and ndelta == 1, keeping the rest of the + * pseudo-group the same. + * + * + * Other terminology: + * "Small" size classes mean those that are allocated out of bins, which is the + * same as those that are slab allocated. + * "Large" size classes are those that are not small. The cutoff for counting as + * large is page size * group size. + */ + +/* + * Size class N + (1 << SC_LG_NGROUP) twice the size of size class N. + */ +#define SC_LG_NGROUP 2 +#define SC_LG_TINY_MIN 3 + +#if SC_LG_TINY_MIN == 0 +/* The div module doesn't support division by 1, which this would require. */ +#error "Unsupported LG_TINY_MIN" +#endif + +/* + * The definitions below are all determined by the above settings and system + * characteristics. + */ +#define SC_NGROUP (1ULL << SC_LG_NGROUP) +#define SC_PTR_BITS ((1ULL << LG_SIZEOF_PTR) * 8) +#define SC_NTINY (LG_QUANTUM - SC_LG_TINY_MIN) +#define SC_LG_TINY_MAXCLASS (LG_QUANTUM > SC_LG_TINY_MIN ? LG_QUANTUM - 1 : -1) +#define SC_NPSEUDO SC_NGROUP +#define SC_LG_FIRST_REGULAR_BASE (LG_QUANTUM + SC_LG_NGROUP) +/* + * We cap allocations to be less than 2 ** (ptr_bits - 1), so the highest base + * we need is 2 ** (ptr_bits - 2). (This also means that the last group is 1 + * size class shorter than the others). + * We could probably save some space in arenas by capping this at LG_VADDR size. + */ +#define SC_LG_BASE_MAX (SC_PTR_BITS - 2) +#define SC_NREGULAR (SC_NGROUP * \ + (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) +#define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR) + + /* The number of size classes that are a multiple of the page size. */ +#define SC_NPSIZES ( \ + /* Start with all the size classes. */ \ + SC_NSIZES \ + /* Subtract out those groups with too small a base. */ \ + - (LG_PAGE - 1 - SC_LG_FIRST_REGULAR_BASE) * SC_NGROUP \ + /* And the pseudo-group. */ \ + - SC_NPSEUDO \ + /* And the tiny group. */ \ + - SC_NTINY \ + /* Groups where ndelta*delta is not a multiple of the page size. */ \ + - (2 * (SC_NGROUP))) + +/* + * We declare a size class is binnable if size < page size * group. Or, in other + * words, lg(size) < lg(page size) + lg(group size). + */ +#define SC_NBINS ( \ + /* Sub-regular size classes. */ \ + SC_NTINY + SC_NPSEUDO \ + /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ + + SC_NGROUP * (LG_PAGE + SC_LG_NGROUP - SC_LG_FIRST_REGULAR_BASE) \ + /* Last SC of the last group hits the bound exactly; exclude it. */ \ + - 1) + +/* + * The size2index_tab lookup table uses uint8_t to encode each bin index, so we + * cannot support more than 256 small size classes. + */ +#if (SC_NBINS > 256) +# error "Too many small size classes" +#endif + +/* The largest size class in the lookup table. */ +#define SC_LOOKUP_MAXCLASS ((size_t)1 << 12) + +/* Internal, only used for the definition of SC_SMALL_MAXCLASS. */ +#define SC_SMALL_MAX_BASE ((size_t)1 << (LG_PAGE + SC_LG_NGROUP - 1)) +#define SC_SMALL_MAX_DELTA ((size_t)1 << (LG_PAGE - 1)) + +/* The largest size class allocated out of a slab. */ +#define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ + + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) + +/* The smallest size class not allocated out of a slab. */ +#define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP)) +#define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP) + +/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */ +#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2)) +#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP)) + +/* The largest size class supported. */ +#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA) + +typedef struct sc_s sc_t; +struct sc_s { + /* Size class index, or -1 if not a valid size class. */ + int index; + /* Lg group base size (no deltas added). */ + int lg_base; + /* Lg delta to previous size class. */ + int lg_delta; + /* Delta multiplier. size == 1<<lg_base + ndelta<<lg_delta */ + int ndelta; + /* + * True if the size class is a multiple of the page size, false + * otherwise. + */ + bool psz; + /* + * True if the size class is a small, bin, size class. False otherwise. + */ + bool bin; + /* The slab page count if a small bin size class, 0 otherwise. */ + int pgs; + /* Same as lg_delta if a lookup table size class, 0 otherwise. */ + int lg_delta_lookup; +}; + +typedef struct sc_data_s sc_data_t; +struct sc_data_s { + /* Number of tiny size classes. */ + unsigned ntiny; + /* Number of bins supported by the lookup table. */ + int nlbins; + /* Number of small size class bins. */ + int nbins; + /* Number of size classes. */ + int nsizes; + /* Number of bits required to store NSIZES. */ + int lg_ceil_nsizes; + /* Number of size classes that are a multiple of (1U << LG_PAGE). */ + unsigned npsizes; + /* Lg of maximum tiny size class (or -1, if none). */ + int lg_tiny_maxclass; + /* Maximum size class included in lookup table. */ + size_t lookup_maxclass; + /* Maximum small size class. */ + size_t small_maxclass; + /* Lg of minimum large size class. */ + int lg_large_minclass; + /* The minimum large size class. */ + size_t large_minclass; + /* Maximum (large) size class. */ + size_t large_maxclass; + /* True if the sc_data_t has been initialized (for debugging only). */ + bool initialized; + + sc_t sc[SC_NSIZES]; +}; + +void sc_data_init(sc_data_t *data); +/* + * Updates slab sizes in [begin, end] to be pgs pages in length, if possible. + * Otherwise, does its best to accomodate the request. + */ +void sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, + int pgs); +void sc_boot(sc_data_t *data); + +#endif /* JEMALLOC_INTERNAL_SC_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/seq.h b/dep/jemalloc/include/jemalloc/internal/seq.h new file mode 100644 index 00000000000..ef2df4c6eef --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/seq.h @@ -0,0 +1,55 @@ +#ifndef JEMALLOC_INTERNAL_SEQ_H +#define JEMALLOC_INTERNAL_SEQ_H + +#include "jemalloc/internal/atomic.h" + +/* + * A simple seqlock implementation. + */ + +#define seq_define(type, short_type) \ +typedef struct { \ + atomic_zu_t seq; \ + atomic_zu_t data[ \ + (sizeof(type) + sizeof(size_t) - 1) / sizeof(size_t)]; \ +} seq_##short_type##_t; \ + \ +/* \ + * No internal synchronization -- the caller must ensure that there's \ + * only a single writer at a time. \ + */ \ +static inline void \ +seq_store_##short_type(seq_##short_type##_t *dst, type *src) { \ + size_t buf[sizeof(dst->data) / sizeof(size_t)]; \ + buf[sizeof(buf) / sizeof(size_t) - 1] = 0; \ + memcpy(buf, src, sizeof(type)); \ + size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED); \ + atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED); \ + atomic_fence(ATOMIC_RELEASE); \ + for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ + atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED); \ + } \ + atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE); \ +} \ + \ +/* Returns whether or not the read was consistent. */ \ +static inline bool \ +seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) { \ + size_t buf[sizeof(src->data) / sizeof(size_t)]; \ + size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE); \ + if (seq1 % 2 != 0) { \ + return false; \ + } \ + for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ + buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED); \ + } \ + atomic_fence(ATOMIC_ACQUIRE); \ + size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED); \ + if (seq1 != seq2) { \ + return false; \ + } \ + memcpy(dst, buf, sizeof(type)); \ + return true; \ +} + +#endif /* JEMALLOC_INTERNAL_SEQ_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/size_classes.h b/dep/jemalloc/include/jemalloc/internal/size_classes.h deleted file mode 100644 index 0b7d3cd369b..00000000000 --- a/dep/jemalloc/include/jemalloc/internal/size_classes.h +++ /dev/null @@ -1,1428 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_SIZE_CLASSES_H -#define JEMALLOC_INTERNAL_SIZE_CLASSES_H - -/* This file was automatically generated by size_classes.sh. */ - -#include "jemalloc/internal/jemalloc_internal_types.h" - -/* - * This header file defines: - * - * LG_SIZE_CLASS_GROUP: Lg of size class count for each size doubling. - * LG_TINY_MIN: Lg of minimum size class to support. - * SIZE_CLASSES: Complete table of SC(index, lg_grp, lg_delta, ndelta, psz, - * bin, pgs, lg_delta_lookup) tuples. - * index: Size class index. - * lg_grp: Lg group base size (no deltas added). - * lg_delta: Lg delta to previous size class. - * ndelta: Delta multiplier. size == 1<<lg_grp + ndelta<<lg_delta - * psz: 'yes' if a multiple of the page size, 'no' otherwise. - * bin: 'yes' if a small bin size class, 'no' otherwise. - * pgs: Slab page count if a small bin size class, 0 otherwise. - * lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no' - * otherwise. - * NTBINS: Number of tiny bins. - * NLBINS: Number of bins supported by the lookup table. - * NBINS: Number of small size class bins. - * NSIZES: Number of size classes. - * LG_CEIL_NSIZES: Number of bits required to store NSIZES. - * NPSIZES: Number of size classes that are a multiple of (1U << LG_PAGE). - * LG_TINY_MAXCLASS: Lg of maximum tiny size class. - * LOOKUP_MAXCLASS: Maximum size class included in lookup table. - * SMALL_MAXCLASS: Maximum small size class. - * LG_LARGE_MINCLASS: Lg of minimum large size class. - * LARGE_MAXCLASS: Maximum (large) size class. - */ - -#define LG_SIZE_CLASS_GROUP 2 -#define LG_TINY_MIN 3 - -#if (LG_SIZEOF_PTR == 2 && LG_TINY_MIN == 3 && LG_QUANTUM == 3 && LG_PAGE == 12) -#define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, no, yes, 1, 3) \ - SC( 1, 3, 3, 1, no, yes, 1, 3) \ - SC( 2, 3, 3, 2, no, yes, 3, 3) \ - SC( 3, 3, 3, 3, no, yes, 1, 3) \ - \ - SC( 4, 5, 3, 1, no, yes, 5, 3) \ - SC( 5, 5, 3, 2, no, yes, 3, 3) \ - SC( 6, 5, 3, 3, no, yes, 7, 3) \ - SC( 7, 5, 3, 4, no, yes, 1, 3) \ - \ - SC( 8, 6, 4, 1, no, yes, 5, 4) \ - SC( 9, 6, 4, 2, no, yes, 3, 4) \ - SC( 10, 6, 4, 3, no, yes, 7, 4) \ - SC( 11, 6, 4, 4, no, yes, 1, 4) \ - \ - SC( 12, 7, 5, 1, no, yes, 5, 5) \ - SC( 13, 7, 5, 2, no, yes, 3, 5) \ - SC( 14, 7, 5, 3, no, yes, 7, 5) \ - SC( 15, 7, 5, 4, no, yes, 1, 5) \ - \ - SC( 16, 8, 6, 1, no, yes, 5, 6) \ - SC( 17, 8, 6, 2, no, yes, 3, 6) \ - SC( 18, 8, 6, 3, no, yes, 7, 6) \ - SC( 19, 8, 6, 4, no, yes, 1, 6) \ - \ - SC( 20, 9, 7, 1, no, yes, 5, 7) \ - SC( 21, 9, 7, 2, no, yes, 3, 7) \ - SC( 22, 9, 7, 3, no, yes, 7, 7) \ - SC( 23, 9, 7, 4, no, yes, 1, 7) \ - \ - SC( 24, 10, 8, 1, no, yes, 5, 8) \ - SC( 25, 10, 8, 2, no, yes, 3, 8) \ - SC( 26, 10, 8, 3, no, yes, 7, 8) \ - SC( 27, 10, 8, 4, no, yes, 1, 8) \ - \ - SC( 28, 11, 9, 1, no, yes, 5, 9) \ - SC( 29, 11, 9, 2, no, yes, 3, 9) \ - SC( 30, 11, 9, 3, no, yes, 7, 9) \ - SC( 31, 11, 9, 4, yes, yes, 1, 9) \ - \ - SC( 32, 12, 10, 1, no, yes, 5, no) \ - SC( 33, 12, 10, 2, no, yes, 3, no) \ - SC( 34, 12, 10, 3, no, yes, 7, no) \ - SC( 35, 12, 10, 4, yes, yes, 2, no) \ - \ - SC( 36, 13, 11, 1, no, yes, 5, no) \ - SC( 37, 13, 11, 2, yes, yes, 3, no) \ - SC( 38, 13, 11, 3, no, yes, 7, no) \ - SC( 39, 13, 11, 4, yes, no, 0, no) \ - \ - SC( 40, 14, 12, 1, yes, no, 0, no) \ - SC( 41, 14, 12, 2, yes, no, 0, no) \ - SC( 42, 14, 12, 3, yes, no, 0, no) \ - SC( 43, 14, 12, 4, yes, no, 0, no) \ - \ - SC( 44, 15, 13, 1, yes, no, 0, no) \ - SC( 45, 15, 13, 2, yes, no, 0, no) \ - SC( 46, 15, 13, 3, yes, no, 0, no) \ - SC( 47, 15, 13, 4, yes, no, 0, no) \ - \ - SC( 48, 16, 14, 1, yes, no, 0, no) \ - SC( 49, 16, 14, 2, yes, no, 0, no) \ - SC( 50, 16, 14, 3, yes, no, 0, no) \ - SC( 51, 16, 14, 4, yes, no, 0, no) \ - \ - SC( 52, 17, 15, 1, yes, no, 0, no) \ - SC( 53, 17, 15, 2, yes, no, 0, no) \ - SC( 54, 17, 15, 3, yes, no, 0, no) \ - SC( 55, 17, 15, 4, yes, no, 0, no) \ - \ - SC( 56, 18, 16, 1, yes, no, 0, no) \ - SC( 57, 18, 16, 2, yes, no, 0, no) \ - SC( 58, 18, 16, 3, yes, no, 0, no) \ - SC( 59, 18, 16, 4, yes, no, 0, no) \ - \ - SC( 60, 19, 17, 1, yes, no, 0, no) \ - SC( 61, 19, 17, 2, yes, no, 0, no) \ - SC( 62, 19, 17, 3, yes, no, 0, no) \ - SC( 63, 19, 17, 4, yes, no, 0, no) \ - \ - SC( 64, 20, 18, 1, yes, no, 0, no) \ - SC( 65, 20, 18, 2, yes, no, 0, no) \ - SC( 66, 20, 18, 3, yes, no, 0, no) \ - SC( 67, 20, 18, 4, yes, no, 0, no) \ - \ - SC( 68, 21, 19, 1, yes, no, 0, no) \ - SC( 69, 21, 19, 2, yes, no, 0, no) \ - SC( 70, 21, 19, 3, yes, no, 0, no) \ - SC( 71, 21, 19, 4, yes, no, 0, no) \ - \ - SC( 72, 22, 20, 1, yes, no, 0, no) \ - SC( 73, 22, 20, 2, yes, no, 0, no) \ - SC( 74, 22, 20, 3, yes, no, 0, no) \ - SC( 75, 22, 20, 4, yes, no, 0, no) \ - \ - SC( 76, 23, 21, 1, yes, no, 0, no) \ - SC( 77, 23, 21, 2, yes, no, 0, no) \ - SC( 78, 23, 21, 3, yes, no, 0, no) \ - SC( 79, 23, 21, 4, yes, no, 0, no) \ - \ - SC( 80, 24, 22, 1, yes, no, 0, no) \ - SC( 81, 24, 22, 2, yes, no, 0, no) \ - SC( 82, 24, 22, 3, yes, no, 0, no) \ - SC( 83, 24, 22, 4, yes, no, 0, no) \ - \ - SC( 84, 25, 23, 1, yes, no, 0, no) \ - SC( 85, 25, 23, 2, yes, no, 0, no) \ - SC( 86, 25, 23, 3, yes, no, 0, no) \ - SC( 87, 25, 23, 4, yes, no, 0, no) \ - \ - SC( 88, 26, 24, 1, yes, no, 0, no) \ - SC( 89, 26, 24, 2, yes, no, 0, no) \ - SC( 90, 26, 24, 3, yes, no, 0, no) \ - SC( 91, 26, 24, 4, yes, no, 0, no) \ - \ - SC( 92, 27, 25, 1, yes, no, 0, no) \ - SC( 93, 27, 25, 2, yes, no, 0, no) \ - SC( 94, 27, 25, 3, yes, no, 0, no) \ - SC( 95, 27, 25, 4, yes, no, 0, no) \ - \ - SC( 96, 28, 26, 1, yes, no, 0, no) \ - SC( 97, 28, 26, 2, yes, no, 0, no) \ - SC( 98, 28, 26, 3, yes, no, 0, no) \ - SC( 99, 28, 26, 4, yes, no, 0, no) \ - \ - SC(100, 29, 27, 1, yes, no, 0, no) \ - SC(101, 29, 27, 2, yes, no, 0, no) \ - SC(102, 29, 27, 3, yes, no, 0, no) \ - SC(103, 29, 27, 4, yes, no, 0, no) \ - \ - SC(104, 30, 28, 1, yes, no, 0, no) \ - SC(105, 30, 28, 2, yes, no, 0, no) \ - SC(106, 30, 28, 3, yes, no, 0, no) \ - -#define SIZE_CLASSES_DEFINED -#define NTBINS 0 -#define NLBINS 32 -#define NBINS 39 -#define NSIZES 107 -#define LG_CEIL_NSIZES 7 -#define NPSIZES 71 -#define LG_TINY_MAXCLASS "NA" -#define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) -#define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) -#define LG_LARGE_MINCLASS 14 -#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -#define LARGE_MAXCLASS ((((size_t)1) << 30) + (((size_t)3) << 28)) -#endif - -#if (LG_SIZEOF_PTR == 2 && LG_TINY_MIN == 3 && LG_QUANTUM == 4 && LG_PAGE == 12) -#define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, no, yes, 1, 3) \ - \ - SC( 1, 3, 3, 1, no, yes, 1, 3) \ - SC( 2, 4, 4, 1, no, yes, 1, 4) \ - SC( 3, 4, 4, 2, no, yes, 3, 4) \ - SC( 4, 4, 4, 3, no, yes, 1, 4) \ - \ - SC( 5, 6, 4, 1, no, yes, 5, 4) \ - SC( 6, 6, 4, 2, no, yes, 3, 4) \ - SC( 7, 6, 4, 3, no, yes, 7, 4) \ - SC( 8, 6, 4, 4, no, yes, 1, 4) \ - \ - SC( 9, 7, 5, 1, no, yes, 5, 5) \ - SC( 10, 7, 5, 2, no, yes, 3, 5) \ - SC( 11, 7, 5, 3, no, yes, 7, 5) \ - SC( 12, 7, 5, 4, no, yes, 1, 5) \ - \ - SC( 13, 8, 6, 1, no, yes, 5, 6) \ - SC( 14, 8, 6, 2, no, yes, 3, 6) \ - SC( 15, 8, 6, 3, no, yes, 7, 6) \ - SC( 16, 8, 6, 4, no, yes, 1, 6) \ - \ - SC( 17, 9, 7, 1, no, yes, 5, 7) \ - SC( 18, 9, 7, 2, no, yes, 3, 7) \ - SC( 19, 9, 7, 3, no, yes, 7, 7) \ - SC( 20, 9, 7, 4, no, yes, 1, 7) \ - \ - SC( 21, 10, 8, 1, no, yes, 5, 8) \ - SC( 22, 10, 8, 2, no, yes, 3, 8) \ - SC( 23, 10, 8, 3, no, yes, 7, 8) \ - SC( 24, 10, 8, 4, no, yes, 1, 8) \ - \ - SC( 25, 11, 9, 1, no, yes, 5, 9) \ - SC( 26, 11, 9, 2, no, yes, 3, 9) \ - SC( 27, 11, 9, 3, no, yes, 7, 9) \ - SC( 28, 11, 9, 4, yes, yes, 1, 9) \ - \ - SC( 29, 12, 10, 1, no, yes, 5, no) \ - SC( 30, 12, 10, 2, no, yes, 3, no) \ - SC( 31, 12, 10, 3, no, yes, 7, no) \ - SC( 32, 12, 10, 4, yes, yes, 2, no) \ - \ - SC( 33, 13, 11, 1, no, yes, 5, no) \ - SC( 34, 13, 11, 2, yes, yes, 3, no) \ - SC( 35, 13, 11, 3, no, yes, 7, no) \ - SC( 36, 13, 11, 4, yes, no, 0, no) \ - \ - SC( 37, 14, 12, 1, yes, no, 0, no) \ - SC( 38, 14, 12, 2, yes, no, 0, no) \ - SC( 39, 14, 12, 3, yes, no, 0, no) \ - SC( 40, 14, 12, 4, yes, no, 0, no) \ - \ - SC( 41, 15, 13, 1, yes, no, 0, no) \ - SC( 42, 15, 13, 2, yes, no, 0, no) \ - SC( 43, 15, 13, 3, yes, no, 0, no) \ - SC( 44, 15, 13, 4, yes, no, 0, no) \ - \ - SC( 45, 16, 14, 1, yes, no, 0, no) \ - SC( 46, 16, 14, 2, yes, no, 0, no) \ - SC( 47, 16, 14, 3, yes, no, 0, no) \ - SC( 48, 16, 14, 4, yes, no, 0, no) \ - \ - SC( 49, 17, 15, 1, yes, no, 0, no) \ - SC( 50, 17, 15, 2, yes, no, 0, no) \ - SC( 51, 17, 15, 3, yes, no, 0, no) \ - SC( 52, 17, 15, 4, yes, no, 0, no) \ - \ - SC( 53, 18, 16, 1, yes, no, 0, no) \ - SC( 54, 18, 16, 2, yes, no, 0, no) \ - SC( 55, 18, 16, 3, yes, no, 0, no) \ - SC( 56, 18, 16, 4, yes, no, 0, no) \ - \ - SC( 57, 19, 17, 1, yes, no, 0, no) \ - SC( 58, 19, 17, 2, yes, no, 0, no) \ - SC( 59, 19, 17, 3, yes, no, 0, no) \ - SC( 60, 19, 17, 4, yes, no, 0, no) \ - \ - SC( 61, 20, 18, 1, yes, no, 0, no) \ - SC( 62, 20, 18, 2, yes, no, 0, no) \ - SC( 63, 20, 18, 3, yes, no, 0, no) \ - SC( 64, 20, 18, 4, yes, no, 0, no) \ - \ - SC( 65, 21, 19, 1, yes, no, 0, no) \ - SC( 66, 21, 19, 2, yes, no, 0, no) \ - SC( 67, 21, 19, 3, yes, no, 0, no) \ - SC( 68, 21, 19, 4, yes, no, 0, no) \ - \ - SC( 69, 22, 20, 1, yes, no, 0, no) \ - SC( 70, 22, 20, 2, yes, no, 0, no) \ - SC( 71, 22, 20, 3, yes, no, 0, no) \ - SC( 72, 22, 20, 4, yes, no, 0, no) \ - \ - SC( 73, 23, 21, 1, yes, no, 0, no) \ - SC( 74, 23, 21, 2, yes, no, 0, no) \ - SC( 75, 23, 21, 3, yes, no, 0, no) \ - SC( 76, 23, 21, 4, yes, no, 0, no) \ - \ - SC( 77, 24, 22, 1, yes, no, 0, no) \ - SC( 78, 24, 22, 2, yes, no, 0, no) \ - SC( 79, 24, 22, 3, yes, no, 0, no) \ - SC( 80, 24, 22, 4, yes, no, 0, no) \ - \ - SC( 81, 25, 23, 1, yes, no, 0, no) \ - SC( 82, 25, 23, 2, yes, no, 0, no) \ - SC( 83, 25, 23, 3, yes, no, 0, no) \ - SC( 84, 25, 23, 4, yes, no, 0, no) \ - \ - SC( 85, 26, 24, 1, yes, no, 0, no) \ - SC( 86, 26, 24, 2, yes, no, 0, no) \ - SC( 87, 26, 24, 3, yes, no, 0, no) \ - SC( 88, 26, 24, 4, yes, no, 0, no) \ - \ - SC( 89, 27, 25, 1, yes, no, 0, no) \ - SC( 90, 27, 25, 2, yes, no, 0, no) \ - SC( 91, 27, 25, 3, yes, no, 0, no) \ - SC( 92, 27, 25, 4, yes, no, 0, no) \ - \ - SC( 93, 28, 26, 1, yes, no, 0, no) \ - SC( 94, 28, 26, 2, yes, no, 0, no) \ - SC( 95, 28, 26, 3, yes, no, 0, no) \ - SC( 96, 28, 26, 4, yes, no, 0, no) \ - \ - SC( 97, 29, 27, 1, yes, no, 0, no) \ - SC( 98, 29, 27, 2, yes, no, 0, no) \ - SC( 99, 29, 27, 3, yes, no, 0, no) \ - SC(100, 29, 27, 4, yes, no, 0, no) \ - \ - SC(101, 30, 28, 1, yes, no, 0, no) \ - SC(102, 30, 28, 2, yes, no, 0, no) \ - SC(103, 30, 28, 3, yes, no, 0, no) \ - -#define SIZE_CLASSES_DEFINED -#define NTBINS 1 -#define NLBINS 29 -#define NBINS 36 -#define NSIZES 104 -#define LG_CEIL_NSIZES 7 -#define NPSIZES 71 -#define LG_TINY_MAXCLASS 3 -#define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) -#define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) -#define LG_LARGE_MINCLASS 14 -#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -#define LARGE_MAXCLASS ((((size_t)1) << 30) + (((size_t)3) << 28)) -#endif - -#if (LG_SIZEOF_PTR == 2 && LG_TINY_MIN == 4 && LG_QUANTUM == 4 && LG_PAGE == 12) -#define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \ - SC( 0, 4, 4, 0, no, yes, 1, 4) \ - SC( 1, 4, 4, 1, no, yes, 1, 4) \ - SC( 2, 4, 4, 2, no, yes, 3, 4) \ - SC( 3, 4, 4, 3, no, yes, 1, 4) \ - \ - SC( 4, 6, 4, 1, no, yes, 5, 4) \ - SC( 5, 6, 4, 2, no, yes, 3, 4) \ - SC( 6, 6, 4, 3, no, yes, 7, 4) \ - SC( 7, 6, 4, 4, no, yes, 1, 4) \ - \ - SC( 8, 7, 5, 1, no, yes, 5, 5) \ - SC( 9, 7, 5, 2, no, yes, 3, 5) \ - SC( 10, 7, 5, 3, no, yes, 7, 5) \ - SC( 11, 7, 5, 4, no, yes, 1, 5) \ - \ - SC( 12, 8, 6, 1, no, yes, 5, 6) \ - SC( 13, 8, 6, 2, no, yes, 3, 6) \ - SC( 14, 8, 6, 3, no, yes, 7, 6) \ - SC( 15, 8, 6, 4, no, yes, 1, 6) \ - \ - SC( 16, 9, 7, 1, no, yes, 5, 7) \ - SC( 17, 9, 7, 2, no, yes, 3, 7) \ - SC( 18, 9, 7, 3, no, yes, 7, 7) \ - SC( 19, 9, 7, 4, no, yes, 1, 7) \ - \ - SC( 20, 10, 8, 1, no, yes, 5, 8) \ - SC( 21, 10, 8, 2, no, yes, 3, 8) \ - SC( 22, 10, 8, 3, no, yes, 7, 8) \ - SC( 23, 10, 8, 4, no, yes, 1, 8) \ - \ - SC( 24, 11, 9, 1, no, yes, 5, 9) \ - SC( 25, 11, 9, 2, no, yes, 3, 9) \ - SC( 26, 11, 9, 3, no, yes, 7, 9) \ - SC( 27, 11, 9, 4, yes, yes, 1, 9) \ - \ - SC( 28, 12, 10, 1, no, yes, 5, no) \ - SC( 29, 12, 10, 2, no, yes, 3, no) \ - SC( 30, 12, 10, 3, no, yes, 7, no) \ - SC( 31, 12, 10, 4, yes, yes, 2, no) \ - \ - SC( 32, 13, 11, 1, no, yes, 5, no) \ - SC( 33, 13, 11, 2, yes, yes, 3, no) \ - SC( 34, 13, 11, 3, no, yes, 7, no) \ - SC( 35, 13, 11, 4, yes, no, 0, no) \ - \ - SC( 36, 14, 12, 1, yes, no, 0, no) \ - SC( 37, 14, 12, 2, yes, no, 0, no) \ - SC( 38, 14, 12, 3, yes, no, 0, no) \ - SC( 39, 14, 12, 4, yes, no, 0, no) \ - \ - SC( 40, 15, 13, 1, yes, no, 0, no) \ - SC( 41, 15, 13, 2, yes, no, 0, no) \ - SC( 42, 15, 13, 3, yes, no, 0, no) \ - SC( 43, 15, 13, 4, yes, no, 0, no) \ - \ - SC( 44, 16, 14, 1, yes, no, 0, no) \ - SC( 45, 16, 14, 2, yes, no, 0, no) \ - SC( 46, 16, 14, 3, yes, no, 0, no) \ - SC( 47, 16, 14, 4, yes, no, 0, no) \ - \ - SC( 48, 17, 15, 1, yes, no, 0, no) \ - SC( 49, 17, 15, 2, yes, no, 0, no) \ - SC( 50, 17, 15, 3, yes, no, 0, no) \ - SC( 51, 17, 15, 4, yes, no, 0, no) \ - \ - SC( 52, 18, 16, 1, yes, no, 0, no) \ - SC( 53, 18, 16, 2, yes, no, 0, no) \ - SC( 54, 18, 16, 3, yes, no, 0, no) \ - SC( 55, 18, 16, 4, yes, no, 0, no) \ - \ - SC( 56, 19, 17, 1, yes, no, 0, no) \ - SC( 57, 19, 17, 2, yes, no, 0, no) \ - SC( 58, 19, 17, 3, yes, no, 0, no) \ - SC( 59, 19, 17, 4, yes, no, 0, no) \ - \ - SC( 60, 20, 18, 1, yes, no, 0, no) \ - SC( 61, 20, 18, 2, yes, no, 0, no) \ - SC( 62, 20, 18, 3, yes, no, 0, no) \ - SC( 63, 20, 18, 4, yes, no, 0, no) \ - \ - SC( 64, 21, 19, 1, yes, no, 0, no) \ - SC( 65, 21, 19, 2, yes, no, 0, no) \ - SC( 66, 21, 19, 3, yes, no, 0, no) \ - SC( 67, 21, 19, 4, yes, no, 0, no) \ - \ - SC( 68, 22, 20, 1, yes, no, 0, no) \ - SC( 69, 22, 20, 2, yes, no, 0, no) \ - SC( 70, 22, 20, 3, yes, no, 0, no) \ - SC( 71, 22, 20, 4, yes, no, 0, no) \ - \ - SC( 72, 23, 21, 1, yes, no, 0, no) \ - SC( 73, 23, 21, 2, yes, no, 0, no) \ - SC( 74, 23, 21, 3, yes, no, 0, no) \ - SC( 75, 23, 21, 4, yes, no, 0, no) \ - \ - SC( 76, 24, 22, 1, yes, no, 0, no) \ - SC( 77, 24, 22, 2, yes, no, 0, no) \ - SC( 78, 24, 22, 3, yes, no, 0, no) \ - SC( 79, 24, 22, 4, yes, no, 0, no) \ - \ - SC( 80, 25, 23, 1, yes, no, 0, no) \ - SC( 81, 25, 23, 2, yes, no, 0, no) \ - SC( 82, 25, 23, 3, yes, no, 0, no) \ - SC( 83, 25, 23, 4, yes, no, 0, no) \ - \ - SC( 84, 26, 24, 1, yes, no, 0, no) \ - SC( 85, 26, 24, 2, yes, no, 0, no) \ - SC( 86, 26, 24, 3, yes, no, 0, no) \ - SC( 87, 26, 24, 4, yes, no, 0, no) \ - \ - SC( 88, 27, 25, 1, yes, no, 0, no) \ - SC( 89, 27, 25, 2, yes, no, 0, no) \ - SC( 90, 27, 25, 3, yes, no, 0, no) \ - SC( 91, 27, 25, 4, yes, no, 0, no) \ - \ - SC( 92, 28, 26, 1, yes, no, 0, no) \ - SC( 93, 28, 26, 2, yes, no, 0, no) \ - SC( 94, 28, 26, 3, yes, no, 0, no) \ - SC( 95, 28, 26, 4, yes, no, 0, no) \ - \ - SC( 96, 29, 27, 1, yes, no, 0, no) \ - SC( 97, 29, 27, 2, yes, no, 0, no) \ - SC( 98, 29, 27, 3, yes, no, 0, no) \ - SC( 99, 29, 27, 4, yes, no, 0, no) \ - \ - SC(100, 30, 28, 1, yes, no, 0, no) \ - SC(101, 30, 28, 2, yes, no, 0, no) \ - SC(102, 30, 28, 3, yes, no, 0, no) \ - -#define SIZE_CLASSES_DEFINED -#define NTBINS 0 -#define NLBINS 28 -#define NBINS 35 -#define NSIZES 103 -#define LG_CEIL_NSIZES 7 -#define NPSIZES 71 -#define LG_TINY_MAXCLASS "NA" -#define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) -#define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) -#define LG_LARGE_MINCLASS 14 -#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -#define LARGE_MAXCLASS ((((size_t)1) << 30) + (((size_t)3) << 28)) -#endif - -#if (LG_SIZEOF_PTR == 3 && LG_TINY_MIN == 3 && LG_QUANTUM == 3 && LG_PAGE == 12) -#define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, no, yes, 1, 3) \ - SC( 1, 3, 3, 1, no, yes, 1, 3) \ - SC( 2, 3, 3, 2, no, yes, 3, 3) \ - SC( 3, 3, 3, 3, no, yes, 1, 3) \ - \ - SC( 4, 5, 3, 1, no, yes, 5, 3) \ - SC( 5, 5, 3, 2, no, yes, 3, 3) \ - SC( 6, 5, 3, 3, no, yes, 7, 3) \ - SC( 7, 5, 3, 4, no, yes, 1, 3) \ - \ - SC( 8, 6, 4, 1, no, yes, 5, 4) \ - SC( 9, 6, 4, 2, no, yes, 3, 4) \ - SC( 10, 6, 4, 3, no, yes, 7, 4) \ - SC( 11, 6, 4, 4, no, yes, 1, 4) \ - \ - SC( 12, 7, 5, 1, no, yes, 5, 5) \ - SC( 13, 7, 5, 2, no, yes, 3, 5) \ - SC( 14, 7, 5, 3, no, yes, 7, 5) \ - SC( 15, 7, 5, 4, no, yes, 1, 5) \ - \ - SC( 16, 8, 6, 1, no, yes, 5, 6) \ - SC( 17, 8, 6, 2, no, yes, 3, 6) \ - SC( 18, 8, 6, 3, no, yes, 7, 6) \ - SC( 19, 8, 6, 4, no, yes, 1, 6) \ - \ - SC( 20, 9, 7, 1, no, yes, 5, 7) \ - SC( 21, 9, 7, 2, no, yes, 3, 7) \ - SC( 22, 9, 7, 3, no, yes, 7, 7) \ - SC( 23, 9, 7, 4, no, yes, 1, 7) \ - \ - SC( 24, 10, 8, 1, no, yes, 5, 8) \ - SC( 25, 10, 8, 2, no, yes, 3, 8) \ - SC( 26, 10, 8, 3, no, yes, 7, 8) \ - SC( 27, 10, 8, 4, no, yes, 1, 8) \ - \ - SC( 28, 11, 9, 1, no, yes, 5, 9) \ - SC( 29, 11, 9, 2, no, yes, 3, 9) \ - SC( 30, 11, 9, 3, no, yes, 7, 9) \ - SC( 31, 11, 9, 4, yes, yes, 1, 9) \ - \ - SC( 32, 12, 10, 1, no, yes, 5, no) \ - SC( 33, 12, 10, 2, no, yes, 3, no) \ - SC( 34, 12, 10, 3, no, yes, 7, no) \ - SC( 35, 12, 10, 4, yes, yes, 2, no) \ - \ - SC( 36, 13, 11, 1, no, yes, 5, no) \ - SC( 37, 13, 11, 2, yes, yes, 3, no) \ - SC( 38, 13, 11, 3, no, yes, 7, no) \ - SC( 39, 13, 11, 4, yes, no, 0, no) \ - \ - SC( 40, 14, 12, 1, yes, no, 0, no) \ - SC( 41, 14, 12, 2, yes, no, 0, no) \ - SC( 42, 14, 12, 3, yes, no, 0, no) \ - SC( 43, 14, 12, 4, yes, no, 0, no) \ - \ - SC( 44, 15, 13, 1, yes, no, 0, no) \ - SC( 45, 15, 13, 2, yes, no, 0, no) \ - SC( 46, 15, 13, 3, yes, no, 0, no) \ - SC( 47, 15, 13, 4, yes, no, 0, no) \ - \ - SC( 48, 16, 14, 1, yes, no, 0, no) \ - SC( 49, 16, 14, 2, yes, no, 0, no) \ - SC( 50, 16, 14, 3, yes, no, 0, no) \ - SC( 51, 16, 14, 4, yes, no, 0, no) \ - \ - SC( 52, 17, 15, 1, yes, no, 0, no) \ - SC( 53, 17, 15, 2, yes, no, 0, no) \ - SC( 54, 17, 15, 3, yes, no, 0, no) \ - SC( 55, 17, 15, 4, yes, no, 0, no) \ - \ - SC( 56, 18, 16, 1, yes, no, 0, no) \ - SC( 57, 18, 16, 2, yes, no, 0, no) \ - SC( 58, 18, 16, 3, yes, no, 0, no) \ - SC( 59, 18, 16, 4, yes, no, 0, no) \ - \ - SC( 60, 19, 17, 1, yes, no, 0, no) \ - SC( 61, 19, 17, 2, yes, no, 0, no) \ - SC( 62, 19, 17, 3, yes, no, 0, no) \ - SC( 63, 19, 17, 4, yes, no, 0, no) \ - \ - SC( 64, 20, 18, 1, yes, no, 0, no) \ - SC( 65, 20, 18, 2, yes, no, 0, no) \ - SC( 66, 20, 18, 3, yes, no, 0, no) \ - SC( 67, 20, 18, 4, yes, no, 0, no) \ - \ - SC( 68, 21, 19, 1, yes, no, 0, no) \ - SC( 69, 21, 19, 2, yes, no, 0, no) \ - SC( 70, 21, 19, 3, yes, no, 0, no) \ - SC( 71, 21, 19, 4, yes, no, 0, no) \ - \ - SC( 72, 22, 20, 1, yes, no, 0, no) \ - SC( 73, 22, 20, 2, yes, no, 0, no) \ - SC( 74, 22, 20, 3, yes, no, 0, no) \ - SC( 75, 22, 20, 4, yes, no, 0, no) \ - \ - SC( 76, 23, 21, 1, yes, no, 0, no) \ - SC( 77, 23, 21, 2, yes, no, 0, no) \ - SC( 78, 23, 21, 3, yes, no, 0, no) \ - SC( 79, 23, 21, 4, yes, no, 0, no) \ - \ - SC( 80, 24, 22, 1, yes, no, 0, no) \ - SC( 81, 24, 22, 2, yes, no, 0, no) \ - SC( 82, 24, 22, 3, yes, no, 0, no) \ - SC( 83, 24, 22, 4, yes, no, 0, no) \ - \ - SC( 84, 25, 23, 1, yes, no, 0, no) \ - SC( 85, 25, 23, 2, yes, no, 0, no) \ - SC( 86, 25, 23, 3, yes, no, 0, no) \ - SC( 87, 25, 23, 4, yes, no, 0, no) \ - \ - SC( 88, 26, 24, 1, yes, no, 0, no) \ - SC( 89, 26, 24, 2, yes, no, 0, no) \ - SC( 90, 26, 24, 3, yes, no, 0, no) \ - SC( 91, 26, 24, 4, yes, no, 0, no) \ - \ - SC( 92, 27, 25, 1, yes, no, 0, no) \ - SC( 93, 27, 25, 2, yes, no, 0, no) \ - SC( 94, 27, 25, 3, yes, no, 0, no) \ - SC( 95, 27, 25, 4, yes, no, 0, no) \ - \ - SC( 96, 28, 26, 1, yes, no, 0, no) \ - SC( 97, 28, 26, 2, yes, no, 0, no) \ - SC( 98, 28, 26, 3, yes, no, 0, no) \ - SC( 99, 28, 26, 4, yes, no, 0, no) \ - \ - SC(100, 29, 27, 1, yes, no, 0, no) \ - SC(101, 29, 27, 2, yes, no, 0, no) \ - SC(102, 29, 27, 3, yes, no, 0, no) \ - SC(103, 29, 27, 4, yes, no, 0, no) \ - \ - SC(104, 30, 28, 1, yes, no, 0, no) \ - SC(105, 30, 28, 2, yes, no, 0, no) \ - SC(106, 30, 28, 3, yes, no, 0, no) \ - SC(107, 30, 28, 4, yes, no, 0, no) \ - \ - SC(108, 31, 29, 1, yes, no, 0, no) \ - SC(109, 31, 29, 2, yes, no, 0, no) \ - SC(110, 31, 29, 3, yes, no, 0, no) \ - SC(111, 31, 29, 4, yes, no, 0, no) \ - \ - SC(112, 32, 30, 1, yes, no, 0, no) \ - SC(113, 32, 30, 2, yes, no, 0, no) \ - SC(114, 32, 30, 3, yes, no, 0, no) \ - SC(115, 32, 30, 4, yes, no, 0, no) \ - \ - SC(116, 33, 31, 1, yes, no, 0, no) \ - SC(117, 33, 31, 2, yes, no, 0, no) \ - SC(118, 33, 31, 3, yes, no, 0, no) \ - SC(119, 33, 31, 4, yes, no, 0, no) \ - \ - SC(120, 34, 32, 1, yes, no, 0, no) \ - SC(121, 34, 32, 2, yes, no, 0, no) \ - SC(122, 34, 32, 3, yes, no, 0, no) \ - SC(123, 34, 32, 4, yes, no, 0, no) \ - \ - SC(124, 35, 33, 1, yes, no, 0, no) \ - SC(125, 35, 33, 2, yes, no, 0, no) \ - SC(126, 35, 33, 3, yes, no, 0, no) \ - SC(127, 35, 33, 4, yes, no, 0, no) \ - \ - SC(128, 36, 34, 1, yes, no, 0, no) \ - SC(129, 36, 34, 2, yes, no, 0, no) \ - SC(130, 36, 34, 3, yes, no, 0, no) \ - SC(131, 36, 34, 4, yes, no, 0, no) \ - \ - SC(132, 37, 35, 1, yes, no, 0, no) \ - SC(133, 37, 35, 2, yes, no, 0, no) \ - SC(134, 37, 35, 3, yes, no, 0, no) \ - SC(135, 37, 35, 4, yes, no, 0, no) \ - \ - SC(136, 38, 36, 1, yes, no, 0, no) \ - SC(137, 38, 36, 2, yes, no, 0, no) \ - SC(138, 38, 36, 3, yes, no, 0, no) \ - SC(139, 38, 36, 4, yes, no, 0, no) \ - \ - SC(140, 39, 37, 1, yes, no, 0, no) \ - SC(141, 39, 37, 2, yes, no, 0, no) \ - SC(142, 39, 37, 3, yes, no, 0, no) \ - SC(143, 39, 37, 4, yes, no, 0, no) \ - \ - SC(144, 40, 38, 1, yes, no, 0, no) \ - SC(145, 40, 38, 2, yes, no, 0, no) \ - SC(146, 40, 38, 3, yes, no, 0, no) \ - SC(147, 40, 38, 4, yes, no, 0, no) \ - \ - SC(148, 41, 39, 1, yes, no, 0, no) \ - SC(149, 41, 39, 2, yes, no, 0, no) \ - SC(150, 41, 39, 3, yes, no, 0, no) \ - SC(151, 41, 39, 4, yes, no, 0, no) \ - \ - SC(152, 42, 40, 1, yes, no, 0, no) \ - SC(153, 42, 40, 2, yes, no, 0, no) \ - SC(154, 42, 40, 3, yes, no, 0, no) \ - SC(155, 42, 40, 4, yes, no, 0, no) \ - \ - SC(156, 43, 41, 1, yes, no, 0, no) \ - SC(157, 43, 41, 2, yes, no, 0, no) \ - SC(158, 43, 41, 3, yes, no, 0, no) \ - SC(159, 43, 41, 4, yes, no, 0, no) \ - \ - SC(160, 44, 42, 1, yes, no, 0, no) \ - SC(161, 44, 42, 2, yes, no, 0, no) \ - SC(162, 44, 42, 3, yes, no, 0, no) \ - SC(163, 44, 42, 4, yes, no, 0, no) \ - \ - SC(164, 45, 43, 1, yes, no, 0, no) \ - SC(165, 45, 43, 2, yes, no, 0, no) \ - SC(166, 45, 43, 3, yes, no, 0, no) \ - SC(167, 45, 43, 4, yes, no, 0, no) \ - \ - SC(168, 46, 44, 1, yes, no, 0, no) \ - SC(169, 46, 44, 2, yes, no, 0, no) \ - SC(170, 46, 44, 3, yes, no, 0, no) \ - SC(171, 46, 44, 4, yes, no, 0, no) \ - \ - SC(172, 47, 45, 1, yes, no, 0, no) \ - SC(173, 47, 45, 2, yes, no, 0, no) \ - SC(174, 47, 45, 3, yes, no, 0, no) \ - SC(175, 47, 45, 4, yes, no, 0, no) \ - \ - SC(176, 48, 46, 1, yes, no, 0, no) \ - SC(177, 48, 46, 2, yes, no, 0, no) \ - SC(178, 48, 46, 3, yes, no, 0, no) \ - SC(179, 48, 46, 4, yes, no, 0, no) \ - \ - SC(180, 49, 47, 1, yes, no, 0, no) \ - SC(181, 49, 47, 2, yes, no, 0, no) \ - SC(182, 49, 47, 3, yes, no, 0, no) \ - SC(183, 49, 47, 4, yes, no, 0, no) \ - \ - SC(184, 50, 48, 1, yes, no, 0, no) \ - SC(185, 50, 48, 2, yes, no, 0, no) \ - SC(186, 50, 48, 3, yes, no, 0, no) \ - SC(187, 50, 48, 4, yes, no, 0, no) \ - \ - SC(188, 51, 49, 1, yes, no, 0, no) \ - SC(189, 51, 49, 2, yes, no, 0, no) \ - SC(190, 51, 49, 3, yes, no, 0, no) \ - SC(191, 51, 49, 4, yes, no, 0, no) \ - \ - SC(192, 52, 50, 1, yes, no, 0, no) \ - SC(193, 52, 50, 2, yes, no, 0, no) \ - SC(194, 52, 50, 3, yes, no, 0, no) \ - SC(195, 52, 50, 4, yes, no, 0, no) \ - \ - SC(196, 53, 51, 1, yes, no, 0, no) \ - SC(197, 53, 51, 2, yes, no, 0, no) \ - SC(198, 53, 51, 3, yes, no, 0, no) \ - SC(199, 53, 51, 4, yes, no, 0, no) \ - \ - SC(200, 54, 52, 1, yes, no, 0, no) \ - SC(201, 54, 52, 2, yes, no, 0, no) \ - SC(202, 54, 52, 3, yes, no, 0, no) \ - SC(203, 54, 52, 4, yes, no, 0, no) \ - \ - SC(204, 55, 53, 1, yes, no, 0, no) \ - SC(205, 55, 53, 2, yes, no, 0, no) \ - SC(206, 55, 53, 3, yes, no, 0, no) \ - SC(207, 55, 53, 4, yes, no, 0, no) \ - \ - SC(208, 56, 54, 1, yes, no, 0, no) \ - SC(209, 56, 54, 2, yes, no, 0, no) \ - SC(210, 56, 54, 3, yes, no, 0, no) \ - SC(211, 56, 54, 4, yes, no, 0, no) \ - \ - SC(212, 57, 55, 1, yes, no, 0, no) \ - SC(213, 57, 55, 2, yes, no, 0, no) \ - SC(214, 57, 55, 3, yes, no, 0, no) \ - SC(215, 57, 55, 4, yes, no, 0, no) \ - \ - SC(216, 58, 56, 1, yes, no, 0, no) \ - SC(217, 58, 56, 2, yes, no, 0, no) \ - SC(218, 58, 56, 3, yes, no, 0, no) \ - SC(219, 58, 56, 4, yes, no, 0, no) \ - \ - SC(220, 59, 57, 1, yes, no, 0, no) \ - SC(221, 59, 57, 2, yes, no, 0, no) \ - SC(222, 59, 57, 3, yes, no, 0, no) \ - SC(223, 59, 57, 4, yes, no, 0, no) \ - \ - SC(224, 60, 58, 1, yes, no, 0, no) \ - SC(225, 60, 58, 2, yes, no, 0, no) \ - SC(226, 60, 58, 3, yes, no, 0, no) \ - SC(227, 60, 58, 4, yes, no, 0, no) \ - \ - SC(228, 61, 59, 1, yes, no, 0, no) \ - SC(229, 61, 59, 2, yes, no, 0, no) \ - SC(230, 61, 59, 3, yes, no, 0, no) \ - SC(231, 61, 59, 4, yes, no, 0, no) \ - \ - SC(232, 62, 60, 1, yes, no, 0, no) \ - SC(233, 62, 60, 2, yes, no, 0, no) \ - SC(234, 62, 60, 3, yes, no, 0, no) \ - -#define SIZE_CLASSES_DEFINED -#define NTBINS 0 -#define NLBINS 32 -#define NBINS 39 -#define NSIZES 235 -#define LG_CEIL_NSIZES 8 -#define NPSIZES 199 -#define LG_TINY_MAXCLASS "NA" -#define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) -#define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) -#define LG_LARGE_MINCLASS 14 -#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -#define LARGE_MAXCLASS ((((size_t)1) << 62) + (((size_t)3) << 60)) -#endif - -#if (LG_SIZEOF_PTR == 3 && LG_TINY_MIN == 3 && LG_QUANTUM == 4 && LG_PAGE == 12) -#define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, no, yes, 1, 3) \ - \ - SC( 1, 3, 3, 1, no, yes, 1, 3) \ - SC( 2, 4, 4, 1, no, yes, 1, 4) \ - SC( 3, 4, 4, 2, no, yes, 3, 4) \ - SC( 4, 4, 4, 3, no, yes, 1, 4) \ - \ - SC( 5, 6, 4, 1, no, yes, 5, 4) \ - SC( 6, 6, 4, 2, no, yes, 3, 4) \ - SC( 7, 6, 4, 3, no, yes, 7, 4) \ - SC( 8, 6, 4, 4, no, yes, 1, 4) \ - \ - SC( 9, 7, 5, 1, no, yes, 5, 5) \ - SC( 10, 7, 5, 2, no, yes, 3, 5) \ - SC( 11, 7, 5, 3, no, yes, 7, 5) \ - SC( 12, 7, 5, 4, no, yes, 1, 5) \ - \ - SC( 13, 8, 6, 1, no, yes, 5, 6) \ - SC( 14, 8, 6, 2, no, yes, 3, 6) \ - SC( 15, 8, 6, 3, no, yes, 7, 6) \ - SC( 16, 8, 6, 4, no, yes, 1, 6) \ - \ - SC( 17, 9, 7, 1, no, yes, 5, 7) \ - SC( 18, 9, 7, 2, no, yes, 3, 7) \ - SC( 19, 9, 7, 3, no, yes, 7, 7) \ - SC( 20, 9, 7, 4, no, yes, 1, 7) \ - \ - SC( 21, 10, 8, 1, no, yes, 5, 8) \ - SC( 22, 10, 8, 2, no, yes, 3, 8) \ - SC( 23, 10, 8, 3, no, yes, 7, 8) \ - SC( 24, 10, 8, 4, no, yes, 1, 8) \ - \ - SC( 25, 11, 9, 1, no, yes, 5, 9) \ - SC( 26, 11, 9, 2, no, yes, 3, 9) \ - SC( 27, 11, 9, 3, no, yes, 7, 9) \ - SC( 28, 11, 9, 4, yes, yes, 1, 9) \ - \ - SC( 29, 12, 10, 1, no, yes, 5, no) \ - SC( 30, 12, 10, 2, no, yes, 3, no) \ - SC( 31, 12, 10, 3, no, yes, 7, no) \ - SC( 32, 12, 10, 4, yes, yes, 2, no) \ - \ - SC( 33, 13, 11, 1, no, yes, 5, no) \ - SC( 34, 13, 11, 2, yes, yes, 3, no) \ - SC( 35, 13, 11, 3, no, yes, 7, no) \ - SC( 36, 13, 11, 4, yes, no, 0, no) \ - \ - SC( 37, 14, 12, 1, yes, no, 0, no) \ - SC( 38, 14, 12, 2, yes, no, 0, no) \ - SC( 39, 14, 12, 3, yes, no, 0, no) \ - SC( 40, 14, 12, 4, yes, no, 0, no) \ - \ - SC( 41, 15, 13, 1, yes, no, 0, no) \ - SC( 42, 15, 13, 2, yes, no, 0, no) \ - SC( 43, 15, 13, 3, yes, no, 0, no) \ - SC( 44, 15, 13, 4, yes, no, 0, no) \ - \ - SC( 45, 16, 14, 1, yes, no, 0, no) \ - SC( 46, 16, 14, 2, yes, no, 0, no) \ - SC( 47, 16, 14, 3, yes, no, 0, no) \ - SC( 48, 16, 14, 4, yes, no, 0, no) \ - \ - SC( 49, 17, 15, 1, yes, no, 0, no) \ - SC( 50, 17, 15, 2, yes, no, 0, no) \ - SC( 51, 17, 15, 3, yes, no, 0, no) \ - SC( 52, 17, 15, 4, yes, no, 0, no) \ - \ - SC( 53, 18, 16, 1, yes, no, 0, no) \ - SC( 54, 18, 16, 2, yes, no, 0, no) \ - SC( 55, 18, 16, 3, yes, no, 0, no) \ - SC( 56, 18, 16, 4, yes, no, 0, no) \ - \ - SC( 57, 19, 17, 1, yes, no, 0, no) \ - SC( 58, 19, 17, 2, yes, no, 0, no) \ - SC( 59, 19, 17, 3, yes, no, 0, no) \ - SC( 60, 19, 17, 4, yes, no, 0, no) \ - \ - SC( 61, 20, 18, 1, yes, no, 0, no) \ - SC( 62, 20, 18, 2, yes, no, 0, no) \ - SC( 63, 20, 18, 3, yes, no, 0, no) \ - SC( 64, 20, 18, 4, yes, no, 0, no) \ - \ - SC( 65, 21, 19, 1, yes, no, 0, no) \ - SC( 66, 21, 19, 2, yes, no, 0, no) \ - SC( 67, 21, 19, 3, yes, no, 0, no) \ - SC( 68, 21, 19, 4, yes, no, 0, no) \ - \ - SC( 69, 22, 20, 1, yes, no, 0, no) \ - SC( 70, 22, 20, 2, yes, no, 0, no) \ - SC( 71, 22, 20, 3, yes, no, 0, no) \ - SC( 72, 22, 20, 4, yes, no, 0, no) \ - \ - SC( 73, 23, 21, 1, yes, no, 0, no) \ - SC( 74, 23, 21, 2, yes, no, 0, no) \ - SC( 75, 23, 21, 3, yes, no, 0, no) \ - SC( 76, 23, 21, 4, yes, no, 0, no) \ - \ - SC( 77, 24, 22, 1, yes, no, 0, no) \ - SC( 78, 24, 22, 2, yes, no, 0, no) \ - SC( 79, 24, 22, 3, yes, no, 0, no) \ - SC( 80, 24, 22, 4, yes, no, 0, no) \ - \ - SC( 81, 25, 23, 1, yes, no, 0, no) \ - SC( 82, 25, 23, 2, yes, no, 0, no) \ - SC( 83, 25, 23, 3, yes, no, 0, no) \ - SC( 84, 25, 23, 4, yes, no, 0, no) \ - \ - SC( 85, 26, 24, 1, yes, no, 0, no) \ - SC( 86, 26, 24, 2, yes, no, 0, no) \ - SC( 87, 26, 24, 3, yes, no, 0, no) \ - SC( 88, 26, 24, 4, yes, no, 0, no) \ - \ - SC( 89, 27, 25, 1, yes, no, 0, no) \ - SC( 90, 27, 25, 2, yes, no, 0, no) \ - SC( 91, 27, 25, 3, yes, no, 0, no) \ - SC( 92, 27, 25, 4, yes, no, 0, no) \ - \ - SC( 93, 28, 26, 1, yes, no, 0, no) \ - SC( 94, 28, 26, 2, yes, no, 0, no) \ - SC( 95, 28, 26, 3, yes, no, 0, no) \ - SC( 96, 28, 26, 4, yes, no, 0, no) \ - \ - SC( 97, 29, 27, 1, yes, no, 0, no) \ - SC( 98, 29, 27, 2, yes, no, 0, no) \ - SC( 99, 29, 27, 3, yes, no, 0, no) \ - SC(100, 29, 27, 4, yes, no, 0, no) \ - \ - SC(101, 30, 28, 1, yes, no, 0, no) \ - SC(102, 30, 28, 2, yes, no, 0, no) \ - SC(103, 30, 28, 3, yes, no, 0, no) \ - SC(104, 30, 28, 4, yes, no, 0, no) \ - \ - SC(105, 31, 29, 1, yes, no, 0, no) \ - SC(106, 31, 29, 2, yes, no, 0, no) \ - SC(107, 31, 29, 3, yes, no, 0, no) \ - SC(108, 31, 29, 4, yes, no, 0, no) \ - \ - SC(109, 32, 30, 1, yes, no, 0, no) \ - SC(110, 32, 30, 2, yes, no, 0, no) \ - SC(111, 32, 30, 3, yes, no, 0, no) \ - SC(112, 32, 30, 4, yes, no, 0, no) \ - \ - SC(113, 33, 31, 1, yes, no, 0, no) \ - SC(114, 33, 31, 2, yes, no, 0, no) \ - SC(115, 33, 31, 3, yes, no, 0, no) \ - SC(116, 33, 31, 4, yes, no, 0, no) \ - \ - SC(117, 34, 32, 1, yes, no, 0, no) \ - SC(118, 34, 32, 2, yes, no, 0, no) \ - SC(119, 34, 32, 3, yes, no, 0, no) \ - SC(120, 34, 32, 4, yes, no, 0, no) \ - \ - SC(121, 35, 33, 1, yes, no, 0, no) \ - SC(122, 35, 33, 2, yes, no, 0, no) \ - SC(123, 35, 33, 3, yes, no, 0, no) \ - SC(124, 35, 33, 4, yes, no, 0, no) \ - \ - SC(125, 36, 34, 1, yes, no, 0, no) \ - SC(126, 36, 34, 2, yes, no, 0, no) \ - SC(127, 36, 34, 3, yes, no, 0, no) \ - SC(128, 36, 34, 4, yes, no, 0, no) \ - \ - SC(129, 37, 35, 1, yes, no, 0, no) \ - SC(130, 37, 35, 2, yes, no, 0, no) \ - SC(131, 37, 35, 3, yes, no, 0, no) \ - SC(132, 37, 35, 4, yes, no, 0, no) \ - \ - SC(133, 38, 36, 1, yes, no, 0, no) \ - SC(134, 38, 36, 2, yes, no, 0, no) \ - SC(135, 38, 36, 3, yes, no, 0, no) \ - SC(136, 38, 36, 4, yes, no, 0, no) \ - \ - SC(137, 39, 37, 1, yes, no, 0, no) \ - SC(138, 39, 37, 2, yes, no, 0, no) \ - SC(139, 39, 37, 3, yes, no, 0, no) \ - SC(140, 39, 37, 4, yes, no, 0, no) \ - \ - SC(141, 40, 38, 1, yes, no, 0, no) \ - SC(142, 40, 38, 2, yes, no, 0, no) \ - SC(143, 40, 38, 3, yes, no, 0, no) \ - SC(144, 40, 38, 4, yes, no, 0, no) \ - \ - SC(145, 41, 39, 1, yes, no, 0, no) \ - SC(146, 41, 39, 2, yes, no, 0, no) \ - SC(147, 41, 39, 3, yes, no, 0, no) \ - SC(148, 41, 39, 4, yes, no, 0, no) \ - \ - SC(149, 42, 40, 1, yes, no, 0, no) \ - SC(150, 42, 40, 2, yes, no, 0, no) \ - SC(151, 42, 40, 3, yes, no, 0, no) \ - SC(152, 42, 40, 4, yes, no, 0, no) \ - \ - SC(153, 43, 41, 1, yes, no, 0, no) \ - SC(154, 43, 41, 2, yes, no, 0, no) \ - SC(155, 43, 41, 3, yes, no, 0, no) \ - SC(156, 43, 41, 4, yes, no, 0, no) \ - \ - SC(157, 44, 42, 1, yes, no, 0, no) \ - SC(158, 44, 42, 2, yes, no, 0, no) \ - SC(159, 44, 42, 3, yes, no, 0, no) \ - SC(160, 44, 42, 4, yes, no, 0, no) \ - \ - SC(161, 45, 43, 1, yes, no, 0, no) \ - SC(162, 45, 43, 2, yes, no, 0, no) \ - SC(163, 45, 43, 3, yes, no, 0, no) \ - SC(164, 45, 43, 4, yes, no, 0, no) \ - \ - SC(165, 46, 44, 1, yes, no, 0, no) \ - SC(166, 46, 44, 2, yes, no, 0, no) \ - SC(167, 46, 44, 3, yes, no, 0, no) \ - SC(168, 46, 44, 4, yes, no, 0, no) \ - \ - SC(169, 47, 45, 1, yes, no, 0, no) \ - SC(170, 47, 45, 2, yes, no, 0, no) \ - SC(171, 47, 45, 3, yes, no, 0, no) \ - SC(172, 47, 45, 4, yes, no, 0, no) \ - \ - SC(173, 48, 46, 1, yes, no, 0, no) \ - SC(174, 48, 46, 2, yes, no, 0, no) \ - SC(175, 48, 46, 3, yes, no, 0, no) \ - SC(176, 48, 46, 4, yes, no, 0, no) \ - \ - SC(177, 49, 47, 1, yes, no, 0, no) \ - SC(178, 49, 47, 2, yes, no, 0, no) \ - SC(179, 49, 47, 3, yes, no, 0, no) \ - SC(180, 49, 47, 4, yes, no, 0, no) \ - \ - SC(181, 50, 48, 1, yes, no, 0, no) \ - SC(182, 50, 48, 2, yes, no, 0, no) \ - SC(183, 50, 48, 3, yes, no, 0, no) \ - SC(184, 50, 48, 4, yes, no, 0, no) \ - \ - SC(185, 51, 49, 1, yes, no, 0, no) \ - SC(186, 51, 49, 2, yes, no, 0, no) \ - SC(187, 51, 49, 3, yes, no, 0, no) \ - SC(188, 51, 49, 4, yes, no, 0, no) \ - \ - SC(189, 52, 50, 1, yes, no, 0, no) \ - SC(190, 52, 50, 2, yes, no, 0, no) \ - SC(191, 52, 50, 3, yes, no, 0, no) \ - SC(192, 52, 50, 4, yes, no, 0, no) \ - \ - SC(193, 53, 51, 1, yes, no, 0, no) \ - SC(194, 53, 51, 2, yes, no, 0, no) \ - SC(195, 53, 51, 3, yes, no, 0, no) \ - SC(196, 53, 51, 4, yes, no, 0, no) \ - \ - SC(197, 54, 52, 1, yes, no, 0, no) \ - SC(198, 54, 52, 2, yes, no, 0, no) \ - SC(199, 54, 52, 3, yes, no, 0, no) \ - SC(200, 54, 52, 4, yes, no, 0, no) \ - \ - SC(201, 55, 53, 1, yes, no, 0, no) \ - SC(202, 55, 53, 2, yes, no, 0, no) \ - SC(203, 55, 53, 3, yes, no, 0, no) \ - SC(204, 55, 53, 4, yes, no, 0, no) \ - \ - SC(205, 56, 54, 1, yes, no, 0, no) \ - SC(206, 56, 54, 2, yes, no, 0, no) \ - SC(207, 56, 54, 3, yes, no, 0, no) \ - SC(208, 56, 54, 4, yes, no, 0, no) \ - \ - SC(209, 57, 55, 1, yes, no, 0, no) \ - SC(210, 57, 55, 2, yes, no, 0, no) \ - SC(211, 57, 55, 3, yes, no, 0, no) \ - SC(212, 57, 55, 4, yes, no, 0, no) \ - \ - SC(213, 58, 56, 1, yes, no, 0, no) \ - SC(214, 58, 56, 2, yes, no, 0, no) \ - SC(215, 58, 56, 3, yes, no, 0, no) \ - SC(216, 58, 56, 4, yes, no, 0, no) \ - \ - SC(217, 59, 57, 1, yes, no, 0, no) \ - SC(218, 59, 57, 2, yes, no, 0, no) \ - SC(219, 59, 57, 3, yes, no, 0, no) \ - SC(220, 59, 57, 4, yes, no, 0, no) \ - \ - SC(221, 60, 58, 1, yes, no, 0, no) \ - SC(222, 60, 58, 2, yes, no, 0, no) \ - SC(223, 60, 58, 3, yes, no, 0, no) \ - SC(224, 60, 58, 4, yes, no, 0, no) \ - \ - SC(225, 61, 59, 1, yes, no, 0, no) \ - SC(226, 61, 59, 2, yes, no, 0, no) \ - SC(227, 61, 59, 3, yes, no, 0, no) \ - SC(228, 61, 59, 4, yes, no, 0, no) \ - \ - SC(229, 62, 60, 1, yes, no, 0, no) \ - SC(230, 62, 60, 2, yes, no, 0, no) \ - SC(231, 62, 60, 3, yes, no, 0, no) \ - -#define SIZE_CLASSES_DEFINED -#define NTBINS 1 -#define NLBINS 29 -#define NBINS 36 -#define NSIZES 232 -#define LG_CEIL_NSIZES 8 -#define NPSIZES 199 -#define LG_TINY_MAXCLASS 3 -#define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) -#define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) -#define LG_LARGE_MINCLASS 14 -#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -#define LARGE_MAXCLASS ((((size_t)1) << 62) + (((size_t)3) << 60)) -#endif - -#if (LG_SIZEOF_PTR == 3 && LG_TINY_MIN == 4 && LG_QUANTUM == 4 && LG_PAGE == 12) -#define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \ - SC( 0, 4, 4, 0, no, yes, 1, 4) \ - SC( 1, 4, 4, 1, no, yes, 1, 4) \ - SC( 2, 4, 4, 2, no, yes, 3, 4) \ - SC( 3, 4, 4, 3, no, yes, 1, 4) \ - \ - SC( 4, 6, 4, 1, no, yes, 5, 4) \ - SC( 5, 6, 4, 2, no, yes, 3, 4) \ - SC( 6, 6, 4, 3, no, yes, 7, 4) \ - SC( 7, 6, 4, 4, no, yes, 1, 4) \ - \ - SC( 8, 7, 5, 1, no, yes, 5, 5) \ - SC( 9, 7, 5, 2, no, yes, 3, 5) \ - SC( 10, 7, 5, 3, no, yes, 7, 5) \ - SC( 11, 7, 5, 4, no, yes, 1, 5) \ - \ - SC( 12, 8, 6, 1, no, yes, 5, 6) \ - SC( 13, 8, 6, 2, no, yes, 3, 6) \ - SC( 14, 8, 6, 3, no, yes, 7, 6) \ - SC( 15, 8, 6, 4, no, yes, 1, 6) \ - \ - SC( 16, 9, 7, 1, no, yes, 5, 7) \ - SC( 17, 9, 7, 2, no, yes, 3, 7) \ - SC( 18, 9, 7, 3, no, yes, 7, 7) \ - SC( 19, 9, 7, 4, no, yes, 1, 7) \ - \ - SC( 20, 10, 8, 1, no, yes, 5, 8) \ - SC( 21, 10, 8, 2, no, yes, 3, 8) \ - SC( 22, 10, 8, 3, no, yes, 7, 8) \ - SC( 23, 10, 8, 4, no, yes, 1, 8) \ - \ - SC( 24, 11, 9, 1, no, yes, 5, 9) \ - SC( 25, 11, 9, 2, no, yes, 3, 9) \ - SC( 26, 11, 9, 3, no, yes, 7, 9) \ - SC( 27, 11, 9, 4, yes, yes, 1, 9) \ - \ - SC( 28, 12, 10, 1, no, yes, 5, no) \ - SC( 29, 12, 10, 2, no, yes, 3, no) \ - SC( 30, 12, 10, 3, no, yes, 7, no) \ - SC( 31, 12, 10, 4, yes, yes, 2, no) \ - \ - SC( 32, 13, 11, 1, no, yes, 5, no) \ - SC( 33, 13, 11, 2, yes, yes, 3, no) \ - SC( 34, 13, 11, 3, no, yes, 7, no) \ - SC( 35, 13, 11, 4, yes, no, 0, no) \ - \ - SC( 36, 14, 12, 1, yes, no, 0, no) \ - SC( 37, 14, 12, 2, yes, no, 0, no) \ - SC( 38, 14, 12, 3, yes, no, 0, no) \ - SC( 39, 14, 12, 4, yes, no, 0, no) \ - \ - SC( 40, 15, 13, 1, yes, no, 0, no) \ - SC( 41, 15, 13, 2, yes, no, 0, no) \ - SC( 42, 15, 13, 3, yes, no, 0, no) \ - SC( 43, 15, 13, 4, yes, no, 0, no) \ - \ - SC( 44, 16, 14, 1, yes, no, 0, no) \ - SC( 45, 16, 14, 2, yes, no, 0, no) \ - SC( 46, 16, 14, 3, yes, no, 0, no) \ - SC( 47, 16, 14, 4, yes, no, 0, no) \ - \ - SC( 48, 17, 15, 1, yes, no, 0, no) \ - SC( 49, 17, 15, 2, yes, no, 0, no) \ - SC( 50, 17, 15, 3, yes, no, 0, no) \ - SC( 51, 17, 15, 4, yes, no, 0, no) \ - \ - SC( 52, 18, 16, 1, yes, no, 0, no) \ - SC( 53, 18, 16, 2, yes, no, 0, no) \ - SC( 54, 18, 16, 3, yes, no, 0, no) \ - SC( 55, 18, 16, 4, yes, no, 0, no) \ - \ - SC( 56, 19, 17, 1, yes, no, 0, no) \ - SC( 57, 19, 17, 2, yes, no, 0, no) \ - SC( 58, 19, 17, 3, yes, no, 0, no) \ - SC( 59, 19, 17, 4, yes, no, 0, no) \ - \ - SC( 60, 20, 18, 1, yes, no, 0, no) \ - SC( 61, 20, 18, 2, yes, no, 0, no) \ - SC( 62, 20, 18, 3, yes, no, 0, no) \ - SC( 63, 20, 18, 4, yes, no, 0, no) \ - \ - SC( 64, 21, 19, 1, yes, no, 0, no) \ - SC( 65, 21, 19, 2, yes, no, 0, no) \ - SC( 66, 21, 19, 3, yes, no, 0, no) \ - SC( 67, 21, 19, 4, yes, no, 0, no) \ - \ - SC( 68, 22, 20, 1, yes, no, 0, no) \ - SC( 69, 22, 20, 2, yes, no, 0, no) \ - SC( 70, 22, 20, 3, yes, no, 0, no) \ - SC( 71, 22, 20, 4, yes, no, 0, no) \ - \ - SC( 72, 23, 21, 1, yes, no, 0, no) \ - SC( 73, 23, 21, 2, yes, no, 0, no) \ - SC( 74, 23, 21, 3, yes, no, 0, no) \ - SC( 75, 23, 21, 4, yes, no, 0, no) \ - \ - SC( 76, 24, 22, 1, yes, no, 0, no) \ - SC( 77, 24, 22, 2, yes, no, 0, no) \ - SC( 78, 24, 22, 3, yes, no, 0, no) \ - SC( 79, 24, 22, 4, yes, no, 0, no) \ - \ - SC( 80, 25, 23, 1, yes, no, 0, no) \ - SC( 81, 25, 23, 2, yes, no, 0, no) \ - SC( 82, 25, 23, 3, yes, no, 0, no) \ - SC( 83, 25, 23, 4, yes, no, 0, no) \ - \ - SC( 84, 26, 24, 1, yes, no, 0, no) \ - SC( 85, 26, 24, 2, yes, no, 0, no) \ - SC( 86, 26, 24, 3, yes, no, 0, no) \ - SC( 87, 26, 24, 4, yes, no, 0, no) \ - \ - SC( 88, 27, 25, 1, yes, no, 0, no) \ - SC( 89, 27, 25, 2, yes, no, 0, no) \ - SC( 90, 27, 25, 3, yes, no, 0, no) \ - SC( 91, 27, 25, 4, yes, no, 0, no) \ - \ - SC( 92, 28, 26, 1, yes, no, 0, no) \ - SC( 93, 28, 26, 2, yes, no, 0, no) \ - SC( 94, 28, 26, 3, yes, no, 0, no) \ - SC( 95, 28, 26, 4, yes, no, 0, no) \ - \ - SC( 96, 29, 27, 1, yes, no, 0, no) \ - SC( 97, 29, 27, 2, yes, no, 0, no) \ - SC( 98, 29, 27, 3, yes, no, 0, no) \ - SC( 99, 29, 27, 4, yes, no, 0, no) \ - \ - SC(100, 30, 28, 1, yes, no, 0, no) \ - SC(101, 30, 28, 2, yes, no, 0, no) \ - SC(102, 30, 28, 3, yes, no, 0, no) \ - SC(103, 30, 28, 4, yes, no, 0, no) \ - \ - SC(104, 31, 29, 1, yes, no, 0, no) \ - SC(105, 31, 29, 2, yes, no, 0, no) \ - SC(106, 31, 29, 3, yes, no, 0, no) \ - SC(107, 31, 29, 4, yes, no, 0, no) \ - \ - SC(108, 32, 30, 1, yes, no, 0, no) \ - SC(109, 32, 30, 2, yes, no, 0, no) \ - SC(110, 32, 30, 3, yes, no, 0, no) \ - SC(111, 32, 30, 4, yes, no, 0, no) \ - \ - SC(112, 33, 31, 1, yes, no, 0, no) \ - SC(113, 33, 31, 2, yes, no, 0, no) \ - SC(114, 33, 31, 3, yes, no, 0, no) \ - SC(115, 33, 31, 4, yes, no, 0, no) \ - \ - SC(116, 34, 32, 1, yes, no, 0, no) \ - SC(117, 34, 32, 2, yes, no, 0, no) \ - SC(118, 34, 32, 3, yes, no, 0, no) \ - SC(119, 34, 32, 4, yes, no, 0, no) \ - \ - SC(120, 35, 33, 1, yes, no, 0, no) \ - SC(121, 35, 33, 2, yes, no, 0, no) \ - SC(122, 35, 33, 3, yes, no, 0, no) \ - SC(123, 35, 33, 4, yes, no, 0, no) \ - \ - SC(124, 36, 34, 1, yes, no, 0, no) \ - SC(125, 36, 34, 2, yes, no, 0, no) \ - SC(126, 36, 34, 3, yes, no, 0, no) \ - SC(127, 36, 34, 4, yes, no, 0, no) \ - \ - SC(128, 37, 35, 1, yes, no, 0, no) \ - SC(129, 37, 35, 2, yes, no, 0, no) \ - SC(130, 37, 35, 3, yes, no, 0, no) \ - SC(131, 37, 35, 4, yes, no, 0, no) \ - \ - SC(132, 38, 36, 1, yes, no, 0, no) \ - SC(133, 38, 36, 2, yes, no, 0, no) \ - SC(134, 38, 36, 3, yes, no, 0, no) \ - SC(135, 38, 36, 4, yes, no, 0, no) \ - \ - SC(136, 39, 37, 1, yes, no, 0, no) \ - SC(137, 39, 37, 2, yes, no, 0, no) \ - SC(138, 39, 37, 3, yes, no, 0, no) \ - SC(139, 39, 37, 4, yes, no, 0, no) \ - \ - SC(140, 40, 38, 1, yes, no, 0, no) \ - SC(141, 40, 38, 2, yes, no, 0, no) \ - SC(142, 40, 38, 3, yes, no, 0, no) \ - SC(143, 40, 38, 4, yes, no, 0, no) \ - \ - SC(144, 41, 39, 1, yes, no, 0, no) \ - SC(145, 41, 39, 2, yes, no, 0, no) \ - SC(146, 41, 39, 3, yes, no, 0, no) \ - SC(147, 41, 39, 4, yes, no, 0, no) \ - \ - SC(148, 42, 40, 1, yes, no, 0, no) \ - SC(149, 42, 40, 2, yes, no, 0, no) \ - SC(150, 42, 40, 3, yes, no, 0, no) \ - SC(151, 42, 40, 4, yes, no, 0, no) \ - \ - SC(152, 43, 41, 1, yes, no, 0, no) \ - SC(153, 43, 41, 2, yes, no, 0, no) \ - SC(154, 43, 41, 3, yes, no, 0, no) \ - SC(155, 43, 41, 4, yes, no, 0, no) \ - \ - SC(156, 44, 42, 1, yes, no, 0, no) \ - SC(157, 44, 42, 2, yes, no, 0, no) \ - SC(158, 44, 42, 3, yes, no, 0, no) \ - SC(159, 44, 42, 4, yes, no, 0, no) \ - \ - SC(160, 45, 43, 1, yes, no, 0, no) \ - SC(161, 45, 43, 2, yes, no, 0, no) \ - SC(162, 45, 43, 3, yes, no, 0, no) \ - SC(163, 45, 43, 4, yes, no, 0, no) \ - \ - SC(164, 46, 44, 1, yes, no, 0, no) \ - SC(165, 46, 44, 2, yes, no, 0, no) \ - SC(166, 46, 44, 3, yes, no, 0, no) \ - SC(167, 46, 44, 4, yes, no, 0, no) \ - \ - SC(168, 47, 45, 1, yes, no, 0, no) \ - SC(169, 47, 45, 2, yes, no, 0, no) \ - SC(170, 47, 45, 3, yes, no, 0, no) \ - SC(171, 47, 45, 4, yes, no, 0, no) \ - \ - SC(172, 48, 46, 1, yes, no, 0, no) \ - SC(173, 48, 46, 2, yes, no, 0, no) \ - SC(174, 48, 46, 3, yes, no, 0, no) \ - SC(175, 48, 46, 4, yes, no, 0, no) \ - \ - SC(176, 49, 47, 1, yes, no, 0, no) \ - SC(177, 49, 47, 2, yes, no, 0, no) \ - SC(178, 49, 47, 3, yes, no, 0, no) \ - SC(179, 49, 47, 4, yes, no, 0, no) \ - \ - SC(180, 50, 48, 1, yes, no, 0, no) \ - SC(181, 50, 48, 2, yes, no, 0, no) \ - SC(182, 50, 48, 3, yes, no, 0, no) \ - SC(183, 50, 48, 4, yes, no, 0, no) \ - \ - SC(184, 51, 49, 1, yes, no, 0, no) \ - SC(185, 51, 49, 2, yes, no, 0, no) \ - SC(186, 51, 49, 3, yes, no, 0, no) \ - SC(187, 51, 49, 4, yes, no, 0, no) \ - \ - SC(188, 52, 50, 1, yes, no, 0, no) \ - SC(189, 52, 50, 2, yes, no, 0, no) \ - SC(190, 52, 50, 3, yes, no, 0, no) \ - SC(191, 52, 50, 4, yes, no, 0, no) \ - \ - SC(192, 53, 51, 1, yes, no, 0, no) \ - SC(193, 53, 51, 2, yes, no, 0, no) \ - SC(194, 53, 51, 3, yes, no, 0, no) \ - SC(195, 53, 51, 4, yes, no, 0, no) \ - \ - SC(196, 54, 52, 1, yes, no, 0, no) \ - SC(197, 54, 52, 2, yes, no, 0, no) \ - SC(198, 54, 52, 3, yes, no, 0, no) \ - SC(199, 54, 52, 4, yes, no, 0, no) \ - \ - SC(200, 55, 53, 1, yes, no, 0, no) \ - SC(201, 55, 53, 2, yes, no, 0, no) \ - SC(202, 55, 53, 3, yes, no, 0, no) \ - SC(203, 55, 53, 4, yes, no, 0, no) \ - \ - SC(204, 56, 54, 1, yes, no, 0, no) \ - SC(205, 56, 54, 2, yes, no, 0, no) \ - SC(206, 56, 54, 3, yes, no, 0, no) \ - SC(207, 56, 54, 4, yes, no, 0, no) \ - \ - SC(208, 57, 55, 1, yes, no, 0, no) \ - SC(209, 57, 55, 2, yes, no, 0, no) \ - SC(210, 57, 55, 3, yes, no, 0, no) \ - SC(211, 57, 55, 4, yes, no, 0, no) \ - \ - SC(212, 58, 56, 1, yes, no, 0, no) \ - SC(213, 58, 56, 2, yes, no, 0, no) \ - SC(214, 58, 56, 3, yes, no, 0, no) \ - SC(215, 58, 56, 4, yes, no, 0, no) \ - \ - SC(216, 59, 57, 1, yes, no, 0, no) \ - SC(217, 59, 57, 2, yes, no, 0, no) \ - SC(218, 59, 57, 3, yes, no, 0, no) \ - SC(219, 59, 57, 4, yes, no, 0, no) \ - \ - SC(220, 60, 58, 1, yes, no, 0, no) \ - SC(221, 60, 58, 2, yes, no, 0, no) \ - SC(222, 60, 58, 3, yes, no, 0, no) \ - SC(223, 60, 58, 4, yes, no, 0, no) \ - \ - SC(224, 61, 59, 1, yes, no, 0, no) \ - SC(225, 61, 59, 2, yes, no, 0, no) \ - SC(226, 61, 59, 3, yes, no, 0, no) \ - SC(227, 61, 59, 4, yes, no, 0, no) \ - \ - SC(228, 62, 60, 1, yes, no, 0, no) \ - SC(229, 62, 60, 2, yes, no, 0, no) \ - SC(230, 62, 60, 3, yes, no, 0, no) \ - -#define SIZE_CLASSES_DEFINED -#define NTBINS 0 -#define NLBINS 28 -#define NBINS 35 -#define NSIZES 231 -#define LG_CEIL_NSIZES 8 -#define NPSIZES 199 -#define LG_TINY_MAXCLASS "NA" -#define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) -#define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) -#define LG_LARGE_MINCLASS 14 -#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -#define LARGE_MAXCLASS ((((size_t)1) << 62) + (((size_t)3) << 60)) -#endif - -#ifndef SIZE_CLASSES_DEFINED -# error "No size class definitions match configuration" -#endif -#undef SIZE_CLASSES_DEFINED -/* - * The size2index_tab lookup table uses uint8_t to encode each bin index, so we - * cannot support more than 256 small size classes. - */ -#if (NBINS > 256) -# error "Too many small size classes" -#endif - -#endif /* JEMALLOC_INTERNAL_SIZE_CLASSES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/stats.h b/dep/jemalloc/include/jemalloc/internal/stats.h index 852e34269ab..3b9e0eac12b 100644 --- a/dep/jemalloc/include/jemalloc/internal/stats.h +++ b/dep/jemalloc/include/jemalloc/internal/stats.h @@ -10,7 +10,8 @@ OPTION('a', unmerged, config_stats, false) \ OPTION('b', bins, true, false) \ OPTION('l', large, true, false) \ - OPTION('x', mutex, true, false) + OPTION('x', mutex, true, false) \ + OPTION('e', extents, true, false) enum { #define OPTION(o, v, d, s) stats_print_option_num_##v, diff --git a/dep/jemalloc/include/jemalloc/internal/sz.h b/dep/jemalloc/include/jemalloc/internal/sz.h index 97946289854..68e558abfea 100644 --- a/dep/jemalloc/include/jemalloc/internal/sz.h +++ b/dep/jemalloc/include/jemalloc/internal/sz.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" /* @@ -26,18 +26,18 @@ * sz_pind2sz_tab encodes the same information as could be computed by * sz_pind2sz_compute(). */ -extern size_t const sz_pind2sz_tab[NPSIZES+1]; +extern size_t sz_pind2sz_tab[SC_NPSIZES + 1]; /* * sz_index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by sz_index2size_compute(). */ -extern size_t const sz_index2size_tab[NSIZES]; +extern size_t sz_index2size_tab[SC_NSIZES]; /* * sz_size2index_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, * and all accesses are via sz_size2index(). */ -extern uint8_t const sz_size2index_tab[]; +extern uint8_t sz_size2index_tab[]; static const size_t sz_large_pad = #ifdef JEMALLOC_CACHE_OBLIVIOUS @@ -47,49 +47,47 @@ static const size_t sz_large_pad = #endif ; +extern void sz_boot(const sc_data_t *sc_data); + JEMALLOC_ALWAYS_INLINE pszind_t sz_psz2ind(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return NPSIZES; + if (unlikely(psz > SC_LARGE_MAXCLASS)) { + return SC_NPSIZES; } - { - pszind_t x = lg_floor((psz<<1)-1); - pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - - (LG_SIZE_CLASS_GROUP + LG_PAGE); - pszind_t grp = shift << LG_SIZE_CLASS_GROUP; + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < SC_LG_NGROUP + LG_PAGE) ? + 0 : x - (SC_LG_NGROUP + LG_PAGE); + pszind_t grp = shift << SC_LG_NGROUP; - pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + pszind_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? + LG_PAGE : x - SC_LG_NGROUP - 1; - size_t delta_inverse_mask = ZU(-1) << lg_delta; - pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + size_t delta_inverse_mask = ZU(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << SC_LG_NGROUP) - 1); - pszind_t ind = grp + mod; - return ind; - } + pszind_t ind = grp + mod; + return ind; } static inline size_t sz_pind2sz_compute(pszind_t pind) { - if (unlikely(pind == NPSIZES)) { - return LARGE_MAXCLASS + PAGE; + if (unlikely(pind == SC_NPSIZES)) { + return SC_LARGE_MAXCLASS + PAGE; } - { - size_t grp = pind >> LG_SIZE_CLASS_GROUP; - size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + size_t grp = pind >> SC_LG_NGROUP; + size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1); - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_PAGE + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP-1))) << grp) + & grp_size_mask; - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_PAGE-1); - size_t mod_size = (mod+1) << lg_delta; + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; - size_t sz = grp_size + mod_size; - return sz; - } + size_t sz = grp_size + mod_size; + return sz; } static inline size_t @@ -101,70 +99,70 @@ sz_pind2sz_lookup(pszind_t pind) { static inline size_t sz_pind2sz(pszind_t pind) { - assert(pind < NPSIZES+1); + assert(pind < SC_NPSIZES + 1); return sz_pind2sz_lookup(pind); } static inline size_t sz_psz2u(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return LARGE_MAXCLASS + PAGE; - } - { - size_t x = lg_floor((psz<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (psz + delta_mask) & ~delta_mask; - return usize; + if (unlikely(psz > SC_LARGE_MAXCLASS)) { + return SC_LARGE_MAXCLASS + PAGE; } + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? + LG_PAGE : x - SC_LG_NGROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return usize; } static inline szind_t sz_size2index_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { - return NSIZES; + if (unlikely(size > SC_LARGE_MAXCLASS)) { + return SC_NSIZES; + } + + if (size == 0) { + return 0; } -#if (NTBINS != 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; +#if (SC_NTINY != 0) + if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { + szind_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); } #endif { szind_t x = lg_floor((size<<1)-1); - szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : - x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); - szind_t grp = shift << LG_SIZE_CLASS_GROUP; + szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) ? 0 : + x - (SC_LG_NGROUP + LG_QUANTUM); + szind_t grp = shift << SC_LG_NGROUP; - szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - SC_LG_NGROUP - 1; size_t delta_inverse_mask = ZU(-1) << lg_delta; szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + ((ZU(1) << SC_LG_NGROUP) - 1); - szind_t index = NTBINS + grp + mod; + szind_t index = SC_NTINY + grp + mod; return index; } } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index_lookup(size_t size) { - assert(size <= LOOKUP_MAXCLASS); - { - szind_t ret = (sz_size2index_tab[(size-1) >> LG_TINY_MIN]); - assert(ret == sz_size2index_compute(size)); - return ret; - } + assert(size <= SC_LOOKUP_MAXCLASS); + szind_t ret = (sz_size2index_tab[(size + (ZU(1) << SC_LG_TINY_MIN) - 1) + >> SC_LG_TINY_MIN]); + assert(ret == sz_size2index_compute(size)); + return ret; } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { + if (likely(size <= SC_LOOKUP_MAXCLASS)) { return sz_size2index_lookup(size); } return sz_size2index_compute(size); @@ -172,20 +170,20 @@ sz_size2index(size_t size) { static inline size_t sz_index2size_compute(szind_t index) { -#if (NTBINS > 0) - if (index < NTBINS) { - return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); +#if (SC_NTINY > 0) + if (index < SC_NTINY) { + return (ZU(1) << (SC_LG_TINY_MAXCLASS - SC_NTINY + 1 + index)); } #endif { - size_t reduced_index = index - NTBINS; - size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; - size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - + size_t reduced_index = index - SC_NTINY; + size_t grp = reduced_index >> SC_LG_NGROUP; + size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - 1); size_t grp_size_mask = ~((!!grp)-1); size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + (SC_LG_NGROUP-1))) << grp) & grp_size_mask; size_t shift = (grp == 0) ? 1 : grp; size_t lg_delta = shift + (LG_QUANTUM-1); @@ -205,18 +203,22 @@ sz_index2size_lookup(szind_t index) { JEMALLOC_ALWAYS_INLINE size_t sz_index2size(szind_t index) { - assert(index < NSIZES); + assert(index < SC_NSIZES); return sz_index2size_lookup(index); } JEMALLOC_ALWAYS_INLINE size_t sz_s2u_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { + if (unlikely(size > SC_LARGE_MAXCLASS)) { return 0; } -#if (NTBINS > 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + + if (size == 0) { + size++; + } +#if (SC_NTINY > 0) + if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { + size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : (ZU(1) << lg_ceil)); @@ -224,8 +226,8 @@ sz_s2u_compute(size_t size) { #endif { size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - SC_LG_NGROUP - 1; size_t delta = ZU(1) << lg_delta; size_t delta_mask = delta - 1; size_t usize = (size + delta_mask) & ~delta_mask; @@ -247,8 +249,7 @@ sz_s2u_lookup(size_t size) { */ JEMALLOC_ALWAYS_INLINE size_t sz_s2u(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { + if (likely(size <= SC_LOOKUP_MAXCLASS)) { return sz_s2u_lookup(size); } return sz_s2u_compute(size); @@ -265,7 +266,7 @@ sz_sa2u(size_t size, size_t alignment) { assert(alignment != 0 && ((alignment - 1) & alignment) == 0); /* Try for a small size class. */ - if (size <= SMALL_MAXCLASS && alignment < PAGE) { + if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) { /* * Round size up to the nearest multiple of alignment. * @@ -281,20 +282,20 @@ sz_sa2u(size_t size, size_t alignment) { * 192 | 11000000 | 64 */ usize = sz_s2u(ALIGNMENT_CEILING(size, alignment)); - if (usize < LARGE_MINCLASS) { + if (usize < SC_LARGE_MINCLASS) { return usize; } } /* Large size class. Beware of overflow. */ - if (unlikely(alignment > LARGE_MAXCLASS)) { + if (unlikely(alignment > SC_LARGE_MAXCLASS)) { return 0; } /* Make sure result is a large size class. */ - if (size <= LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (size <= SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } else { usize = sz_s2u(size); if (usize < size) { diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_externs.h b/dep/jemalloc/include/jemalloc/internal/tcache_externs.h index 790367bd481..d63eafde8ce 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_externs.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_externs.h @@ -1,15 +1,13 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H -#include "jemalloc/internal/size_classes.h" - extern bool opt_tcache; extern ssize_t opt_lg_tcache_max; extern cache_bin_info_t *tcache_bin_info; /* - * Number of tcache bins. There are NBINS small-object bins, plus 0 or more + * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more * large-object bins. */ extern unsigned nhbins; diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h b/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h index 0f6ab8cb50a..5eca20e893b 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_inlines.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/util.h" @@ -40,13 +40,13 @@ tcache_event(tsd_t *tsd, tcache_t *tcache) { JEMALLOC_ALWAYS_INLINE void * tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - UNUSED size_t size, szind_t binind, bool zero, bool slow_path) { + size_t size, szind_t binind, bool zero, bool slow_path) { void *ret; cache_bin_t *bin; bool tcache_success; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - assert(binind < NBINS); + assert(binind < SC_NBINS); bin = tcache_small_bin_get(tcache, binind); ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); @@ -107,7 +107,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, cache_bin_t *bin; bool tcache_success; - assert(binind >= NBINS &&binind < nhbins); + assert(binind >= SC_NBINS &&binind < nhbins); bin = tcache_large_bin_get(tcache, binind); ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); @@ -166,7 +166,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin; cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) + <= SC_SMALL_MAXCLASS); if (slow_path && config_fill && unlikely(opt_junk_free)) { arena_dalloc_junk_small(ptr, &bin_infos[binind]); @@ -174,13 +175,12 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bin = tcache_small_bin_get(tcache, binind); bin_info = &tcache_bin_info[binind]; - if (unlikely(bin->ncached == bin_info->ncached_max)) { + if (unlikely(!cache_bin_dalloc_easy(bin, bin_info, ptr))) { tcache_bin_flush_small(tsd, tcache, bin, binind, (bin_info->ncached_max >> 1)); + bool ret = cache_bin_dalloc_easy(bin, bin_info, ptr); + assert(ret); } - assert(bin->ncached < bin_info->ncached_max); - bin->ncached++; - *(bin->avail - bin->ncached) = ptr; tcache_event(tsd, tcache); } @@ -191,7 +191,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin; cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) + > SC_SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); if (slow_path && config_fill && unlikely(opt_junk_free)) { @@ -215,6 +216,9 @@ JEMALLOC_ALWAYS_INLINE tcache_t * tcaches_get(tsd_t *tsd, unsigned ind) { tcaches_t *elm = &tcaches[ind]; if (unlikely(elm->tcache == NULL)) { + malloc_printf("<jemalloc>: invalid tcache id (%u).\n", ind); + abort(); + } else if (unlikely(elm->tcache == TCACHES_ELM_NEED_REINIT)) { elm->tcache = tcache_create_explicit(tsd); } return elm->tcache; diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_structs.h b/dep/jemalloc/include/jemalloc/internal/tcache_structs.h index 07b7387059f..172ef9040c0 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_structs.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_structs.h @@ -1,10 +1,14 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H #define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H -#include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/cache_bin.h" +#include "jemalloc/internal/ql.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/ticker.h" +#include "jemalloc/internal/tsd_types.h" + +/* Various uses of this struct need it to be a named type. */ +typedef ql_elm(tsd_t) tsd_link_t; struct tcache_s { /* @@ -21,7 +25,7 @@ struct tcache_s { * During tcache initialization, the avail pointer in each element of * tbins is initialized to point to the proper offset within this array. */ - cache_bin_t bins_small[NBINS]; + cache_bin_t bins_small[SC_NBINS]; /* * This data is less hot; we can be a little less careful with our @@ -29,6 +33,11 @@ struct tcache_s { */ /* Lets us track all the tcaches in an arena. */ ql_elm(tcache_t) link; + + /* Logically scoped to tsd, but put here for cache layout reasons. */ + ql_elm(tsd_t) tsd_link; + bool in_hook; + /* * The descriptor lets the arena find our cache bins without seeing the * tcache definition. This enables arenas to aggregate stats across @@ -41,13 +50,13 @@ struct tcache_s { /* Next bin to GC. */ szind_t next_gc_bin; /* For small bins, fill (ncached_max >> lg_fill_div). */ - uint8_t lg_fill_div[NBINS]; + uint8_t lg_fill_div[SC_NBINS]; /* * We put the cache bins for large size classes at the end of the * struct, since some of them might not get used. This might end up * letting us avoid touching an extra page if we don't have to. */ - cache_bin_t bins_large[NSIZES-NBINS]; + cache_bin_t bins_large[SC_NSIZES-SC_NBINS]; }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/dep/jemalloc/include/jemalloc/internal/tcache_types.h b/dep/jemalloc/include/jemalloc/internal/tcache_types.h index e49bc9d79eb..dce69382ebb 100644 --- a/dep/jemalloc/include/jemalloc/internal/tcache_types.h +++ b/dep/jemalloc/include/jemalloc/internal/tcache_types.h @@ -1,7 +1,7 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H #define JEMALLOC_INTERNAL_TCACHE_TYPES_H -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef struct tcache_s tcache_t; typedef struct tcaches_s tcaches_t; @@ -45,7 +45,7 @@ typedef struct tcaches_s tcaches_t; /* Number of tcache allocation/deallocation events between incremental GCs. */ #define TCACHE_GC_INCR \ - ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) + ((TCACHE_GC_SWEEP / SC_NBINS) + ((TCACHE_GC_SWEEP / SC_NBINS == 0) ? 0 : 1)) /* Used in TSD static initializer only. Real init in tcache_data_init(). */ #define TCACHE_ZERO_INITIALIZER {0} @@ -53,4 +53,7 @@ typedef struct tcaches_s tcaches_t; /* Used in TSD static initializer only. Will be initialized to opt_tcache. */ #define TCACHE_ENABLED_ZERO_INITIALIZER false +/* Used for explicit tcache only. Means flushed but not destroyed. */ +#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1) + #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/test_hooks.h b/dep/jemalloc/include/jemalloc/internal/test_hooks.h new file mode 100644 index 00000000000..a6351e59aff --- /dev/null +++ b/dep/jemalloc/include/jemalloc/internal/test_hooks.h @@ -0,0 +1,19 @@ +#ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H +#define JEMALLOC_INTERNAL_TEST_HOOKS_H + +extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(); +extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); + +#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) + +#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) +#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) +#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) +#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) +#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) +#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) +#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) +/* Note that this is undef'd and re-define'd in src/prof.c. */ +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) + +#endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/ticker.h b/dep/jemalloc/include/jemalloc/internal/ticker.h index 4b3604708e1..52d0db4c89c 100644 --- a/dep/jemalloc/include/jemalloc/internal/ticker.h +++ b/dep/jemalloc/include/jemalloc/internal/ticker.h @@ -75,4 +75,17 @@ ticker_tick(ticker_t *ticker) { return ticker_ticks(ticker, 1); } +/* + * Try to tick. If ticker would fire, return true, but rely on + * slowpath to reset ticker. + */ +static inline bool +ticker_trytick(ticker_t *ticker) { + --ticker->tick; + if (unlikely(ticker->tick < 0)) { + return true; + } + return false; +} + #endif /* JEMALLOC_INTERNAL_TICKER_H */ diff --git a/dep/jemalloc/include/jemalloc/internal/tsd.h b/dep/jemalloc/include/jemalloc/internal/tsd.h index 0b9841aa7db..9ba2600453b 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd.h @@ -3,6 +3,7 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" @@ -68,17 +69,19 @@ typedef void (*test_callback_t)(int *); O(offset_state, uint64_t, uint64_t) \ O(thread_allocated, uint64_t, uint64_t) \ O(thread_deallocated, uint64_t, uint64_t) \ + O(bytes_until_sample, int64_t, int64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \ O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ + O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(tcache, tcache_t, tcache_t) \ O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ MALLOC_TEST_TSD #define TSD_INITIALIZER { \ - tsd_state_uninitialized, \ + ATOMIC_INIT(tsd_state_uninitialized), \ TCACHE_ENABLED_ZERO_INITIALIZER, \ false, \ 0, \ @@ -86,29 +89,97 @@ typedef void (*test_callback_t)(int *); 0, \ 0, \ 0, \ + 0, \ NULL, \ RTREE_CTX_ZERO_INITIALIZER, \ NULL, \ NULL, \ NULL, \ + TSD_BINSHARDS_ZERO_INITIALIZER, \ TCACHE_ZERO_INITIALIZER, \ WITNESS_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \ } +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +void malloc_tsd_cleanup_register(bool (*f)(void)); +tsd_t *malloc_tsd_boot0(void); +void malloc_tsd_boot1(void); +void tsd_cleanup(void *arg); +tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); +void tsd_state_set(tsd_t *tsd, uint8_t new_state); +void tsd_slow_update(tsd_t *tsd); +void tsd_prefork(tsd_t *tsd); +void tsd_postfork_parent(tsd_t *tsd); +void tsd_postfork_child(tsd_t *tsd); + +/* + * Call ..._inc when your module wants to take all threads down the slow paths, + * and ..._dec when it no longer needs to. + */ +void tsd_global_slow_inc(tsdn_t *tsdn); +void tsd_global_slow_dec(tsdn_t *tsdn); +bool tsd_global_slow(); + enum { - tsd_state_nominal = 0, /* Common case --> jnz. */ - tsd_state_nominal_slow = 1, /* Initialized but on slow path. */ - /* the above 2 nominal states should be lower values. */ - tsd_state_nominal_max = 1, /* used for comparison only. */ - tsd_state_minimal_initialized = 2, - tsd_state_purgatory = 3, - tsd_state_reincarnated = 4, - tsd_state_uninitialized = 5 + /* Common case --> jnz. */ + tsd_state_nominal = 0, + /* Initialized but on slow path. */ + tsd_state_nominal_slow = 1, + /* + * Some thread has changed global state in such a way that all nominal + * threads need to recompute their fast / slow status the next time they + * get a chance. + * + * Any thread can change another thread's status *to* recompute, but + * threads are the only ones who can change their status *from* + * recompute. + */ + tsd_state_nominal_recompute = 2, + /* + * The above nominal states should be lower values. We use + * tsd_nominal_max to separate nominal states from threads in the + * process of being born / dying. + */ + tsd_state_nominal_max = 2, + + /* + * A thread might free() during its death as its only allocator action; + * in such scenarios, we need tsd, but set up in such a way that no + * cleanup is necessary. + */ + tsd_state_minimal_initialized = 3, + /* States during which we know we're in thread death. */ + tsd_state_purgatory = 4, + tsd_state_reincarnated = 5, + /* + * What it says on the tin; tsd that hasn't been initialized. Note + * that even when the tsd struct lives in TLS, when need to keep track + * of stuff like whether or not our pthread destructors have been + * scheduled, so this really truly is different than the nominal state. + */ + tsd_state_uninitialized = 6 }; -/* Manually limit tsd_state_t to a single byte. */ -typedef uint8_t tsd_state_t; +/* + * Some TSD accesses can only be done in a nominal state. To enforce this, we + * wrap TSD member access in a function that asserts on TSD state, and mangle + * field names to prevent touching them accidentally. + */ +#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n + +#ifdef JEMALLOC_U8_ATOMICS +# define tsd_state_t atomic_u8_t +# define tsd_atomic_load atomic_load_u8 +# define tsd_atomic_store atomic_store_u8 +# define tsd_atomic_exchange atomic_exchange_u8 +#else +# define tsd_state_t atomic_u32_t +# define tsd_atomic_load atomic_load_u32 +# define tsd_atomic_store atomic_store_u32 +# define tsd_atomic_exchange atomic_exchange_u32 +#endif /* The actual tsd. */ struct tsd_s { @@ -117,13 +188,29 @@ struct tsd_s { * module. Access any thread-local state through the getters and * setters below. */ - tsd_state_t state; + + /* + * We manually limit the state to just a single byte. Unless the 8-bit + * atomics are unavailable (which is rare). + */ + tsd_state_t state; #define O(n, t, nt) \ - t use_a_getter_or_setter_instead_##n; + t TSD_MANGLE(n); MALLOC_TSD #undef O }; +JEMALLOC_ALWAYS_INLINE uint8_t +tsd_state_get(tsd_t *tsd) { + /* + * This should be atomic. Unfortunately, compilers right now can't tell + * that this can be done as a memory comparison, and forces a load into + * a register that hurts fast-path performance. + */ + /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ + return *(uint8_t *)&tsd->state; +} + /* * Wrapper around tsd_t that makes it possible to avoid implicit conversion * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be @@ -150,15 +237,6 @@ tsdn_tsd(tsdn_t *tsdn) { return &tsdn->tsd; } -void *malloc_tsd_malloc(size_t size); -void malloc_tsd_dalloc(void *wrapper); -void malloc_tsd_cleanup_register(bool (*f)(void)); -tsd_t *malloc_tsd_boot0(void); -void malloc_tsd_boot1(void); -void tsd_cleanup(void *arg); -tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); -void tsd_slow_update(tsd_t *tsd); - /* * We put the platform-specific data declarations and inlines into their own * header files to avoid cluttering this file. They define tsd_boot0, @@ -182,7 +260,7 @@ void tsd_slow_update(tsd_t *tsd); #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get_unsafe(tsd_t *tsd) { \ - return &tsd->use_a_getter_or_setter_instead_##n; \ + return &tsd->TSD_MANGLE(n); \ } MALLOC_TSD #undef O @@ -191,10 +269,16 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) { \ - assert(tsd->state == tsd_state_nominal || \ - tsd->state == tsd_state_nominal_slow || \ - tsd->state == tsd_state_reincarnated || \ - tsd->state == tsd_state_minimal_initialized); \ + /* \ + * Because the state might change asynchronously if it's \ + * nominal, we need to make sure that we only read it once. \ + */ \ + uint8_t state = tsd_state_get(tsd); \ + assert(state == tsd_state_nominal || \ + state == tsd_state_nominal_slow || \ + state == tsd_state_nominal_recompute || \ + state == tsd_state_reincarnated || \ + state == tsd_state_minimal_initialized); \ return tsd_##n##p_get_unsafe(tsd); \ } MALLOC_TSD @@ -229,8 +313,8 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE void \ tsd_##n##_set(tsd_t *tsd, t val) { \ - assert(tsd->state != tsd_state_reincarnated && \ - tsd->state != tsd_state_minimal_initialized); \ + assert(tsd_state_get(tsd) != tsd_state_reincarnated && \ + tsd_state_get(tsd) != tsd_state_minimal_initialized); \ *tsd_##n##p_get(tsd) = val; \ } MALLOC_TSD @@ -238,13 +322,18 @@ MALLOC_TSD JEMALLOC_ALWAYS_INLINE void tsd_assert_fast(tsd_t *tsd) { + /* + * Note that our fastness assertion does *not* include global slowness + * counters; it's not in general possible to ensure that they won't + * change asynchronously from underneath us. + */ assert(!malloc_slow && tsd_tcache_enabled_get(tsd) && tsd_reentrancy_level_get(tsd) == 0); } JEMALLOC_ALWAYS_INLINE bool tsd_fast(tsd_t *tsd) { - bool fast = (tsd->state == tsd_state_nominal); + bool fast = (tsd_state_get(tsd) == tsd_state_nominal); if (fast) { tsd_assert_fast(tsd); } @@ -261,7 +350,7 @@ tsd_fetch_impl(bool init, bool minimal) { } assert(tsd != NULL); - if (unlikely(tsd->state != tsd_state_nominal)) { + if (unlikely(tsd_state_get(tsd) != tsd_state_nominal)) { return tsd_fetch_slow(tsd, minimal); } assert(tsd_fast(tsd)); @@ -281,7 +370,7 @@ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_internal_fetch(void) { tsd_t *tsd = tsd_fetch_min(); /* Use reincarnated state to prevent full initialization. */ - tsd->state = tsd_state_reincarnated; + tsd_state_set(tsd, tsd_state_reincarnated); return tsd; } @@ -293,7 +382,7 @@ tsd_fetch(void) { static inline bool tsd_nominal(tsd_t *tsd) { - return (tsd->state <= tsd_state_nominal_max); + return (tsd_state_get(tsd) <= tsd_state_nominal_max); } JEMALLOC_ALWAYS_INLINE tsdn_t * diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_generic.h b/dep/jemalloc/include/jemalloc/internal/tsd_generic.h index 1e52ef767f1..cf73c0c7155 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_generic.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_generic.h @@ -77,7 +77,10 @@ tsd_wrapper_get(bool init) { abort(); } else { wrapper->initialized = false; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; } tsd_wrapper_set(wrapper); @@ -107,7 +110,10 @@ tsd_boot1(void) { tsd_boot_wrapper.initialized = false; tsd_cleanup(&tsd_boot_wrapper.val); wrapper->initialized = false; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; tsd_wrapper_set(wrapper); } diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index beb467a67e9..bf8801effe6 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -47,7 +47,6 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - assert(tsd_booted); return &tsd_tls; } JEMALLOC_ALWAYS_INLINE void diff --git a/dep/jemalloc/include/jemalloc/internal/tsd_tls.h b/dep/jemalloc/include/jemalloc/internal/tsd_tls.h index 0de64b7b8bf..f4f165c7f59 100644 --- a/dep/jemalloc/include/jemalloc/internal/tsd_tls.h +++ b/dep/jemalloc/include/jemalloc/internal/tsd_tls.h @@ -39,8 +39,7 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * -tsd_get(UNUSED bool init) { - assert(tsd_booted); +tsd_get(bool init) { return &tsd_tls; } diff --git a/dep/jemalloc/include/jemalloc/internal/witness.h b/dep/jemalloc/include/jemalloc/internal/witness.h index 7ace8ae4a11..fff9e98cb64 100644 --- a/dep/jemalloc/include/jemalloc/internal/witness.h +++ b/dep/jemalloc/include/jemalloc/internal/witness.h @@ -27,9 +27,9 @@ #define WITNESS_RANK_PROF_BT2GCTX 6U #define WITNESS_RANK_PROF_TDATAS 7U #define WITNESS_RANK_PROF_TDATA 8U -#define WITNESS_RANK_PROF_GCTX 9U - -#define WITNESS_RANK_BACKGROUND_THREAD 10U +#define WITNESS_RANK_PROF_LOG 9U +#define WITNESS_RANK_PROF_GCTX 10U +#define WITNESS_RANK_BACKGROUND_THREAD 11U /* * Used as an argument to witness_assert_depth_to_rank() in order to validate @@ -37,18 +37,19 @@ * witness_assert_depth_to_rank() is inclusive rather than exclusive, this * definition can have the same value as the minimally ranked core lock. */ -#define WITNESS_RANK_CORE 11U - -#define WITNESS_RANK_DECAY 11U -#define WITNESS_RANK_TCACHE_QL 12U -#define WITNESS_RANK_EXTENT_GROW 13U -#define WITNESS_RANK_EXTENTS 14U -#define WITNESS_RANK_EXTENT_AVAIL 15U - -#define WITNESS_RANK_EXTENT_POOL 16U -#define WITNESS_RANK_RTREE 17U -#define WITNESS_RANK_BASE 18U -#define WITNESS_RANK_ARENA_LARGE 19U +#define WITNESS_RANK_CORE 12U + +#define WITNESS_RANK_DECAY 12U +#define WITNESS_RANK_TCACHE_QL 13U +#define WITNESS_RANK_EXTENT_GROW 14U +#define WITNESS_RANK_EXTENTS 15U +#define WITNESS_RANK_EXTENT_AVAIL 16U + +#define WITNESS_RANK_EXTENT_POOL 17U +#define WITNESS_RANK_RTREE 18U +#define WITNESS_RANK_BASE 19U +#define WITNESS_RANK_ARENA_LARGE 20U +#define WITNESS_RANK_HOOK 21U #define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_BIN WITNESS_RANK_LEAF diff --git a/dep/jemalloc/include/jemalloc/jemalloc.h b/dep/jemalloc/include/jemalloc/jemalloc.h index c41a9b48974..8f38feb2f0a 100644 --- a/dep/jemalloc/include/jemalloc/jemalloc.h +++ b/dep/jemalloc/include/jemalloc/jemalloc.h @@ -66,6 +66,7 @@ extern "C" { # define je_malloc_stats_print malloc_stats_print # define je_malloc_usable_size malloc_usable_size # define je_mallocx mallocx +# define je_smallocx_b0b3e49a54ec29e32636f4577d9d5a896d67fd20 smallocx_b0b3e49a54ec29e32636f4577d9d5a896d67fd20 # define je_nallocx nallocx # define je_posix_memalign posix_memalign # define je_rallocx rallocx @@ -83,12 +84,13 @@ extern "C" { #include <limits.h> #include <strings.h> -#define JEMALLOC_VERSION "5.1.0-0-g61efbda7098de6fe64c362d309824864308c36d4" +#define JEMALLOC_VERSION "5.2.0-0-gb0b3e49a54ec29e32636f4577d9d5a896d67fd20" #define JEMALLOC_VERSION_MAJOR 5 -#define JEMALLOC_VERSION_MINOR 1 +#define JEMALLOC_VERSION_MINOR 2 #define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "61efbda7098de6fe64c362d309824864308c36d4" +#define JEMALLOC_VERSION_GID "b0b3e49a54ec29e32636f4577d9d5a896d67fd20" +#define JEMALLOC_VERSION_GID_IDENT b0b3e49a54ec29e32636f4577d9d5a896d67fd20 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -369,6 +371,7 @@ struct extent_hooks_s { # define malloc_stats_print je_malloc_stats_print # define malloc_usable_size je_malloc_usable_size # define mallocx je_mallocx +# define smallocx_b0b3e49a54ec29e32636f4577d9d5a896d67fd20 je_smallocx_b0b3e49a54ec29e32636f4577d9d5a896d67fd20 # define nallocx je_nallocx # define posix_memalign je_posix_memalign # define rallocx je_rallocx @@ -401,6 +404,7 @@ struct extent_hooks_s { # undef je_malloc_stats_print # undef je_malloc_usable_size # undef je_mallocx +# undef je_smallocx_b0b3e49a54ec29e32636f4577d9d5a896d67fd20 # undef je_nallocx # undef je_posix_memalign # undef je_rallocx diff --git a/dep/jemalloc/jemalloc_internal_defs.h.in.cmake b/dep/jemalloc/jemalloc_internal_defs.h.in.cmake index be6f92713a4..ffb57e0348a 100644 --- a/dep/jemalloc/jemalloc_internal_defs.h.in.cmake +++ b/dep/jemalloc/jemalloc_internal_defs.h.in.cmake @@ -49,25 +49,13 @@ /* Defined if GCC __atomic atomics are available. */ #define JEMALLOC_GCC_ATOMIC_ATOMICS 1 +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1 /* Defined if GCC __sync atomics are available. */ #define JEMALLOC_GCC_SYNC_ATOMICS 1 - -/* - * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and - * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */ - -/* - * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and - * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */ +/* and the 8-bit variant support. */ +#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1 /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -79,12 +67,6 @@ */ /* #undef JEMALLOC_OS_UNFAIR_LOCK */ -/* - * Defined if OSSpin*() functions are available, as provided by Darwin, and - * documented in the spinlock(3) manual page. - */ -/* #undef JEMALLOC_OSSPIN */ - /* Defined if syscall(2) is usable. */ #define JEMALLOC_USE_SYSCALL @@ -154,6 +136,9 @@ /* JEMALLOC_STATS enables statistics calculation. */ /* #undef JEMALLOC_STATS */ +/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */ +/* #undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API */ + /* JEMALLOC_PROF enables allocation profiling. */ /* #undef JEMALLOC_PROF */ @@ -235,6 +220,12 @@ #define JEMALLOC_INTERNAL_FFS __builtin_ffs /* + * popcount*() functions to use for bitmapping. + */ +#define JEMALLOC_INTERNAL_POPCOUNTL __builtin_popcountl +#define JEMALLOC_INTERNAL_POPCOUNT __builtin_popcount + +/* * If defined, explicitly attempt to more uniformly distribute large allocation * pointer alignments across all cache indices. */ @@ -247,6 +238,12 @@ /* #undef JEMALLOC_LOG */ /* + * If defined, use readlinkat() (instead of readlink()) to follow + * /etc/malloc_conf. + */ +/* #undef JEMALLOC_READLINKAT */ + +/* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ /* #undef JEMALLOC_ZONE */ @@ -364,6 +361,9 @@ */ #define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE +/* Performs additional size-matching sanity checks when defined. */ +/* #undef JEMALLOC_EXTRA_SIZE_CHECK */ + /* sizeof(void *) == 2^LG_SIZEOF_PTR. */ #define LG_SIZEOF_PTR @JEM_SIZEDEF@ diff --git a/dep/jemalloc/src/arena.c b/dep/jemalloc/src/arena.c index 5d55bf1a060..60eac232a93 100644 --- a/dep/jemalloc/src/arena.c +++ b/dep/jemalloc/src/arena.c @@ -8,9 +8,10 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/util.h" +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + /******************************************************************************/ /* Data. */ @@ -40,7 +41,11 @@ const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { #undef STEP }; -static div_info_t arena_binind_div_info[NBINS]; +static div_info_t arena_binind_div_info[SC_NBINS]; + +size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; +size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; +static unsigned huge_arena_ind; /******************************************************************************/ /* @@ -61,7 +66,7 @@ static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, /******************************************************************************/ void -arena_basic_stats_merge(UNUSED tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, +arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy) { *nthreads += arena_nthreads_get(arena, false); @@ -77,7 +82,8 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats) { + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats) { cassert(config_stats); arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, @@ -94,6 +100,10 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->retained, extents_npages_get(&arena->extents_retained) << LG_PAGE); + atomic_store_zu(&astats->extent_avail, + atomic_load_zu(&arena->extent_avail_cnt, ATOMIC_RELAXED), + ATOMIC_RELAXED); + arena_stats_accum_u64(&astats->decay_dirty.npurge, arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.decay_dirty.npurge)); @@ -122,7 +132,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, extents_npages_get(&arena->extents_dirty) + extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); - for (szind_t i = 0; i < NSIZES - NBINS; i++) { + for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) { uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.lstats[i].nmalloc); arena_stats_accum_u64(&lstats[i].nmalloc, nmalloc); @@ -145,7 +155,29 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, size_t curlextents = (size_t)(nmalloc - ndalloc); lstats[i].curlextents += curlextents; arena_stats_accum_zu(&astats->allocated_large, - curlextents * sz_index2size(NBINS + i)); + curlextents * sz_index2size(SC_NBINS + i)); + } + + for (pszind_t i = 0; i < SC_NPSIZES; i++) { + size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, + retained_bytes; + dirty = extents_nextents_get(&arena->extents_dirty, i); + muzzy = extents_nextents_get(&arena->extents_muzzy, i); + retained = extents_nextents_get(&arena->extents_retained, i); + dirty_bytes = extents_nbytes_get(&arena->extents_dirty, i); + muzzy_bytes = extents_nbytes_get(&arena->extents_muzzy, i); + retained_bytes = + extents_nbytes_get(&arena->extents_retained, i); + + atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].nretained, retained, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].dirty_bytes, dirty_bytes, + ATOMIC_RELAXED); + atomic_store_zu(&estats[i].muzzy_bytes, muzzy_bytes, + ATOMIC_RELAXED); + atomic_store_zu(&estats[i].retained_bytes, retained_bytes, + ATOMIC_RELAXED); } arena_stats_unlock(tsdn, &arena->stats); @@ -156,7 +188,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, cache_bin_array_descriptor_t *descriptor; ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { szind_t i = 0; - for (; i < NBINS; i++) { + for (; i < SC_NBINS; i++) { cache_bin_t *tbin = &descriptor->bins_small[i]; arena_stats_accum_zu(&astats->tcache_bytes, tbin->ncached * sz_index2size(i)); @@ -200,8 +232,11 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, nstime_update(&astats->uptime); nstime_subtract(&astats->uptime, &arena->create_time); - for (szind_t i = 0; i < NBINS; i++) { - bin_stats_merge(tsdn, &bstats[i], &arena->bins[i]); + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_stats_merge(tsdn, &bstats[i], + &arena->bins[i].bin_shards[j]); + } } } @@ -236,6 +271,54 @@ arena_slab_reg_alloc(extent_t *slab, const bin_info_t *bin_info) { return ret; } +static void +arena_slab_reg_alloc_batch(extent_t *slab, const bin_info_t *bin_info, + unsigned cnt, void** ptrs) { + arena_slab_data_t *slab_data = extent_slab_data_get(slab); + + assert(extent_nfree_get(slab) >= cnt); + assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); + +#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) + for (unsigned i = 0; i < cnt; i++) { + size_t regind = bitmap_sfu(slab_data->bitmap, + &bin_info->bitmap_info); + *(ptrs + i) = (void *)((uintptr_t)extent_addr_get(slab) + + (uintptr_t)(bin_info->reg_size * regind)); + } +#else + unsigned group = 0; + bitmap_t g = slab_data->bitmap[group]; + unsigned i = 0; + while (i < cnt) { + while (g == 0) { + g = slab_data->bitmap[++group]; + } + size_t shift = group << LG_BITMAP_GROUP_NBITS; + size_t pop = popcount_lu(g); + if (pop > (cnt - i)) { + pop = cnt - i; + } + + /* + * Load from memory locations only once, outside the + * hot loop below. + */ + uintptr_t base = (uintptr_t)extent_addr_get(slab); + uintptr_t regsize = (uintptr_t)bin_info->reg_size; + while (pop--) { + size_t bit = cfs_lu(&g); + size_t regind = shift + bit; + *(ptrs + i) = (void *)(base + regsize * regind); + + i++; + } + slab_data->bitmap[group] = g; + } +#endif + extent_nfree_sub(slab, cnt); +} + #ifndef JEMALLOC_JET static #endif @@ -291,11 +374,11 @@ arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (usize < SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= NBINS) ? index - NBINS : 0; + hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].nmalloc, 1); @@ -307,11 +390,11 @@ arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (usize < SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= NBINS) ? index - NBINS : 0; + hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].ndalloc, 1); @@ -324,6 +407,11 @@ arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, arena_large_malloc_stats_update(tsdn, arena, usize); } +static bool +arena_may_have_muzzy(arena_t *arena) { + return (pages_can_purge_lazy && (arena_muzzy_decay_ms_get(arena) != 0)); +} + extent_t * arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { @@ -338,7 +426,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); - if (extent == NULL) { + if (extent == NULL && arena_may_have_muzzy(arena)) { extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); @@ -743,7 +831,7 @@ static size_t arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, bool all, extent_list_t *decay_extents, bool is_background_thread) { - UNUSED size_t nmadvise, nunmapped; + size_t nmadvise, nunmapped; size_t npurged; if (config_stats) { @@ -834,7 +922,7 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { - UNUSED size_t npurged = arena_decay_stashed(tsdn, arena, + size_t npurged = arena_decay_stashed(tsdn, arena, &extent_hooks, decay, extents, all, &decay_extents, is_background_thread); assert(npurged == npurge); @@ -863,7 +951,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents, is_background_thread); - UNUSED size_t npages_new; + size_t npages_new; if (epoch_advanced) { /* Backlog is updated on epoch advance. */ npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1]; @@ -954,6 +1042,37 @@ arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) { extent_list_remove(&bin->slabs_full, slab); } +static void +arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { + extent_t *slab; + + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + if (bin->slabcur != NULL) { + slab = bin->slabcur; + bin->slabcur = NULL; + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != NULL) { + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + for (slab = extent_list_first(&bin->slabs_full); slab != NULL; + slab = extent_list_first(&bin->slabs_full)) { + arena_bin_slabs_full_remove(arena, bin, slab); + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + if (config_stats) { + bin->stats.curregs = 0; + bin->stats.curslabs = 0; + } + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); +} + void arena_reset(tsd_t *tsd, arena_t *arena) { /* @@ -983,7 +1102,7 @@ arena_reset(tsd_t *tsd, arena_t *arena) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); if (config_stats || (config_prof && opt_prof)) { usize = sz_index2size(alloc_ctx.szind); @@ -999,35 +1118,11 @@ arena_reset(tsd_t *tsd, arena_t *arena) { malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); /* Bins. */ - for (unsigned i = 0; i < NBINS; i++) { - extent_t *slab; - bin_t *bin = &arena->bins[i]; - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (bin->slabcur != NULL) { - slab = bin->slabcur; - bin->slabcur = NULL; - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != - NULL) { - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - for (slab = extent_list_first(&bin->slabs_full); slab != NULL; - slab = extent_list_first(&bin->slabs_full)) { - arena_bin_slabs_full_remove(arena, bin, slab); - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + arena_bin_reset(tsd, arena, + &arena->bins[i].bin_shards[j]); } - if (config_stats) { - bin->stats.curregs = 0; - bin->stats.curslabs = 0; - } - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); @@ -1112,7 +1207,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, } static extent_t * -arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, +arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard, const bin_info_t *bin_info) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -1124,7 +1219,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); - if (slab == NULL) { + if (slab == NULL && arena_may_have_muzzy(arena)) { slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); @@ -1140,7 +1235,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, /* Initialize slab internals. */ arena_slab_data_t *slab_data = extent_slab_data_get(slab); - extent_nfree_set(slab, bin_info->nregs); + extent_nfree_binshard_set(slab, bin_info->nregs, binshard); bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false); arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); @@ -1150,7 +1245,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, static extent_t * arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind) { + szind_t binind, unsigned binshard) { extent_t *slab; const bin_info_t *bin_info; @@ -1166,7 +1261,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, /* Allocate a new slab. */ malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - slab = arena_slab_alloc(tsdn, arena, binind, bin_info); + slab = arena_slab_alloc(tsdn, arena, binind, binshard, bin_info); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); if (slab != NULL) { @@ -1193,7 +1288,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, /* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */ static void * arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind) { + szind_t binind, unsigned binshard) { const bin_info_t *bin_info; extent_t *slab; @@ -1202,7 +1297,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, arena_bin_slabs_full_insert(arena, bin, bin->slabcur); bin->slabcur = NULL; } - slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind); + slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind, binshard); if (bin->slabcur != NULL) { /* * Another thread updated slabcur while this one ran without the @@ -1246,46 +1341,75 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, return arena_slab_reg_alloc(slab, bin_info); } +/* Choose a bin shard and return the locked bin. */ +bin_t * +arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard) { + bin_t *bin; + if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { + *binshard = 0; + } else { + *binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; + } + assert(*binshard < bin_infos[binind].n_shards); + bin = &arena->bins[binind].bin_shards[*binshard]; + malloc_mutex_lock(tsdn, &bin->lock); + + return bin; +} + void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { - unsigned i, nfill; - bin_t *bin; + unsigned i, nfill, cnt; assert(tbin->ncached == 0); if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) { prof_idump(tsdn); } - bin = &arena->bins[binind]; - malloc_mutex_lock(tsdn, &bin->lock); + + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); + for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> - tcache->lg_fill_div[binind]); i < nfill; i++) { + tcache->lg_fill_div[binind]); i < nfill; i += cnt) { extent_t *slab; - void *ptr; if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { - ptr = arena_slab_reg_alloc(slab, &bin_infos[binind]); + unsigned tofill = nfill - i; + cnt = tofill < extent_nfree_get(slab) ? + tofill : extent_nfree_get(slab); + arena_slab_reg_alloc_batch( + slab, &bin_infos[binind], cnt, + tbin->avail - nfill + i); } else { - ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind); - } - if (ptr == NULL) { + cnt = 1; + void *ptr = arena_bin_malloc_hard(tsdn, arena, bin, + binind, binshard); /* * OOM. tbin->avail isn't yet filled down to its first * element, so the successful allocations (if any) must * be moved just before tbin->avail before bailing out. */ - if (i > 0) { - memmove(tbin->avail - i, tbin->avail - nfill, - i * sizeof(void *)); + if (ptr == NULL) { + if (i > 0) { + memmove(tbin->avail - i, + tbin->avail - nfill, + i * sizeof(void *)); + } + break; } - break; + /* Insert such that low regions get used first. */ + *(tbin->avail - nfill + i) = ptr; } if (config_fill && unlikely(opt_junk_alloc)) { - arena_alloc_junk_small(ptr, &bin_infos[binind], true); + for (unsigned j = 0; j < cnt; j++) { + void* ptr = *(tbin->avail - nfill + i + j); + arena_alloc_junk_small(ptr, &bin_infos[binind], + true); + } } - /* Insert such that low regions get used first. */ - *(tbin->avail - nfill + i) = ptr; } if (config_stats) { bin->stats.nmalloc += i; @@ -1320,15 +1444,15 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { size_t usize; extent_t *slab; - assert(binind < NBINS); - bin = &arena->bins[binind]; + assert(binind < SC_NBINS); usize = sz_index2size(binind); + unsigned binshard; + bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); - malloc_mutex_lock(tsdn, &bin->lock); if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { ret = arena_slab_reg_alloc(slab, &bin_infos[binind]); } else { - ret = arena_bin_malloc_hard(tsdn, arena, bin, binind); + ret = arena_bin_malloc_hard(tsdn, arena, bin, binind, binshard); } if (ret == NULL) { @@ -1373,13 +1497,13 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, assert(!tsdn_null(tsdn) || arena != NULL); if (likely(!tsdn_null(tsdn))) { - arena = arena_choose(tsdn_tsd(tsdn), arena); + arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size); } if (unlikely(arena == NULL)) { return NULL; } - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return arena_malloc_small(tsdn, arena, ind, zero); } return large_malloc(tsdn, arena, sz_index2size(ind), zero); @@ -1390,8 +1514,9 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache) { void *ret; - if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE - && (usize & PAGE_MASK) == 0))) { + if (usize <= SC_SMALL_MAXCLASS + && (alignment < PAGE + || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { /* Small; alignment doesn't require special slab placement. */ ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), zero, tcache, true); @@ -1409,8 +1534,8 @@ void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize) { cassert(config_prof); assert(ptr != NULL); - assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); - assert(usize <= SMALL_MAXCLASS); + assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); + assert(usize <= SC_SMALL_MAXCLASS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -1434,15 +1559,15 @@ arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) { cassert(config_prof); assert(ptr != NULL); - extent_szind_set(extent, NBINS); + extent_szind_set(extent, SC_NBINS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, - NBINS, false); + SC_NBINS, false); - assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); + assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); - return LARGE_MINCLASS; + return SC_LARGE_MINCLASS; } void @@ -1499,7 +1624,7 @@ arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_bin_lower_slab(UNUSED tsdn_t *tsdn, arena_t *arena, extent_t *slab, +arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, bin_t *bin) { assert(extent_nfree_get(slab) > 0); @@ -1526,11 +1651,9 @@ arena_bin_lower_slab(UNUSED tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, - void *ptr, bool junked) { +arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *slab, void *ptr, bool junked) { arena_slab_data_t *slab_data = extent_slab_data_get(slab); - szind_t binind = extent_szind_get(slab); - bin_t *bin = &arena->bins[binind]; const bin_info_t *bin_info = &bin_infos[binind]; if (!junked && config_fill && unlikely(opt_junk_free)) { @@ -1554,18 +1677,21 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } void -arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - void *ptr) { - arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, true); +arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr) { + arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, + true); } static void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { szind_t binind = extent_szind_get(extent); - bin_t *bin = &arena->bins[binind]; + unsigned binshard = extent_binshard_get(extent); + bin_t *bin = &arena->bins[binind].bin_shards[binshard]; malloc_mutex_lock(tsdn, &bin->lock); - arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, false); + arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, + false); malloc_mutex_unlock(tsdn, &bin->lock); } @@ -1580,38 +1706,48 @@ arena_dalloc_small(tsdn_t *tsdn, void *ptr) { bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero) { + size_t extra, bool zero, size_t *newsize) { + bool ret; /* Calls with non-zero extra had to clamp extra. */ - assert(extra == 0 || size + extra <= LARGE_MAXCLASS); + assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS); - if (unlikely(size > LARGE_MAXCLASS)) { - return true; + extent_t *extent = iealloc(tsdn, ptr); + if (unlikely(size > SC_LARGE_MAXCLASS)) { + ret = true; + goto done; } - extent_t *extent = iealloc(tsdn, ptr); size_t usize_min = sz_s2u(size); size_t usize_max = sz_s2u(size + extra); - if (likely(oldsize <= SMALL_MAXCLASS && usize_min <= SMALL_MAXCLASS)) { + if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min + <= SC_SMALL_MAXCLASS)) { /* * Avoid moving the allocation if the size class can be left the * same. */ assert(bin_infos[sz_size2index(oldsize)].reg_size == oldsize); - if ((usize_max > SMALL_MAXCLASS || sz_size2index(usize_max) != - sz_size2index(oldsize)) && (size > oldsize || usize_max < - oldsize)) { - return true; + if ((usize_max > SC_SMALL_MAXCLASS + || sz_size2index(usize_max) != sz_size2index(oldsize)) + && (size > oldsize || usize_max < oldsize)) { + ret = true; + goto done; } arena_decay_tick(tsdn, extent_arena_get(extent)); - return false; - } else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) { - return large_ralloc_no_move(tsdn, extent, usize_min, usize_max, + ret = false; + } else if (oldsize >= SC_LARGE_MINCLASS + && usize_max >= SC_LARGE_MINCLASS) { + ret = large_ralloc_no_move(tsdn, extent, usize_min, usize_max, zero); + } else { + ret = true; } +done: + assert(extent == iealloc(tsdn, ptr)); + *newsize = extent_usize_get(extent); - return true; + return ret; } static void * @@ -1622,7 +1758,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, zero, tcache, true); } usize = sz_sa2u(usize, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } return ipalloct(tsdn, usize, alignment, zero, tcache, arena); @@ -1630,22 +1766,30 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, void * arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache) { + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args) { size_t usize = sz_s2u(size); - if (unlikely(usize == 0 || size > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) { return NULL; } - if (likely(usize <= SMALL_MAXCLASS)) { + if (likely(usize <= SC_SMALL_MAXCLASS)) { /* Try to avoid moving the allocation. */ - if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero)) { + UNUSED size_t newsize; + if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero, + &newsize)) { + hook_invoke_expand(hook_args->is_realloc + ? hook_expand_realloc : hook_expand_rallocx, + ptr, oldsize, usize, (uintptr_t)ptr, + hook_args->args); return ptr; } } - if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) { - return large_ralloc(tsdn, arena, iealloc(tsdn, ptr), usize, - alignment, zero, tcache); + if (oldsize >= SC_LARGE_MINCLASS + && usize >= SC_LARGE_MINCLASS) { + return large_ralloc(tsdn, arena, ptr, usize, + alignment, zero, tcache, hook_args); } /* @@ -1658,11 +1802,16 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, return NULL; } + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + /* * Junk/zero-filling were already done by * ipalloc()/arena_malloc(). */ - size_t copysize = (usize < oldsize) ? usize : oldsize; memcpy(ret, ptr, copysize); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); @@ -1720,8 +1869,7 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, if (new_limit != NULL) { size_t limit = *new_limit; /* Grow no more than the new limit. */ - if ((new_ind = sz_psz2ind(limit + 1) - 1) > - EXTENT_GROW_MAX_PIND) { + if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) { return true; } } @@ -1773,7 +1921,12 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } } - arena = (arena_t *)base_alloc(tsdn, base, sizeof(arena_t), CACHELINE); + unsigned nbins_total = 0; + for (i = 0; i < SC_NBINS; i++) { + nbins_total += bin_infos[i].n_shards; + } + size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; + arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); if (arena == NULL) { goto label_error; } @@ -1865,7 +2018,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } arena->extent_grow_next = sz_psz2ind(HUGEPAGE); - arena->retain_grow_limit = EXTENT_GROW_MAX_PIND; + arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS); if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow", WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { goto label_error; @@ -1878,12 +2031,20 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } /* Initialize bins. */ - for (i = 0; i < NBINS; i++) { - bool err = bin_init(&arena->bins[i]); - if (err) { - goto label_error; + uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); + atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); + for (i = 0; i < SC_NBINS; i++) { + unsigned nshards = bin_infos[i].n_shards; + arena->bins[i].bin_shards = (bin_t *)bin_addr; + bin_addr += nshards * sizeof(bin_t); + for (unsigned j = 0; j < nshards; j++) { + bool err = bin_init(&arena->bins[i].bin_shards[j]); + if (err) { + goto label_error; + } } } + assert(bin_addr == (uintptr_t)arena + arena_size); arena->base = base; /* Set arena before creating background threads. */ @@ -1900,8 +2061,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ assert(!tsdn_null(tsdn)); pre_reentrancy(tsdn_tsd(tsdn), arena); - if (hooks_arena_new_hook) { - hooks_arena_new_hook(); + if (test_hooks_arena_new_hook) { + test_hooks_arena_new_hook(); } post_reentrancy(tsdn_tsd(tsdn)); } @@ -1914,20 +2075,75 @@ label_error: return NULL; } +arena_t * +arena_choose_huge(tsd_t *tsd) { + /* huge_arena_ind can be 0 during init (will use a0). */ + if (huge_arena_ind == 0) { + assert(!malloc_initialized()); + } + + arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false); + if (huge_arena == NULL) { + /* Create the huge arena on demand. */ + assert(huge_arena_ind != 0); + huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true); + if (huge_arena == NULL) { + return NULL; + } + /* + * Purge eagerly for huge allocations, because: 1) number of + * huge allocations is usually small, which means ticker based + * decay is not reliable; and 2) less immediate reuse is + * expected for huge allocations. + */ + if (arena_dirty_decay_ms_default_get() > 0) { + arena_dirty_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); + } + if (arena_muzzy_decay_ms_default_get() > 0) { + arena_muzzy_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); + } + } + + return huge_arena; +} + +bool +arena_init_huge(void) { + bool huge_enabled; + + /* The threshold should be large size class. */ + if (opt_oversize_threshold > SC_LARGE_MAXCLASS || + opt_oversize_threshold < SC_LARGE_MINCLASS) { + opt_oversize_threshold = 0; + oversize_threshold = SC_LARGE_MAXCLASS + PAGE; + huge_enabled = false; + } else { + /* Reserve the index for the huge arena. */ + huge_arena_ind = narenas_total_get(); + oversize_threshold = opt_oversize_threshold; + huge_enabled = true; + } + + return huge_enabled; +} + +bool +arena_is_huge(unsigned arena_ind) { + if (huge_arena_ind == 0) { + return false; + } + return (arena_ind == huge_arena_ind); +} + void -arena_boot(void) { +arena_boot(sc_data_t *sc_data) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms); -#define REGIND_bin_yes(index, reg_size) \ - div_init(&arena_binind_div_info[(index)], (reg_size)); -#define REGIND_bin_no(index, reg_size) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ - lg_delta_lookup) \ - REGIND_bin_##bin(index, (1U<<lg_grp) + (ndelta << lg_delta)) - SIZE_CLASSES -#undef REGIND_bin_yes -#undef REGIND_bin_no -#undef SC + for (unsigned i = 0; i < SC_NBINS; i++) { + sc_t *sc = &sc_data->sc[i]; + div_init(&arena_binind_div_info[i], + (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta)); + } } void @@ -1972,8 +2188,10 @@ arena_prefork6(tsdn_t *tsdn, arena_t *arena) { void arena_prefork7(tsdn_t *tsdn, arena_t *arena) { - for (unsigned i = 0; i < NBINS; i++) { - bin_prefork(tsdn, &arena->bins[i]); + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_prefork(tsdn, &arena->bins[i].bin_shards[j]); + } } } @@ -1981,8 +2199,11 @@ void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { unsigned i; - for (i = 0; i < NBINS; i++) { - bin_postfork_parent(tsdn, &arena->bins[i]); + for (i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_postfork_parent(tsdn, + &arena->bins[i].bin_shards[j]); + } } malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); base_postfork_parent(tsdn, arena->base); @@ -2025,8 +2246,10 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { } } - for (i = 0; i < NBINS; i++) { - bin_postfork_child(tsdn, &arena->bins[i]); + for (i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_postfork_child(tsdn, &arena->bins[i].bin_shards[j]); + } } malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); diff --git a/dep/jemalloc/src/background_thread.c b/dep/jemalloc/src/background_thread.c index 3517a3bb8ed..5ed6c1c959b 100644 --- a/dep/jemalloc/src/background_thread.c +++ b/dep/jemalloc/src/background_thread.c @@ -4,6 +4,8 @@ #include "jemalloc/internal/assert.h" +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + /******************************************************************************/ /* Data. */ @@ -11,7 +13,7 @@ #define BACKGROUND_THREAD_DEFAULT false /* Read-only after initialization. */ bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; -size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT; +size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; /* Used for thread creation, termination and stats. */ malloc_mutex_t background_thread_lock; @@ -22,13 +24,9 @@ size_t max_background_threads; /* Thread info per-index. */ background_thread_info_t *background_thread_info; -/* False if no necessary runtime support. */ -bool can_enable_background_thread; - /******************************************************************************/ #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER -#include <dlfcn.h> static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, void *(*)(void *), void *__restrict); @@ -81,7 +79,7 @@ background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { } static inline bool -set_current_thread_affinity(UNUSED int cpu) { +set_current_thread_affinity(int cpu) { #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) cpu_set_t cpuset; CPU_ZERO(&cpuset); @@ -510,6 +508,8 @@ background_thread_entry(void *ind_arg) { assert(thread_ind < max_background_threads); #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); +#elif defined(__FreeBSD__) + pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); #endif if (opt_percpu_arena != percpu_arena_disabled) { set_current_thread_affinity((int)thread_ind); @@ -534,9 +534,8 @@ background_thread_init(tsd_t *tsd, background_thread_info_t *info) { n_background_threads++; } -/* Create a new background thread if needed. */ -bool -background_thread_create(tsd_t *tsd, unsigned arena_ind) { +static bool +background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { assert(have_background_thread); malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); @@ -589,6 +588,19 @@ background_thread_create(tsd_t *tsd, unsigned arena_ind) { return false; } +/* Create a new background thread if needed. */ +bool +background_thread_create(tsd_t *tsd, unsigned arena_ind) { + assert(have_background_thread); + + bool ret; + malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); + ret = background_thread_create_locked(tsd, arena_ind); + malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); + + return ret; +} + bool background_threads_enable(tsd_t *tsd) { assert(n_background_threads == 0); @@ -622,7 +634,7 @@ background_threads_enable(tsd_t *tsd) { } } - return background_thread_create(tsd, 0); + return background_thread_create_locked(tsd, 0); } bool @@ -807,21 +819,34 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { #undef BILLION #undef BACKGROUND_THREAD_MIN_INTERVAL_NS +#ifdef JEMALLOC_HAVE_DLSYM +#include <dlfcn.h> +#endif + static bool pthread_create_fptr_init(void) { if (pthread_create_fptr != NULL) { return false; } + /* + * Try the next symbol first, because 1) when use lazy_lock we have a + * wrapper for pthread_create; and 2) application may define its own + * wrapper as well (and can call malloc within the wrapper). + */ +#ifdef JEMALLOC_HAVE_DLSYM pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); +#else + pthread_create_fptr = NULL; +#endif if (pthread_create_fptr == NULL) { - can_enable_background_thread = false; - if (config_lazy_lock || opt_background_thread) { + if (config_lazy_lock) { malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, " "\"pthread_create\")\n"); abort(); + } else { + /* Fall back to the default symbol. */ + pthread_create_fptr = pthread_create; } - } else { - can_enable_background_thread = true; } return false; @@ -866,9 +891,8 @@ background_thread_boot1(tsdn_t *tsdn) { assert(have_background_thread); assert(narenas_total_get() > 0); - if (opt_max_background_threads == MAX_BACKGROUND_THREAD_LIMIT && - ncpus < MAX_BACKGROUND_THREAD_LIMIT) { - opt_max_background_threads = ncpus; + if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { + opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; } max_background_threads = opt_max_background_threads; diff --git a/dep/jemalloc/src/base.c b/dep/jemalloc/src/base.c index b0324b5d758..f3c61661a20 100644 --- a/dep/jemalloc/src/base.c +++ b/dep/jemalloc/src/base.c @@ -262,8 +262,8 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, */ size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size + usize)); - pszind_t pind_next = (*pind_last + 1 < NPSIZES) ? *pind_last + 1 : - *pind_last; + pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ? + *pind_last + 1 : *pind_last; size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); size_t block_size = (min_block_size > next_block_size) ? min_block_size : next_block_size; @@ -372,7 +372,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { base->extent_sn_next = extent_sn_next; base->blocks = block; base->auto_thp_switched = false; - for (szind_t i = 0; i < NSIZES; i++) { + for (szind_t i = 0; i < SC_NSIZES; i++) { extent_heap_new(&base->avail[i]); } if (config_stats) { @@ -426,7 +426,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, extent_t *extent = NULL; malloc_mutex_lock(tsdn, &base->mtx); - for (szind_t i = sz_size2index(asize); i < NSIZES; i++) { + for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) { extent = extent_heap_remove_first(&base->avail[i]); if (extent != NULL) { /* Use existing space. */ diff --git a/dep/jemalloc/src/bin.c b/dep/jemalloc/src/bin.c index 0886bc4ea92..bca6b12c352 100644 --- a/dep/jemalloc/src/bin.c +++ b/dep/jemalloc/src/bin.c @@ -1,23 +1,68 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" +#include "jemalloc/internal/assert.h" #include "jemalloc/internal/bin.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/witness.h" -const bin_info_t bin_infos[NBINS] = { -#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ - {reg_size, slab_size, nregs, BITMAP_INFO_INITIALIZER(nregs)}, -#define BIN_INFO_bin_no(reg_size, slab_size, nregs) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ - lg_delta_lookup) \ - BIN_INFO_bin_##bin((1U<<lg_grp) + (ndelta<<lg_delta), \ - (pgs << LG_PAGE), (pgs << LG_PAGE) / ((1U<<lg_grp) + \ - (ndelta<<lg_delta))) - SIZE_CLASSES -#undef BIN_INFO_bin_yes -#undef BIN_INFO_bin_no -#undef SC -}; +bin_info_t bin_infos[SC_NBINS]; + +static void +bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + bin_info_t bin_infos[SC_NBINS]) { + for (unsigned i = 0; i < SC_NBINS; i++) { + bin_info_t *bin_info = &bin_infos[i]; + sc_t *sc = &sc_data->sc[i]; + bin_info->reg_size = ((size_t)1U << sc->lg_base) + + ((size_t)sc->ndelta << sc->lg_delta); + bin_info->slab_size = (sc->pgs << LG_PAGE); + bin_info->nregs = + (uint32_t)(bin_info->slab_size / bin_info->reg_size); + bin_info->n_shards = bin_shard_sizes[i]; + bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( + bin_info->nregs); + bin_info->bitmap_info = bitmap_info; + } +} + +bool +bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards) { + if (nshards > BIN_SHARDS_MAX || nshards == 0) { + return true; + } + + if (start_size > SC_SMALL_MAXCLASS) { + return false; + } + if (end_size > SC_SMALL_MAXCLASS) { + end_size = SC_SMALL_MAXCLASS; + } + + /* Compute the index since this may happen before sz init. */ + szind_t ind1 = sz_size2index_compute(start_size); + szind_t ind2 = sz_size2index_compute(end_size); + for (unsigned i = ind1; i <= ind2; i++) { + bin_shard_sizes[i] = (unsigned)nshards; + } + + return false; +} + +void +bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { + /* Load the default number of shards. */ + for (unsigned i = 0; i < SC_NBINS; i++) { + bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT; + } +} + +void +bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { + assert(sc_data->initialized); + bin_infos_init(sc_data, bin_shard_sizes, bin_infos); +} bool bin_init(bin_t *bin) { diff --git a/dep/jemalloc/src/ckh.c b/dep/jemalloc/src/ckh.c index e95e0a3ed59..1bf6df5a115 100644 --- a/dep/jemalloc/src/ckh.c +++ b/dep/jemalloc/src/ckh.c @@ -275,7 +275,8 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) { lg_curcells++; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } @@ -320,7 +321,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return; } tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL, @@ -396,7 +397,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh->keycomp = keycomp; usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } diff --git a/dep/jemalloc/src/ctl.c b/dep/jemalloc/src/ctl.c index 1e713a3d104..09310a9d191 100644 --- a/dep/jemalloc/src/ctl.c +++ b/dep/jemalloc/src/ctl.c @@ -8,7 +8,7 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" /******************************************************************************/ @@ -85,6 +85,7 @@ CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) CTL_PROTO(opt_narenas) CTL_PROTO(opt_percpu_arena) +CTL_PROTO(opt_oversize_threshold) CTL_PROTO(opt_background_thread) CTL_PROTO(opt_max_background_threads) CTL_PROTO(opt_dirty_decay_ms) @@ -126,6 +127,7 @@ INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_slab_size) +CTL_PROTO(arenas_bin_i_nshards) INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_lextent_i_size) INDEX_PROTO(arenas_lextent_i) @@ -147,6 +149,8 @@ CTL_PROTO(prof_gdump) CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) CTL_PROTO(lg_prof_sample) +CTL_PROTO(prof_log_start) +CTL_PROTO(prof_log_stop) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) @@ -170,6 +174,13 @@ CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) CTL_PROTO(stats_arenas_i_lextents_j_nrequests) CTL_PROTO(stats_arenas_i_lextents_j_curlextents) INDEX_PROTO(stats_arenas_i_lextents_j) +CTL_PROTO(stats_arenas_i_extents_j_ndirty) +CTL_PROTO(stats_arenas_i_extents_j_nmuzzy) +CTL_PROTO(stats_arenas_i_extents_j_nretained) +CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) +CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) +CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) +INDEX_PROTO(stats_arenas_i_extents_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_uptime) CTL_PROTO(stats_arenas_i_dss) @@ -180,6 +191,7 @@ CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_pmuzzy) CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_retained) +CTL_PROTO(stats_arenas_i_extent_avail) CTL_PROTO(stats_arenas_i_dirty_npurge) CTL_PROTO(stats_arenas_i_dirty_nmadvise) CTL_PROTO(stats_arenas_i_dirty_purged) @@ -202,6 +214,8 @@ CTL_PROTO(stats_metadata_thp) CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) CTL_PROTO(stats_retained) +CTL_PROTO(experimental_hooks_install) +CTL_PROTO(experimental_hooks_remove) #define MUTEX_STATS_CTL_PROTO_GEN(n) \ CTL_PROTO(stats_##n##_num_ops) \ @@ -286,6 +300,7 @@ static const ctl_named_node_t opt_node[] = { {NAME("dss"), CTL(opt_dss)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("percpu_arena"), CTL(opt_percpu_arena)}, + {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, {NAME("background_thread"), CTL(opt_background_thread)}, {NAME("max_background_threads"), CTL(opt_max_background_threads)}, {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, @@ -341,7 +356,8 @@ static const ctl_indexed_node_t arena_node[] = { static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} + {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, + {NAME("nshards"), CTL(arenas_bin_i_nshards)} }; static const ctl_named_node_t super_arenas_bin_i_node[] = { {NAME(""), CHILD(named, arenas_bin_i)} @@ -385,9 +401,10 @@ static const ctl_named_node_t prof_node[] = { {NAME("gdump"), CTL(prof_gdump)}, {NAME("reset"), CTL(prof_reset)}, {NAME("interval"), CTL(prof_interval)}, - {NAME("lg_sample"), CTL(lg_prof_sample)} + {NAME("lg_sample"), CTL(lg_prof_sample)}, + {NAME("log_start"), CTL(prof_log_start)}, + {NAME("log_stop"), CTL(prof_log_stop)} }; - static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, @@ -458,6 +475,23 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { {INDEX(stats_arenas_i_lextents_j)} }; +static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { + {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, + {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, + {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, + {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, + {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, + {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} +}; + +static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_extents_j)} +}; + +static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { + {INDEX(stats_arenas_i_extents_j)} +}; + #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) MUTEX_PROF_ARENA_MUTEXES #undef OP @@ -479,6 +513,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("retained"), CTL(stats_arenas_i_retained)}, + {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, @@ -494,6 +529,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, + {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { @@ -536,6 +572,15 @@ static const ctl_named_node_t stats_node[] = { {NAME("arenas"), CHILD(indexed, stats_arenas)} }; +static const ctl_named_node_t hooks_node[] = { + {NAME("install"), CTL(experimental_hooks_install)}, + {NAME("remove"), CTL(experimental_hooks_remove)}, +}; + +static const ctl_named_node_t experimental_node[] = { + {NAME("hooks"), CHILD(named, hooks)} +}; + static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, @@ -548,7 +593,8 @@ static const ctl_named_node_t root_node[] = { {NAME("arena"), CHILD(indexed, arena)}, {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, - {NAME("stats"), CHILD(named, stats)} + {NAME("stats"), CHILD(named, stats)}, + {NAME("experimental"), CHILD(named, experimental)} }; static const ctl_named_node_t super_root_node[] = { {NAME(""), CHILD(named, root)} @@ -696,10 +742,12 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena->astats->nmalloc_small = 0; ctl_arena->astats->ndalloc_small = 0; ctl_arena->astats->nrequests_small = 0; - memset(ctl_arena->astats->bstats, 0, NBINS * + memset(ctl_arena->astats->bstats, 0, SC_NBINS * sizeof(bin_stats_t)); - memset(ctl_arena->astats->lstats, 0, (NSIZES - NBINS) * + memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * sizeof(arena_stats_large_t)); + memset(ctl_arena->astats->estats, 0, SC_NPSIZES * + sizeof(arena_stats_extents_t)); } } @@ -713,9 +761,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, - ctl_arena->astats->lstats); + ctl_arena->astats->lstats, ctl_arena->astats->estats); - for (i = 0; i < NBINS; i++) { + for (i = 0; i < SC_NBINS; i++) { ctl_arena->astats->allocated_small += ctl_arena->astats->bstats[i].curregs * sz_index2size(i); @@ -760,6 +808,8 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, &astats->astats.mapped); accum_atomic_zu(&sdstats->astats.retained, &astats->astats.retained); + accum_atomic_zu(&sdstats->astats.extent_avail, + &astats->astats.extent_avail); } ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, @@ -827,7 +877,8 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->astats.uptime = astats->astats.uptime; } - for (i = 0; i < NBINS; i++) { + /* Merge bin stats. */ + for (i = 0; i < SC_NBINS; i++) { sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sdstats->bstats[i].nrequests += @@ -853,7 +904,8 @@ MUTEX_PROF_ARENA_MUTEXES &astats->bstats[i].mutex_data); } - for (i = 0; i < NSIZES - NBINS; i++) { + /* Merge stats for large allocations. */ + for (i = 0; i < SC_NSIZES - SC_NBINS; i++) { ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, &astats->lstats[i].nmalloc); ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, @@ -867,6 +919,22 @@ MUTEX_PROF_ARENA_MUTEXES assert(astats->lstats[i].curlextents == 0); } } + + /* Merge extents stats. */ + for (i = 0; i < SC_NPSIZES; i++) { + accum_atomic_zu(&sdstats->estats[i].ndirty, + &astats->estats[i].ndirty); + accum_atomic_zu(&sdstats->estats[i].nmuzzy, + &astats->estats[i].nmuzzy); + accum_atomic_zu(&sdstats->estats[i].nretained, + &astats->estats[i].nretained); + accum_atomic_zu(&sdstats->estats[i].dirty_bytes, + &astats->estats[i].dirty_bytes); + accum_atomic_zu(&sdstats->estats[i].muzzy_bytes, + &astats->estats[i].muzzy_bytes); + accum_atomic_zu(&sdstats->estats[i].retained_bytes, + &astats->estats[i].retained_bytes); + } } } @@ -1378,8 +1446,8 @@ label_return: \ #define CTL_RO_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1421,8 +1489,8 @@ label_return: \ */ #define CTL_RO_NL_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1440,8 +1508,8 @@ label_return: \ #define CTL_RO_NL_GEN(n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1475,8 +1543,8 @@ label_return: \ #define CTL_RO_CONFIG_GEN(n, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1494,8 +1562,8 @@ label_return: \ CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int -epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; UNUSED uint64_t newval; @@ -1513,8 +1581,9 @@ label_return: } static int -background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +background_thread_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { int ret; bool oldval; @@ -1544,13 +1613,6 @@ background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, background_thread_enabled_set(tsd_tsdn(tsd), newval); if (newval) { - if (!can_enable_background_thread) { - malloc_printf("<jemalloc>: Error in dlsym(" - "RTLD_NEXT, \"pthread_create\"). Cannot " - "enable background_thread\n"); - ret = EFAULT; - goto label_return; - } if (background_threads_enable(tsd)) { ret = EFAULT; goto label_return; @@ -1571,8 +1633,9 @@ label_return: } static int -max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +max_background_threads_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; size_t oldval; @@ -1605,13 +1668,6 @@ max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, } if (background_thread_enabled()) { - if (!can_enable_background_thread) { - malloc_printf("<jemalloc>: Error in dlsym(" - "RTLD_NEXT, \"pthread_create\"). Cannot " - "enable background_thread\n"); - ret = EFAULT; - goto label_return; - } background_thread_enabled_set(tsd_tsdn(tsd), false); if (background_threads_disable(tsd)) { ret = EFAULT; @@ -1660,6 +1716,7 @@ CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], const char *) +CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) @@ -1690,8 +1747,8 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) /******************************************************************************/ static int -thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; arena_t *oldarena; unsigned newind, oldind; @@ -1755,8 +1812,9 @@ CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, tsd_thread_deallocatedp_get, uint64_t *) static int -thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -1776,8 +1834,9 @@ label_return: } static int -thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; if (!tcache_available(tsd)) { @@ -1796,8 +1855,9 @@ label_return: } static int -thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; if (!config_prof) { @@ -1827,8 +1887,9 @@ label_return: } static int -thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -1857,8 +1918,8 @@ label_return: /******************************************************************************/ static int -tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1875,8 +1936,8 @@ label_return: } static int -tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1895,8 +1956,8 @@ label_return: } static int -tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -2044,9 +2105,8 @@ arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); if (background_thread_enabled()) { - unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - &background_thread_info[ind]; + background_thread_info_get(arena_ind); assert(info->state == background_thread_started); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_paused; @@ -2059,9 +2119,8 @@ static void arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { if (background_thread_enabled()) { - unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - &background_thread_info[ind]; + background_thread_info_get(arena_ind); assert(info->state == background_thread_paused); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_started; @@ -2217,6 +2276,17 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ret = EINVAL; goto label_return; } + if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { + /* + * By default the huge arena purges eagerly. If it is + * set to non-zero decay time afterwards, background + * thread might be needed. + */ + if (background_thread_create(tsd, arena_ind)) { + ret = EFAULT; + goto label_return; + } + } if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp)) { @@ -2300,8 +2370,9 @@ label_return: } static int -arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; unsigned arena_ind; arena_t *arena; @@ -2336,7 +2407,8 @@ label_return: } static const ctl_named_node_t * -arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { +arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, + size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -2361,8 +2433,8 @@ label_return: /******************************************************************************/ static int -arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned narenas; @@ -2382,8 +2454,9 @@ label_return: } static int -arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { +arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen, bool dirty) { int ret; if (oldp != NULL && oldlenp != NULL) { @@ -2425,34 +2498,36 @@ arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) +CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) static const ctl_named_node_t * -arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { - if (i > NBINS) { +arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + if (i > SC_NBINS) { return NULL; } return super_arenas_bin_i_node; } -CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned) -CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(NBINS+(szind_t)mib[2]), +CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) +CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), size_t) static const ctl_named_node_t * -arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { - if (i > NSIZES - NBINS) { +arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + if (i > SC_NSIZES - SC_NBINS) { return NULL; } return super_arenas_lextent_i_node; } static int -arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; extent_hooks_t *extent_hooks; unsigned arena_ind; @@ -2474,8 +2549,9 @@ label_return: } static int -arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; unsigned arena_ind; void *ptr; @@ -2506,8 +2582,9 @@ label_return: /******************************************************************************/ static int -prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -2533,8 +2610,8 @@ label_return: } static int -prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2559,8 +2636,8 @@ label_return: } static int -prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; const char *filename = NULL; @@ -2582,8 +2659,8 @@ label_return: } static int -prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2608,8 +2685,8 @@ label_return: } static int -prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; size_t lg_sample = lg_prof_sample; @@ -2633,6 +2710,44 @@ label_return: CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) +static int +prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + const char *filename = NULL; + + if (!config_prof) { + return ENOENT; + } + + WRITEONLY(); + WRITE(filename, const char *); + + if (prof_log_start(tsd_tsdn(tsd), filename)) { + ret = EFAULT; + goto label_return; + } + + ret = 0; +label_return: + return ret; +} + +static int +prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + if (!config_prof) { + return ENOENT; + } + + if (prof_log_stop(tsd_tsdn(tsd))) { + return EFAULT; + } + + return 0; +} + /******************************************************************************/ CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) @@ -2667,6 +2782,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, CTL_RO_CGEN(config_stats, stats_arenas_i_retained, atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, + atomic_load_zu(&arenas_i(mib[2])->astats->astats.extent_avail, + ATOMIC_RELAXED), + size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, ctl_arena_stats_read_u64( @@ -2765,8 +2884,9 @@ RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, /* Resets all mutex stats, including global, arena and bin mutexes. */ static int -stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { if (!config_stats) { return ENOENT; } @@ -2806,9 +2926,11 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, MUTEX_PROF_RESET(arena->tcache_ql_mtx); MUTEX_PROF_RESET(arena->base->mtx); - for (szind_t i = 0; i < NBINS; i++) { - bin_t *bin = &arena->bins[i]; - MUTEX_PROF_RESET(bin->lock); + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = &arena->bins[i].bin_shards[j]; + MUTEX_PROF_RESET(bin->lock); + } } } #undef MUTEX_PROF_RESET @@ -2835,9 +2957,9 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) static const ctl_named_node_t * -stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t j) { - if (j > NBINS) { +stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j > SC_NBINS) { return NULL; } return super_stats_arenas_i_bins_j_node; @@ -2856,16 +2978,51 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) static const ctl_named_node_t * -stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t j) { - if (j > NSIZES - NBINS) { +stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j > SC_NSIZES - SC_NBINS) { return NULL; } return super_stats_arenas_i_lextents_j_node; } +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].ndirty, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].nretained, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, + ATOMIC_RELAXED), size_t); + static const ctl_named_node_t * -stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { +stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j >= SC_NPSIZES) { + return NULL; + } + return super_stats_arenas_i_extents_j_node; +} + +static const ctl_named_node_t * +stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { const ctl_named_node_t *ret; size_t a; @@ -2881,3 +3038,48 @@ label_return: malloc_mutex_unlock(tsdn, &ctl_mtx); return ret; } + +static int +experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + if (oldp == NULL || oldlenp == NULL|| newp == NULL) { + ret = EINVAL; + goto label_return; + } + /* + * Note: this is a *private* struct. This is an experimental interface; + * forcing the user to know the jemalloc internals well enough to + * extract the ABI hopefully ensures nobody gets too comfortable with + * this API, which can change at a moment's notice. + */ + hooks_t hooks; + WRITE(hooks, hooks_t); + void *handle = hook_install(tsd_tsdn(tsd), &hooks); + if (handle == NULL) { + ret = EAGAIN; + goto label_return; + } + READ(handle, void *); + + ret = 0; +label_return: + return ret; +} + +static int +experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + WRITEONLY(); + void *handle = NULL; + WRITE(handle, void *); + if (handle == NULL) { + ret = EINVAL; + goto label_return; + } + hook_remove(tsd_tsdn(tsd), handle); + ret = 0; +label_return: + return ret; +} diff --git a/dep/jemalloc/src/extent.c b/dep/jemalloc/src/extent.c index 09d6d771817..62086c7d7b5 100644 --- a/dep/jemalloc/src/extent.c +++ b/dep/jemalloc/src/extent.c @@ -20,7 +20,7 @@ mutex_pool_t extent_mutex_pool; size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; static const bitmap_info_t extents_bitmap_info = - BITMAP_INFO_INITIALIZER(NPSIZES+1); + BITMAP_INFO_INITIALIZER(SC_NPSIZES+1); static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, @@ -119,9 +119,13 @@ static void extent_record(tsdn_t *tsdn, arena_t *arena, /******************************************************************************/ -ph_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, ph_link, +#define ATTR_NONE /* does nothing */ + +ph_gen(ATTR_NONE, extent_avail_, extent_tree_t, extent_t, ph_link, extent_esnead_comp) +#undef ATTR_NONE + typedef enum { lock_result_success, lock_result_failure, @@ -130,13 +134,16 @@ typedef enum { static lock_result_t extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, - extent_t **result) { + extent_t **result, bool inactive_only) { extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree, elm, true); - if (extent1 == NULL) { + /* Slab implies active extents and should be skipped. */ + if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn, + &extents_rtree, elm, true))) { return lock_result_no_extent; } + /* * It's possible that the extent changed out from under us, and with it * the leaf->extent mapping. We have to recheck while holding the lock. @@ -159,7 +166,8 @@ extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, * address, and NULL otherwise. */ static extent_t * -extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { +extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr, + bool inactive_only) { extent_t *ret = NULL; rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)addr, false, false); @@ -168,7 +176,8 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { } lock_result_t lock_result; do { - lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret); + lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret, + inactive_only); } while (lock_result == lock_result_failure); return ret; } @@ -182,6 +191,7 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) { return base_alloc_extent(tsdn, arena->base); } extent_avail_remove(&arena->extent_avail, extent); + atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); return extent; } @@ -190,6 +200,7 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); extent_avail_insert(&arena->extent_avail, extent); + atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); } @@ -255,7 +266,7 @@ extent_size_quantize_ceil(size_t size) { size_t ret; assert(size > 0); - assert(size - sz_large_pad <= LARGE_MAXCLASS); + assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); assert((size & PAGE_MASK) == 0); ret = extent_size_quantize_floor(size); @@ -284,7 +295,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, malloc_mutex_rank_exclusive)) { return true; } - for (unsigned i = 0; i < NPSIZES+1; i++) { + for (unsigned i = 0; i < SC_NPSIZES + 1; i++) { extent_heap_new(&extents->heaps[i]); } bitmap_init(extents->bitmap, &extents_bitmap_info, true); @@ -305,6 +316,32 @@ extents_npages_get(extents_t *extents) { return atomic_load_zu(&extents->npages, ATOMIC_RELAXED); } +size_t +extents_nextents_get(extents_t *extents, pszind_t pind) { + return atomic_load_zu(&extents->nextents[pind], ATOMIC_RELAXED); +} + +size_t +extents_nbytes_get(extents_t *extents, pszind_t pind) { + return atomic_load_zu(&extents->nbytes[pind], ATOMIC_RELAXED); +} + +static void +extents_stats_add(extents_t *extent, pszind_t pind, size_t sz) { + size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nextents[pind], cur + 1, ATOMIC_RELAXED); + cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nbytes[pind], cur + sz, ATOMIC_RELAXED); +} + +static void +extents_stats_sub(extents_t *extent, pszind_t pind, size_t sz) { + size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nextents[pind], cur - 1, ATOMIC_RELAXED); + cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nbytes[pind], cur - sz, ATOMIC_RELAXED); +} + static void extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &extents->mtx); @@ -318,6 +355,11 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { (size_t)pind); } extent_heap_insert(&extents->heaps[pind], extent); + + if (config_stats) { + extents_stats_add(extents, pind, size); + } + extent_list_append(&extents->lru, extent); size_t npages = size >> LG_PAGE; /* @@ -340,6 +382,11 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { size_t psz = extent_size_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); extent_heap_remove(&extents->heaps[pind], extent); + + if (config_stats) { + extents_stats_sub(extents, pind, size); + } + if (extent_heap_empty(&extents->heaps[pind])) { bitmap_set(extents->bitmap, &extents_bitmap_info, (size_t)pind); @@ -371,7 +418,7 @@ extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, &extents_bitmap_info, (size_t)pind); i < pind_max; i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { - assert(i < NPSIZES); + assert(i < SC_NPSIZES); assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); uintptr_t base = (uintptr_t)extent_base_get(extent); @@ -401,7 +448,7 @@ extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)pind); - if (i < NPSIZES+1) { + if (i < SC_NPSIZES + 1) { /* * In order to reduce fragmentation, avoid reusing and splitting * large extents for much smaller sizes. @@ -429,8 +476,9 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, - &extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i = - (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, + &extents_bitmap_info, (size_t)pind); + i < SC_NPSIZES + 1; + i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); @@ -438,10 +486,10 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, if (ret == NULL || extent_snad_comp(extent, ret) < 0) { ret = extent; } - if (i == NPSIZES) { + if (i == SC_NPSIZES) { break; } - assert(i < NPSIZES); + assert(i < SC_NPSIZES); } return ret; @@ -748,6 +796,7 @@ extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) { if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true, &elm_a, &elm_b)) { + extent_unlock(tsdn, extent); return true; } @@ -817,7 +866,7 @@ extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) { extent_lock(tsdn, extent); - extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, NSIZES, false); + extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -874,7 +923,8 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); extent_t *extent; if (new_addr != NULL) { - extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr); + extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr, + false); if (extent != NULL) { /* * We might null-out extent to report an error, but we @@ -958,7 +1008,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, if (leadsize != 0) { *lead = *extent; *extent = extent_split_impl(tsdn, arena, r_extent_hooks, - *lead, leadsize, NSIZES, false, esize + trailsize, szind, + *lead, leadsize, SC_NSIZES, false, esize + trailsize, szind, slab, growing_retained); if (*extent == NULL) { *to_leak = *lead; @@ -970,7 +1020,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, /* Split the trail. */ if (trailsize != 0) { *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, - esize, szind, slab, trailsize, NSIZES, false, + esize, szind, slab, trailsize, SC_NSIZES, false, growing_retained); if (*trail == NULL) { *to_leak = *extent; @@ -987,7 +1037,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, * splitting occurred. */ extent_szind_set(*extent, szind); - if (szind != NSIZES) { + if (szind != SC_NSIZES) { rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_addr_get(*extent), szind, slab); if (slab && extent_size_get(*extent) > PAGE) { @@ -1045,14 +1095,25 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, extent_deregister_no_gdump_sub(tsdn, to_leak); extents_leak(tsdn, arena, r_extent_hooks, extents, to_leak, growing_retained); - assert(extent_lock_from_addr(tsdn, rtree_ctx, leak) - == NULL); + assert(extent_lock_from_addr(tsdn, rtree_ctx, leak, + false) == NULL); } return NULL; } unreachable(); } +static bool +extent_need_manual_zero(arena_t *arena) { + /* + * Need to manually zero the extent on repopulating if either; 1) non + * default extent hooks installed (in which case the purge semantics may + * change); or 2) transparent huge pages enabled. + */ + return (!arena_has_default_hooks(arena) || + (opt_thp == thp_mode_always)); +} + /* * Tries to satisfy the given allocation request by reusing one of the extents * in the given extents_t. @@ -1092,7 +1153,9 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent, growing_retained); return NULL; } - extent_zeroed_set(extent, true); + if (!extent_need_manual_zero(arena)) { + extent_zeroed_set(extent, true); + } } if (extent_committed_get(extent)) { @@ -1113,14 +1176,16 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (*zero) { void *addr = extent_base_get(extent); - size_t size = extent_size_get(extent); if (!extent_zeroed_get(extent)) { - if (pages_purge_forced(addr, size)) { + size_t size = extent_size_get(extent); + if (extent_need_manual_zero(arena) || + pages_purge_forced(addr, size)) { memset(addr, 0, size); } } else if (config_debug) { size_t *p = (size_t *)(uintptr_t)addr; - for (size_t i = 0; i < size / sizeof(size_t); i++) { + /* Check the first page only. */ + for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { assert(p[i] == 0); } } @@ -1244,11 +1309,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); while (alloc_size < alloc_size_min) { egn_skip++; - if (arena->extent_grow_next + egn_skip == NPSIZES) { + if (arena->extent_grow_next + egn_skip >= + sz_psz2ind(SC_LARGE_MAXCLASS)) { /* Outside legal range. */ goto label_err; } - assert(arena->extent_grow_next + egn_skip < NPSIZES); alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); } @@ -1271,7 +1336,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, extent_hook_post_reentrancy(tsdn); } - extent_init(extent, arena, ptr, alloc_size, false, NSIZES, + extent_init(extent, arena, ptr, alloc_size, false, SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed, committed, true); if (ptr == NULL) { @@ -1341,7 +1406,9 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, &arena->extents_retained, extent, true); goto label_err; } - extent_zeroed_set(extent, true); + if (!extent_need_manual_zero(arena)) { + extent_zeroed_set(extent, true); + } } /* @@ -1375,7 +1442,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (*zero && !extent_zeroed_get(extent)) { void *addr = extent_base_get(extent); size_t size = extent_size_get(extent); - if (pages_purge_forced(addr, size)) { + if (extent_need_manual_zero(arena) || + pages_purge_forced(addr, size)) { memset(addr, 0, size); } } @@ -1524,9 +1592,15 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, } static extent_t * -extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, +extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, - extent_t *extent, bool *coalesced, bool growing_retained) { + extent_t *extent, bool *coalesced, bool growing_retained, + bool inactive_only) { + /* + * We avoid checking / locking inactive neighbors for large size + * classes, since they are eagerly coalesced on deallocation which can + * cause lock contention. + */ /* * Continue attempting to coalesce until failure, to protect against * races with other threads that are thwarted by this one. @@ -1537,7 +1611,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce forward. */ extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx, - extent_past_get(extent)); + extent_past_get(extent), inactive_only); if (next != NULL) { /* * extents->mtx only protects against races for @@ -1563,7 +1637,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce backward. */ extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx, - extent_before_get(extent)); + extent_before_get(extent), inactive_only); if (prev != NULL) { bool can_coalesce = extent_can_coalesce(arena, extents, extent, prev); @@ -1589,6 +1663,22 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, return extent; } +static extent_t * +extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + extent_t *extent, bool *coalesced, bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, extent, coalesced, growing_retained, false); +} + +static extent_t * +extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + extent_t *extent, bool *coalesced, bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, extent, coalesced, growing_retained, true); +} + /* * Does the metadata management portions of putting an unused extent into the * given extents_t (coalesces, deregisters slab interiors, the heap operations). @@ -1606,7 +1696,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, malloc_mutex_lock(tsdn, &extents->mtx); extent_hooks_assure_initialized(arena, r_extent_hooks); - extent_szind_set(extent, NSIZES); + extent_szind_set(extent, SC_NSIZES); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -1618,18 +1708,22 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (!extents->delay_coalesce) { extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, NULL, growing_retained); - } else if (extent_size_get(extent) >= LARGE_MINCLASS) { + } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) { + assert(extents == &arena->extents_dirty); /* Always coalesce large extents eagerly. */ bool coalesced; - size_t prev_size; do { - prev_size = extent_size_get(extent); assert(extent_state_get(extent) == extent_state_active); - extent = extent_try_coalesce(tsdn, arena, + extent = extent_try_coalesce_large(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, &coalesced, growing_retained); - } while (coalesced && - extent_size_get(extent) >= prev_size + LARGE_MINCLASS); + } while (coalesced); + if (extent_size_get(extent) >= oversize_threshold) { + /* Shortcut to purge the oversize extent eagerly. */ + malloc_mutex_unlock(tsdn, &extents->mtx); + arena_decay_extent(tsdn, arena, r_extent_hooks, extent); + return; + } } extent_deactivate_locked(tsdn, arena, extents, extent); @@ -1652,6 +1746,12 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { } static bool +extent_may_dalloc(void) { + /* With retain enabled, the default dalloc always fails. */ + return !opt_retain; +} + +static bool extent_dalloc_default_impl(void *addr, size_t size) { if (!have_dss || !extent_in_dss(addr)) { return extent_dalloc_mmap(addr, size); @@ -1706,16 +1806,20 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - /* - * Deregister first to avoid a race with other allocating threads, and - * reregister if deallocation fails. - */ - extent_deregister(tsdn, extent); - if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { - return; + /* Avoid calling the default extent_dalloc unless have to. */ + if (*r_extent_hooks != &extent_hooks_default || extent_may_dalloc()) { + /* + * Deregister first to avoid a race with other allocating + * threads, and reregister if deallocation fails. + */ + extent_deregister(tsdn, extent); + if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, + extent)) { + return; + } + extent_reregister(tsdn, extent); } - extent_reregister(tsdn, extent); if (*r_extent_hooks != &extent_hooks_default) { extent_hook_pre_reentrancy(tsdn, arena); } @@ -2128,22 +2232,23 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, if (a_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL, - NSIZES, false); + SC_NSIZES, false); } if (b_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL, - NSIZES, false); + SC_NSIZES, false); } else { b_elm_b = b_elm_a; } extent_size_set(a, extent_size_get(a) + extent_size_get(b)); - extent_szind_set(a, NSIZES); + extent_szind_set(a, SC_NSIZES); extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ? extent_sn_get(a) : extent_sn_get(b)); extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); - extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES, false); + extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES, + false); extent_unlock2(tsdn, a, b); diff --git a/dep/jemalloc/src/extent_dss.c b/dep/jemalloc/src/extent_dss.c index 2b1ea9cafa0..6c56cf6568c 100644 --- a/dep/jemalloc/src/extent_dss.c +++ b/dep/jemalloc/src/extent_dss.c @@ -154,7 +154,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, (uintptr_t)gap_addr_page; if (gap_size_page != 0) { extent_init(gap, arena, gap_addr_page, - gap_size_page, false, NSIZES, + gap_size_page, false, SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, false, true, true); } @@ -198,7 +198,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_t extent; extent_init(&extent, arena, ret, size, - size, false, NSIZES, + size, false, SC_NSIZES, extent_state_active, false, true, true); if (extent_purge_forced_wrapper(tsdn, diff --git a/dep/jemalloc/src/hook.c b/dep/jemalloc/src/hook.c new file mode 100644 index 00000000000..9ac703cf9f5 --- /dev/null +++ b/dep/jemalloc/src/hook.c @@ -0,0 +1,195 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/hook.h" + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/seq.h" + +typedef struct hooks_internal_s hooks_internal_t; +struct hooks_internal_s { + hooks_t hooks; + bool in_use; +}; + +seq_define(hooks_internal_t, hooks) + +static atomic_u_t nhooks = ATOMIC_INIT(0); +static seq_hooks_t hooks[HOOK_MAX]; +static malloc_mutex_t hooks_mu; + +bool +hook_boot() { + return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK, + malloc_mutex_rank_exclusive); +} + +static void * +hook_install_locked(hooks_t *to_install) { + hooks_internal_t hooks_internal; + for (int i = 0; i < HOOK_MAX; i++) { + bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]); + /* We hold mu; no concurrent access. */ + assert(success); + if (!hooks_internal.in_use) { + hooks_internal.hooks = *to_install; + hooks_internal.in_use = true; + seq_store_hooks(&hooks[i], &hooks_internal); + atomic_store_u(&nhooks, + atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1, + ATOMIC_RELAXED); + return &hooks[i]; + } + } + return NULL; +} + +void * +hook_install(tsdn_t *tsdn, hooks_t *to_install) { + malloc_mutex_lock(tsdn, &hooks_mu); + void *ret = hook_install_locked(to_install); + if (ret != NULL) { + tsd_global_slow_inc(tsdn); + } + malloc_mutex_unlock(tsdn, &hooks_mu); + return ret; +} + +static void +hook_remove_locked(seq_hooks_t *to_remove) { + hooks_internal_t hooks_internal; + bool success = seq_try_load_hooks(&hooks_internal, to_remove); + /* We hold mu; no concurrent access. */ + assert(success); + /* Should only remove hooks that were added. */ + assert(hooks_internal.in_use); + hooks_internal.in_use = false; + seq_store_hooks(to_remove, &hooks_internal); + atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1, + ATOMIC_RELAXED); +} + +void +hook_remove(tsdn_t *tsdn, void *opaque) { + if (config_debug) { + char *hooks_begin = (char *)&hooks[0]; + char *hooks_end = (char *)&hooks[HOOK_MAX]; + char *hook = (char *)opaque; + assert(hooks_begin <= hook && hook < hooks_end + && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0); + } + malloc_mutex_lock(tsdn, &hooks_mu); + hook_remove_locked((seq_hooks_t *)opaque); + tsd_global_slow_dec(tsdn); + malloc_mutex_unlock(tsdn, &hooks_mu); +} + +#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ +for (int for_each_hook_counter = 0; \ + for_each_hook_counter < HOOK_MAX; \ + for_each_hook_counter++) { \ + bool for_each_hook_success = seq_try_load_hooks( \ + (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ + if (!for_each_hook_success) { \ + continue; \ + } \ + if (!(hooks_internal_ptr)->in_use) { \ + continue; \ + } +#define FOR_EACH_HOOK_END \ +} + +static bool * +hook_reentrantp() { + /* + * We prevent user reentrancy within hooks. This is basically just a + * thread-local bool that triggers an early-exit. + * + * We don't fold in_hook into reentrancy. There are two reasons for + * this: + * - Right now, we turn on reentrancy during things like extent hook + * execution. Allocating during extent hooks is not officially + * supported, but we don't want to break it for the time being. These + * sorts of allocations should probably still be hooked, though. + * - If a hook allocates, we may want it to be relatively fast (after + * all, it executes on every allocator operation). Turning on + * reentrancy is a fairly heavyweight mode (disabling tcache, + * redirecting to arena 0, etc.). It's possible we may one day want + * to turn on reentrant mode here, if it proves too difficult to keep + * this working. But that's fairly easy for us to see; OTOH, people + * not using hooks because they're too slow is easy for us to miss. + * + * The tricky part is + * that this code might get invoked even if we don't have access to tsd. + * This function mimics getting a pointer to thread-local data, except + * that it might secretly return a pointer to some global data if we + * know that the caller will take the early-exit path. + * If we return a bool that indicates that we are reentrant, then the + * caller will go down the early exit path, leaving the global + * untouched. + */ + static bool in_hook_global = true; + tsdn_t *tsdn = tsdn_fetch(); + tcache_t *tcache = tsdn_tcachep_get(tsdn); + if (tcache != NULL) { + return &tcache->in_hook; + } + return &in_hook_global; +} + +#define HOOK_PROLOGUE \ + if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ + return; \ + } \ + bool *in_hook = hook_reentrantp(); \ + if (*in_hook) { \ + return; \ + } \ + *in_hook = true; + +#define HOOK_EPILOGUE \ + *in_hook = false; + +void +hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]) { + HOOK_PROLOGUE + + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_alloc h = hook.hooks.alloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, result, result_raw, args_raw); + } + FOR_EACH_HOOK_END + + HOOK_EPILOGUE +} + +void +hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) { + HOOK_PROLOGUE + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_dalloc h = hook.hooks.dalloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, args_raw); + } + FOR_EACH_HOOK_END + HOOK_EPILOGUE +} + +void +hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) { + HOOK_PROLOGUE + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_expand h = hook.hooks.expand_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, old_usize, new_usize, + result_raw, args_raw); + } + FOR_EACH_HOOK_END + HOOK_EPILOGUE +} diff --git a/dep/jemalloc/src/jemalloc.c b/dep/jemalloc/src/jemalloc.c index f93c16fa3cf..c8afa9c4d55 100644 --- a/dep/jemalloc/src/jemalloc.c +++ b/dep/jemalloc/src/jemalloc.c @@ -7,12 +7,13 @@ #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/log.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/spin.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" @@ -85,8 +86,10 @@ malloc_mutex_t arenas_lock; JEMALLOC_ALIGNED(CACHELINE) atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; static atomic_u_t narenas_total; /* Use narenas_total_*(). */ -static arena_t *a0; /* arenas[0]; read-only after initialization. */ -unsigned narenas_auto; /* Read-only after initialization. */ +/* Below three are read-only after initialization. */ +static arena_t *a0; /* arenas[0]. */ +unsigned narenas_auto; +unsigned manual_arena_base; typedef enum { malloc_init_uninitialized = 3, @@ -326,7 +329,7 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ arena = arena_get(tsdn, ind, false); if (arena != NULL) { - assert(ind < narenas_auto); + assert(arena_is_auto(arena)); return arena; } @@ -341,12 +344,12 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { if (ind == 0) { return; } - if (have_background_thread) { - bool err; - malloc_mutex_lock(tsdn, &background_thread_lock); - err = background_thread_create(tsdn_tsd(tsdn), ind); - malloc_mutex_unlock(tsdn, &background_thread_lock); - if (err) { + /* + * Avoid creating a new background thread just for the huge arena, which + * purges eagerly by default. + */ + if (have_background_thread && !arena_is_huge(ind)) { + if (background_thread_create(tsdn_tsd(tsdn), ind)) { malloc_printf("<jemalloc>: error in background thread " "creation for arena %u. Abort.\n", ind); abort(); @@ -376,6 +379,14 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal) { tsd_iarena_set(tsd, arena); } else { tsd_arena_set(tsd, arena); + unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, + ATOMIC_RELAXED); + tsd_binshards_t *bins = tsd_binshardsp_get(tsd); + for (unsigned i = 0; i < SC_NBINS; i++) { + assert(bin_infos[i].n_shards > 0 && + bin_infos[i].n_shards <= BIN_SHARDS_MAX); + bins->binshard[i] = shard % bin_infos[i].n_shards; + } } } @@ -761,6 +772,50 @@ init_opt_stats_print_opts(const char *v, size_t vlen) { assert(opts_len == strlen(opt_stats_print_opts)); } +/* Reads the next size pair in a multi-sized option. */ +static bool +malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, + size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { + const char *cur = *slab_size_segment_cur; + char *end; + uintmax_t um; + + set_errno(0); + + /* First number, then '-' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != '-') { + return true; + } + *slab_start = (size_t)um; + cur = end + 1; + + /* Second number, then ':' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != ':') { + return true; + } + *slab_end = (size_t)um; + cur = end + 1; + + /* Last number */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0) { + return true; + } + *new_size = (size_t)um; + + /* Consume the separator if there is one. */ + if (*end == '|') { + end++; + } + + *vlen_left -= end - *slab_size_segment_cur; + *slab_size_segment_cur = end; + + return false; +} + static bool malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p) { @@ -850,6 +905,11 @@ malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k, (int)vlen, v); /* If abort_conf is set, error out after processing all options. */ + const char *experimental = "experimental_"; + if (strncmp(k, experimental, strlen(experimental)) == 0) { + /* However, tolerate experimental features. */ + return; + } had_conf_error = true; } @@ -869,7 +929,7 @@ malloc_slow_flag_init(void) { } static void -malloc_conf_init(void) { +malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { unsigned i; char buf[PATH_MAX + 1]; const char *opts, *k, *v; @@ -910,7 +970,12 @@ malloc_conf_init(void) { * Try to use the contents of the "/etc/malloc.conf" * symbolic link's name. */ +#ifndef JEMALLOC_READLINKAT linklen = readlink(linkname, buf, sizeof(buf) - 1); +#else + linklen = readlinkat(AT_FDCWD, linkname, buf, + sizeof(buf) - 1); +#endif if (linklen == -1) { /* No configuration specified. */ linklen = 0; @@ -967,6 +1032,14 @@ malloc_conf_init(void) { } \ continue; \ } + /* + * One of the CONF_MIN macros below expands, in one of the use points, + * to "unsigned integer < 0", which is always false, triggering the + * GCC -Wtype-limits warning, which we disable here and re-enable below. + */ + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS + #define CONF_MIN_no(um, min) false #define CONF_MIN_yes(um, min) ((um) < (min)) #define CONF_MAX_no(um, max) false @@ -1094,6 +1167,28 @@ malloc_conf_init(void) { } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, UINT_MAX, yes, no, false) + if (CONF_MATCH("bin_shards")) { + const char *bin_shards_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t size_start; + size_t size_end; + size_t nshards; + bool err = malloc_conf_multi_sizes_next( + &bin_shards_segment_cur, &vlen_left, + &size_start, &size_end, &nshards); + if (err || bin_update_shard_size( + bin_shard_sizes, size_start, + size_end, nshards)) { + malloc_conf_error( + "Invalid settings for " + "bin_shards", k, klen, v, + vlen); + break; + } + } while (vlen_left > 0); + continue; + } CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : @@ -1141,11 +1236,24 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") } CONF_HANDLE_BOOL(opt_tcache, "tcache") - CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, - "lg_extent_max_active_fit", 0, - (sizeof(size_t) << 3), yes, yes, false) CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", -1, (sizeof(size_t) << 3) - 1) + + /* + * The runtime option of oversize_threshold remains + * undocumented. It may be tweaked in the next major + * release (6.0). The default value 8M is rather + * conservative / safe. Tuning it further down may + * improve fragmentation a bit more, but may also cause + * contention on the huge arena. + */ + CONF_HANDLE_SIZE_T(opt_oversize_threshold, + "oversize_threshold", 0, SC_LARGE_MAXCLASS, no, yes, + false) + CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, + "lg_extent_max_active_fit", 0, + (sizeof(size_t) << 3), no, yes, false) + if (strncmp("percpu_arena", k, klen) == 0) { bool match = false; for (int i = percpu_arena_mode_names_base; i < @@ -1174,6 +1282,31 @@ malloc_conf_init(void) { "max_background_threads", 1, opt_max_background_threads, yes, yes, true); + if (CONF_MATCH("slab_sizes")) { + bool err; + const char *slab_size_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t slab_start; + size_t slab_end; + size_t pgs; + err = malloc_conf_multi_sizes_next( + &slab_size_segment_cur, + &vlen_left, &slab_start, &slab_end, + &pgs); + if (!err) { + sc_data_update_slab_size( + sc_data, slab_start, + slab_end, (int)pgs); + } else { + malloc_conf_error( + "Invalid settings for " + "slab_sizes", k, klen, v, + vlen); + } + } while (!err && vlen_left > 0); + continue; + } if (config_prof) { CONF_HANDLE_BOOL(opt_prof, "prof") CONF_HANDLE_CHAR_P(opt_prof_prefix, @@ -1191,6 +1324,7 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") CONF_HANDLE_BOOL(opt_prof_final, "prof_final") CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + CONF_HANDLE_BOOL(opt_prof_log, "prof_log") } if (config_log) { if (CONF_MATCH("log")) { @@ -1237,6 +1371,8 @@ malloc_conf_init(void) { #undef CONF_HANDLE_SIZE_T #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P + /* Re-enable diagnostic "-Wtype-limits" */ + JEMALLOC_DIAGNOSTIC_POP } if (opt_abort_conf && had_conf_error) { malloc_abort_invalid_conf(); @@ -1275,10 +1411,33 @@ static bool malloc_init_hard_a0_locked() { malloc_initializer = INITIALIZER; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + sc_data_t sc_data = {0}; + JEMALLOC_DIAGNOSTIC_POP + + /* + * Ordering here is somewhat tricky; we need sc_boot() first, since that + * determines what the size classes will be, and then + * malloc_conf_init(), since any slab size tweaking will need to be done + * before sz_boot and bin_boot, which assume that the values they read + * out of sc_data_global are final. + */ + sc_boot(&sc_data); + unsigned bin_shard_sizes[SC_NBINS]; + bin_shard_sizes_boot(bin_shard_sizes); + /* + * prof_boot0 only initializes opt_prof_prefix. We need to do it before + * we parse malloc_conf options, in case malloc_conf parsing overwrites + * it. + */ if (config_prof) { prof_boot0(); } - malloc_conf_init(); + malloc_conf_init(&sc_data, bin_shard_sizes); + sz_boot(&sc_data); + bin_boot(&sc_data, bin_shard_sizes); + if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -1303,7 +1462,7 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot1(); } - arena_boot(); + arena_boot(&sc_data); if (tcache_boot(TSDN_NULL)) { return true; } @@ -1311,11 +1470,13 @@ malloc_init_hard_a0_locked() { malloc_mutex_rank_exclusive)) { return true; } + hook_boot(); /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ narenas_auto = 1; + manual_arena_base = narenas_auto + 1; memset(arenas, 0, sizeof(arena_t *) * narenas_auto); /* * Initialize one arena here. The rest are lazily created in @@ -1463,6 +1624,10 @@ malloc_init_narenas(void) { narenas_auto); } narenas_total_set(narenas_auto); + if (arena_init_huge()) { + narenas_total_inc(); + } + manual_arena_base = narenas_total_get(); return false; } @@ -1560,11 +1725,7 @@ malloc_init_hard(void) { * sets isthreaded) needs to be called without holding any lock. */ background_thread_ctl_init(tsd_tsdn(tsd)); - - malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); - bool err = background_thread_create(tsd, 0); - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); - if (err) { + if (background_thread_create(tsd, 0)) { return true; } } @@ -1587,8 +1748,7 @@ typedef struct static_opts_s static_opts_t; struct static_opts_s { /* Whether or not allocation size may overflow. */ bool may_overflow; - /* Whether or not allocations of size 0 should be treated as size 1. */ - bool bump_empty_alloc; + /* * Whether to assert that allocations are not of size 0 (after any * bumping). @@ -1621,12 +1781,15 @@ struct static_opts_s { * initialization) options. */ bool slow; + /* + * Return size. + */ + bool usize; }; JEMALLOC_ALWAYS_INLINE void static_opts_init(static_opts_t *static_opts) { static_opts->may_overflow = false; - static_opts->bump_empty_alloc = false; static_opts->assert_nonempty_alloc = false; static_opts->null_out_result_on_error = false; static_opts->set_errno_on_error = false; @@ -1634,6 +1797,7 @@ static_opts_init(static_opts_t *static_opts) { static_opts->oom_string = ""; static_opts->invalid_alignment_string = ""; static_opts->slow = false; + static_opts->usize = false; } /* @@ -1648,6 +1812,7 @@ static_opts_init(static_opts_t *static_opts) { typedef struct dynamic_opts_s dynamic_opts_t; struct dynamic_opts_s { void **result; + size_t usize; size_t num_items; size_t item_size; size_t alignment; @@ -1659,6 +1824,7 @@ struct dynamic_opts_s { JEMALLOC_ALWAYS_INLINE void dynamic_opts_init(dynamic_opts_t *dynamic_opts) { dynamic_opts->result = NULL; + dynamic_opts->usize = 0; dynamic_opts->num_items = 0; dynamic_opts->item_size = 0; dynamic_opts->alignment = 0; @@ -1722,12 +1888,13 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, szind_t ind_large; size_t bumped_usize = usize; - if (usize <= SMALL_MAXCLASS) { - assert(((dopts->alignment == 0) ? sz_s2u(LARGE_MINCLASS) : - sz_sa2u(LARGE_MINCLASS, dopts->alignment)) - == LARGE_MINCLASS); - ind_large = sz_size2index(LARGE_MINCLASS); - bumped_usize = sz_s2u(LARGE_MINCLASS); + if (usize <= SC_SMALL_MAXCLASS) { + assert(((dopts->alignment == 0) ? + sz_s2u(SC_LARGE_MINCLASS) : + sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) + == SC_LARGE_MINCLASS); + ind_large = sz_size2index(SC_LARGE_MINCLASS); + bumped_usize = sz_s2u(SC_LARGE_MINCLASS); ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, bumped_usize, ind_large); if (unlikely(ret == NULL)) { @@ -1811,12 +1978,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { } /* Validate the user input. */ - if (sopts->bump_empty_alloc) { - if (unlikely(size == 0)) { - size = 1; - } - } - if (sopts->assert_nonempty_alloc) { assert (size != 0); } @@ -1830,16 +1991,20 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { if (dopts->alignment == 0) { ind = sz_size2index(size); - if (unlikely(ind >= NSIZES)) { + if (unlikely(ind >= SC_NSIZES)) { goto label_oom; } - if (config_stats || (config_prof && opt_prof)) { + if (config_stats || (config_prof && opt_prof) || sopts->usize) { usize = sz_index2size(ind); - assert(usize > 0 && usize <= LARGE_MAXCLASS); + dopts->usize = usize; + assert(usize > 0 && usize + <= SC_LARGE_MAXCLASS); } } else { usize = sz_sa2u(size, dopts->alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + dopts->usize = usize; + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } } @@ -1875,7 +2040,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { alloc_ctx_t alloc_ctx; if (likely((uintptr_t)tctx == (uintptr_t)1U)) { - alloc_ctx.slab = (usize <= SMALL_MAXCLASS); + alloc_ctx.slab = (usize + <= SC_SMALL_MAXCLASS); allocation = imalloc_no_sample( sopts, dopts, tsd, usize, usize, ind); } else if ((uintptr_t)tctx > (uintptr_t)1U) { @@ -1980,9 +2146,8 @@ label_invalid_alignment: return EINVAL; } -/* Returns the errno-style error code of the allocation. */ -JEMALLOC_ALWAYS_INLINE int -imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { +JEMALLOC_ALWAYS_INLINE bool +imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) { if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) { if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(sopts->oom_string); @@ -1992,6 +2157,16 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { set_errno(ENOMEM); *dopts->result = NULL; + return false; + } + + return true; +} + +/* Returns the errno-style error code of the allocation. */ +JEMALLOC_ALWAYS_INLINE int +imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { + if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { return ENOMEM; } @@ -2004,19 +2179,18 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { sopts->slow = false; return imalloc_body(sopts, dopts, tsd); } else { + if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { + return ENOMEM; + } + sopts->slow = true; return imalloc_body(sopts, dopts, tsd); } } -/******************************************************************************/ -/* - * Begin malloc(3)-compatible functions. - */ -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) -je_malloc(size_t size) { +JEMALLOC_NOINLINE +void * +malloc_default(size_t size) { void *ret; static_opts_t sopts; dynamic_opts_t dopts; @@ -2026,7 +2200,6 @@ je_malloc(size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = "<jemalloc>: Error in malloc(): out of memory\n"; @@ -2036,12 +2209,107 @@ je_malloc(size_t size) { dopts.item_size = size; imalloc(&sopts, &dopts); + /* + * Note that this branch gets optimized away -- it immediately follows + * the check on tsd_fast that sets sopts.slow. + */ + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args); + } LOG("core.malloc.exit", "result: %p", ret); return ret; } +/******************************************************************************/ +/* + * Begin malloc(3)-compatible functions. + */ + +/* + * malloc() fastpath. + * + * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit + * tcache. If either of these is false, we tail-call to the slowpath, + * malloc_default(). Tail-calling is used to avoid any caller-saved + * registers. + * + * fastpath supports ticker and profiling, both of which will also + * tail-call to the slowpath if they fire. + */ +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) +je_malloc(size_t size) { + LOG("core.malloc.entry", "size: %zu", size); + + if (tsd_get_allocates() && unlikely(!malloc_initialized())) { + return malloc_default(size); + } + + tsd_t *tsd = tsd_get(false); + if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) { + return malloc_default(size); + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + + if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + return malloc_default(size); + } + + szind_t ind = sz_size2index_lookup(size); + size_t usize; + if (config_stats || config_prof) { + usize = sz_index2size(ind); + } + /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */ + assert(ind < SC_NBINS); + assert(size <= SC_SMALL_MAXCLASS); + + if (config_prof) { + int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); + bytes_until_sample -= usize; + tsd_bytes_until_sample_set(tsd, bytes_until_sample); + + if (unlikely(bytes_until_sample < 0)) { + /* + * Avoid a prof_active check on the fastpath. + * If prof_active is false, set bytes_until_sample to + * a large value. If prof_active is set to true, + * bytes_until_sample will be reset. + */ + if (!prof_active) { + tsd_bytes_until_sample_set(tsd, SSIZE_MAX); + } + return malloc_default(size); + } + } + + cache_bin_t *bin = tcache_small_bin_get(tcache, ind); + bool tcache_success; + void* ret = cache_bin_alloc_easy(bin, &tcache_success); + + if (tcache_success) { + if (config_stats) { + *tsd_thread_allocatedp_get(tsd) += usize; + bin->tstats.nrequests++; + } + if (config_prof) { + tcache->prof_accumbytes += usize; + } + + LOG("core.malloc.exit", "result: %p", ret); + + /* Fastpath success */ + return ret; + } + + return malloc_default(size); +} + JEMALLOC_EXPORT int JEMALLOC_NOTHROW JEMALLOC_ATTR(nonnull(1)) je_posix_memalign(void **memptr, size_t alignment, size_t size) { @@ -2055,7 +2323,6 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.min_alignment = sizeof(void *); sopts.oom_string = "<jemalloc>: Error allocating aligned memory: out of memory\n"; @@ -2068,6 +2335,12 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { dopts.alignment = alignment; ret = imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment, + (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_posix_memalign, *memptr, + (uintptr_t)ret, args); + } LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, *memptr); @@ -2090,7 +2363,6 @@ je_aligned_alloc(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.min_alignment = 1; @@ -2105,6 +2377,11 @@ je_aligned_alloc(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_aligned_alloc, ret, + (uintptr_t)ret, args); + } LOG("core.aligned_alloc.exit", "result: %p", ret); @@ -2125,7 +2402,6 @@ je_calloc(size_t num, size_t size) { dynamic_opts_init(&dopts); sopts.may_overflow = true; - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = "<jemalloc>: Error in calloc(): out of memory\n"; @@ -2136,6 +2412,10 @@ je_calloc(size_t num, size_t size) { dopts.zero = true; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args); + } LOG("core.calloc.exit", "result: %p", ret); @@ -2144,20 +2424,22 @@ je_calloc(size_t num, size_t size) { static void * irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - prof_tctx_t *tctx) { + prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SMALL_MAXCLASS) { - p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); + if (usize <= SC_SMALL_MAXCLASS) { + p = iralloc(tsd, old_ptr, old_usize, + SC_LARGE_MINCLASS, 0, false, hook_args); if (p == NULL) { return NULL; } arena_prof_promote(tsd_tsdn(tsd), p, usize); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, + hook_args); } return p; @@ -2165,7 +2447,7 @@ irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, JEMALLOC_ALWAYS_INLINE void * irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - alloc_ctx_t *alloc_ctx) { + alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2174,9 +2456,11 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); + p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx, + hook_args); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, + hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); @@ -2205,7 +2489,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); size_t usize; if (config_prof && opt_prof) { @@ -2286,11 +2570,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ALLOC_SIZE(2) -je_realloc(void *ptr, size_t size) { +je_realloc(void *ptr, size_t arg_size) { void *ret; tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_usize = 0; + size_t size = arg_size; LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); @@ -2305,6 +2590,10 @@ je_realloc(void *ptr, size_t size) { } else { tcache = NULL; } + + uintptr_t args[3] = {(uintptr_t)ptr, size}; + hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); + ifree(tsd, ptr, tcache, true); LOG("core.realloc.exit", "result: %p", NULL); @@ -2319,29 +2608,58 @@ je_realloc(void *ptr, size_t size) { check_entry_exit_locking(tsd_tsdn(tsd)); + + hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr, + (uintptr_t)arg_size, 0, 0}}; + alloc_ctx_t alloc_ctx; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); if (config_prof && opt_prof) { usize = sz_s2u(size); - ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ? - NULL : irealloc_prof(tsd, ptr, old_usize, usize, - &alloc_ctx); + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { + ret = NULL; + } else { + ret = irealloc_prof(tsd, ptr, old_usize, usize, + &alloc_ctx, &hook_args); + } } else { if (config_stats) { usize = sz_s2u(size); } - ret = iralloc(tsd, ptr, old_usize, size, 0, false); + ret = iralloc(tsd, ptr, old_usize, size, 0, false, + &hook_args); } tsdn = tsd_tsdn(tsd); } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - void *ret = je_malloc(size); - LOG("core.realloc.exit", "result: %p", ret); + static_opts_t sopts; + dynamic_opts_t dopts; + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.null_out_result_on_error = true; + sopts.set_errno_on_error = true; + sopts.oom_string = + "<jemalloc>: Error in realloc(): out of memory\n"; + + dopts.result = &ret; + dopts.num_items = 1; + dopts.item_size = size; + + imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)ptr, arg_size}; + hook_invoke_alloc(hook_alloc_realloc, ret, + (uintptr_t)ret, args); + } + return ret; } @@ -2368,10 +2686,9 @@ je_realloc(void *ptr, size_t size) { return ret; } -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_free(void *ptr) { - LOG("core.free.entry", "ptr: %p", ptr); - +JEMALLOC_NOINLINE +void +free_default(void *ptr) { UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { /* @@ -2397,10 +2714,79 @@ je_free(void *ptr) { } else { tcache = NULL; } + uintptr_t args_raw[3] = {(uintptr_t)ptr}; + hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); } +} + +JEMALLOC_ALWAYS_INLINE +bool free_fastpath(void *ptr, size_t size, bool size_hint) { + tsd_t *tsd = tsd_get(false); + if (unlikely(!tsd || !tsd_fast(tsd))) { + return false; + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + + alloc_ctx_t alloc_ctx; + /* + * If !config_cache_oblivious, we can check PAGE alignment to + * detect sampled objects. Otherwise addresses are + * randomized, and we have to look it up in the rtree anyway. + * See also isfree(). + */ + if (!size_hint || config_cache_oblivious) { + rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); + bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree, + rtree_ctx, (uintptr_t)ptr, + &alloc_ctx.szind, &alloc_ctx.slab); + assert(alloc_ctx.szind != SC_NSIZES); + + /* Note: profiled objects will have alloc_ctx.slab set */ + if (!res || !alloc_ctx.slab) { + return false; + } + } else { + /* + * Check for both sizes that are too large, and for sampled objects. + * Sampled objects are always page-aligned. The sampled object check + * will also check for null ptr. + */ + if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) { + return false; + } + alloc_ctx.szind = sz_size2index_lookup(size); + } + + if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + return false; + } + + cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind); + cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind]; + if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) { + return false; + } + + if (config_stats) { + size_t usize = sz_index2size(alloc_ctx.szind); + *tsd_thread_deallocatedp_get(tsd) += usize; + } + + return true; +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_free(void *ptr) { + LOG("core.free.entry", "ptr: %p", ptr); + + if (!free_fastpath(ptr, 0, false)) { + free_default(ptr); + } + LOG("core.free.exit", ""); } @@ -2427,7 +2813,6 @@ je_memalign(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.min_alignment = 1; sopts.oom_string = "<jemalloc>: Error allocating aligned memory: out of memory\n"; @@ -2441,6 +2826,11 @@ je_memalign(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {alignment, size}; + hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret, + args); + } LOG("core.memalign.exit", "result: %p", ret); return ret; @@ -2462,7 +2852,6 @@ je_valloc(size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.min_alignment = PAGE; sopts.oom_string = @@ -2476,6 +2865,10 @@ je_valloc(size_t size) { dopts.alignment = PAGE; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args); + } LOG("core.valloc.exit", "result: %p\n", ret); return ret; @@ -2543,6 +2936,82 @@ int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); * Begin non-standard functions. */ +#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API + +#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y +#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ + JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) + +typedef struct { + void *ptr; + size_t size; +} smallocx_return_t; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +smallocx_return_t JEMALLOC_NOTHROW +/* + * The attribute JEMALLOC_ATTR(malloc) cannot be used due to: + * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488 + */ +JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) + (size_t size, int flags) { + /* + * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be + * used here because it makes writing beyond the `size` + * of the `ptr` undefined behavior, but the objective + * of this function is to allow writing beyond `size` + * up to `smallocx_return_t::size`. + */ + smallocx_return_t ret; + static_opts_t sopts; + dynamic_opts_t dopts; + + LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags); + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.assert_nonempty_alloc = true; + sopts.null_out_result_on_error = true; + sopts.oom_string = "<jemalloc>: Error in mallocx(): out of memory\n"; + sopts.usize = true; + + dopts.result = &ret.ptr; + dopts.num_items = 1; + dopts.item_size = size; + if (unlikely(flags != 0)) { + if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { + dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); + } + + dopts.zero = MALLOCX_ZERO_GET(flags); + + if ((flags & MALLOCX_TCACHE_MASK) != 0) { + if ((flags & MALLOCX_TCACHE_MASK) + == MALLOCX_TCACHE_NONE) { + dopts.tcache_ind = TCACHE_IND_NONE; + } else { + dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); + } + } else { + dopts.tcache_ind = TCACHE_IND_AUTOMATIC; + } + + if ((flags & MALLOCX_ARENA_MASK) != 0) + dopts.arena_ind = MALLOCX_ARENA_GET(flags); + } + + imalloc(&sopts, &dopts); + assert(dopts.usize == je_nallocx(size, flags)); + ret.size = dopts.usize; + + LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size); + return ret; +} +#undef JEMALLOC_SMALLOCX_CONCAT_HELPER +#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 +#endif + JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) @@ -2586,6 +3055,11 @@ je_mallocx(size_t size, int flags) { } imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size, flags}; + hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret, + args); + } LOG("core.mallocx.exit", "result: %p", ret); return ret; @@ -2594,22 +3068,23 @@ je_mallocx(size_t size, int flags) { static void * irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, - prof_tctx_t *tctx) { + prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SMALL_MAXCLASS) { - p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS, - alignment, zero, tcache, arena); + if (usize <= SC_SMALL_MAXCLASS) { + p = iralloct(tsdn, old_ptr, old_usize, + SC_LARGE_MINCLASS, alignment, zero, tcache, + arena, hook_args); if (p == NULL) { return NULL; } arena_prof_promote(tsdn, p, usize); } else { p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, - tcache, arena); + tcache, arena, hook_args); } return p; @@ -2618,7 +3093,7 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, JEMALLOC_ALWAYS_INLINE void * irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache, - arena_t *arena, alloc_ctx_t *alloc_ctx) { + arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2628,10 +3103,10 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, *usize, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, - *usize, alignment, zero, tcache, arena, tctx); + *usize, alignment, zero, tcache, arena, tctx, hook_args); } else { p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, - zero, tcache, arena); + zero, tcache, arena, hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, false); @@ -2702,23 +3177,27 @@ je_rallocx(void *ptr, size_t size, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); + + hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags, + 0}}; if (config_prof && opt_prof) { usize = (alignment == 0) ? sz_s2u(size) : sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, - zero, tcache, arena, &alloc_ctx); + zero, tcache, arena, &alloc_ctx, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } } else { p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, - zero, tcache, arena); + zero, tcache, arena, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } @@ -2752,14 +3231,14 @@ label_oom: JEMALLOC_ALWAYS_INLINE size_t ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { - size_t usize; + size_t newsize; - if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) { + if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero, + &newsize)) { return old_usize; } - usize = isalloc(tsdn, ptr); - return usize; + return newsize; } static size_t @@ -2793,17 +3272,19 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, */ if (alignment == 0) { usize_max = sz_s2u(size+extra); - assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS); + assert(usize_max > 0 + && usize_max <= SC_LARGE_MAXCLASS); } else { usize_max = sz_sa2u(size+extra, alignment); - if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) { + if (unlikely(usize_max == 0 + || usize_max > SC_LARGE_MAXCLASS)) { /* * usize_max is out of range, and chances are that * allocation will fail, but use the maximum possible * value and carry on with prof_alloc_prep(), just in * case allocation succeeds. */ - usize_max = LARGE_MAXCLASS; + usize_max = SC_LARGE_MAXCLASS; } } tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); @@ -2846,24 +3327,24 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); /* * The API explicitly absolves itself of protecting against (size + * extra) numerical overflow, but we may need to clamp extra to avoid - * exceeding LARGE_MAXCLASS. + * exceeding SC_LARGE_MAXCLASS. * * Ordinarily, size limit checking is handled deeper down, but here we * have to check as part of (size + extra) clamping, since we need the * clamped value in the above helper functions. */ - if (unlikely(size > LARGE_MAXCLASS)) { + if (unlikely(size > SC_LARGE_MAXCLASS)) { usize = old_usize; goto label_not_resized; } - if (unlikely(LARGE_MAXCLASS - size < extra)) { - extra = LARGE_MAXCLASS - size; + if (unlikely(SC_LARGE_MAXCLASS - size < extra)) { + extra = SC_LARGE_MAXCLASS - size; } if (config_prof && opt_prof) { @@ -2882,6 +3363,12 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { *tsd_thread_deallocatedp_get(tsd) += old_usize; } label_not_resized: + if (unlikely(!tsd_fast(tsd))) { + uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; + hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, + usize, (uintptr_t)usize, args); + } + UTRACE(ptr, size, ptr); check_entry_exit_locking(tsd_tsdn(tsd)); @@ -2891,7 +3378,7 @@ label_not_resized: JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_ATTR(pure) -je_sallocx(const void *ptr, UNUSED int flags) { +je_sallocx(const void *ptr, int flags) { size_t usize; tsdn_t *tsdn; @@ -2954,6 +3441,8 @@ je_dallocx(void *ptr, int flags) { tsd_assert_fast(tsd); ifree(tsd, ptr, tcache, false); } else { + uintptr_t args_raw[3] = {(uintptr_t)ptr, flags}; + hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); @@ -2975,14 +3464,11 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { return usize; } -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_sdallocx(void *ptr, size_t size, int flags) { +JEMALLOC_NOINLINE void +sdallocx_default(void *ptr, size_t size, int flags) { assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); - LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, - size, flags); - tsd_t *tsd = tsd_fetch(); bool fast = tsd_fast(tsd); size_t usize = inallocx(tsd_tsdn(tsd), size, flags); @@ -3016,10 +3502,23 @@ je_sdallocx(void *ptr, size_t size, int flags) { tsd_assert_fast(tsd); isfree(tsd, ptr, usize, tcache, false); } else { + uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags}; + hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw); isfree(tsd, ptr, usize, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_sdallocx(void *ptr, size_t size, int flags) { + LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, + size, flags); + + if (flags !=0 || !free_fastpath(ptr, size, true)) { + sdallocx_default(ptr, size, flags); + } + LOG("core.sdallocx.exit", ""); } @@ -3040,7 +3539,7 @@ je_nallocx(size_t size, int flags) { check_entry_exit_locking(tsdn); usize = inallocx(tsdn, size, flags); - if (unlikely(usize > LARGE_MAXCLASS)) { + if (unlikely(usize > SC_LARGE_MAXCLASS)) { LOG("core.nallocx.exit", "result: %zu", ZU(0)); return 0; } @@ -3256,6 +3755,7 @@ _malloc_prefork(void) } } prof_prefork1(tsd_tsdn(tsd)); + tsd_prefork(tsd); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -3278,6 +3778,8 @@ _malloc_postfork(void) tsd = tsd_fetch(); + tsd_postfork_parent(tsd); + witness_postfork_parent(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { @@ -3305,6 +3807,8 @@ jemalloc_postfork_child(void) { tsd = tsd_fetch(); + tsd_postfork_child(tsd); + witness_postfork_child(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { diff --git a/dep/jemalloc/src/large.c b/dep/jemalloc/src/large.c index 27a2c679876..8e7a781d330 100644 --- a/dep/jemalloc/src/large.c +++ b/dep/jemalloc/src/large.c @@ -28,7 +28,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, assert(!tsdn_null(tsdn) || arena != NULL); ausize = sz_sa2u(usize, alignment); - if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS)) { + if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { return NULL; } @@ -42,7 +42,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, */ is_zeroed = zero; if (likely(!tsdn_null(tsdn))) { - arena = arena_choose(tsdn_tsd(tsdn), arena); + arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); } if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn, arena, usize, alignment, &is_zeroed)) == NULL) { @@ -109,7 +109,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, &extent_hooks, extent, usize + sz_large_pad, - sz_size2index(usize), false, diff, NSIZES, false); + sz_size2index(usize), false, diff, SC_NSIZES, false); if (trail == NULL) { return true; } @@ -154,17 +154,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool new_mapping; if ((trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, extent_past_get(extent), trailsize, 0, - CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL + CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL || (trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, extent_past_get(extent), trailsize, 0, - CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL) { + CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { if (config_stats) { new_mapping = false; } } else { if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, 0, CACHELINE, false, - NSIZES, &is_zeroed_trail, &commit)) == NULL) { + SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { return true; } if (config_stats) { @@ -221,9 +221,10 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS); + assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS + && usize_max >= SC_LARGE_MINCLASS); if (usize_max > oldusize) { /* Attempt to expand the allocation in-place. */ @@ -270,17 +271,23 @@ large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, } void * -large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, - size_t alignment, bool zero, tcache_t *tcache) { - size_t oldusize = extent_usize_get(extent); +large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, + size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args) { + extent_t *extent = iealloc(tsdn, ptr); + size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize > 0 && usize <= LARGE_MAXCLASS); + assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS + && usize >= SC_LARGE_MINCLASS); /* Try to avoid moving the allocation. */ if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) { + hook_invoke_expand(hook_args->is_realloc + ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, + usize, (uintptr_t)ptr, hook_args->args); return extent_addr_get(extent); } @@ -295,6 +302,12 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, return NULL; } + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + size_t copysize = (usize < oldusize) ? usize : oldusize; memcpy(ret, extent_addr_get(extent), copysize); isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true); @@ -318,8 +331,9 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, large_dalloc_maybe_junk(extent_addr_get(extent), extent_usize_get(extent)); } else { - malloc_mutex_assert_owner(tsdn, &arena->large_mtx); + /* Only hold the large_mtx if necessary. */ if (!arena_is_auto(arena)) { + malloc_mutex_assert_owner(tsdn, &arena->large_mtx); extent_list_remove(&arena->large, extent); } } @@ -369,3 +383,13 @@ void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) { large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U); } + +nstime_t +large_prof_alloc_time_get(const extent_t *extent) { + return extent_prof_alloc_time_get(extent); +} + +void +large_prof_alloc_time_set(extent_t *extent, nstime_t t) { + extent_prof_alloc_time_set(extent, t); +} diff --git a/dep/jemalloc/src/mutex.c b/dep/jemalloc/src/mutex.c index 30222b3e582..3f920f5b1cd 100644 --- a/dep/jemalloc/src/mutex.c +++ b/dep/jemalloc/src/mutex.c @@ -46,7 +46,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; - UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER; + nstime_t before = NSTIME_ZERO_INITIALIZER; if (ncpus == 1) { goto label_spin_done; @@ -55,7 +55,8 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN; do { spin_cpu_spinwait(); - if (!malloc_mutex_trylock_final(mutex)) { + if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED) + && !malloc_mutex_trylock_final(mutex)) { data->n_spin_acquired++; return; } @@ -144,9 +145,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, } # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - mutex->lock = OS_UNFAIR_LOCK_INIT; -#elif (defined(JEMALLOC_OSSPIN)) - mutex->lock = 0; + mutex->lock = OS_UNFAIR_LOCK_INIT; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) if (postpone_init) { mutex->postponed_next = postponed_mutexes; diff --git a/dep/jemalloc/src/pages.c b/dep/jemalloc/src/pages.c index 26002692d60..13de27a0087 100644 --- a/dep/jemalloc/src/pages.c +++ b/dep/jemalloc/src/pages.c @@ -180,6 +180,35 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { assert(alignment >= PAGE); assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr); +#if defined(__FreeBSD__) && defined(MAP_EXCL) + /* + * FreeBSD has mechanisms both to mmap at specific address without + * touching existing mappings, and to mmap with specific alignment. + */ + { + if (os_overcommits) { + *commit = true; + } + + int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + int flags = mmap_flags; + + if (addr != NULL) { + flags |= MAP_FIXED | MAP_EXCL; + } else { + unsigned alignment_bits = ffs_zu(alignment); + assert(alignment_bits > 1); + flags |= MAP_ALIGNED(alignment_bits - 1); + } + + void *ret = mmap(addr, size, prot, flags, -1, 0); + if (ret == MAP_FAILED) { + ret = NULL; + } + + return ret; + } +#endif /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work @@ -261,7 +290,7 @@ pages_decommit(void *addr, size_t size) { bool pages_purge_lazy(void *addr, size_t size) { - assert(PAGE_ADDR2BASE(addr) == addr); + assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); assert(PAGE_CEILING(size) == size); if (!pages_can_purge_lazy) { @@ -391,6 +420,10 @@ os_page_detect(void) { GetSystemInfo(&si); return si.dwPageSize; #elif defined(__FreeBSD__) + /* + * This returns the value obtained from + * the auxv vector, avoiding a syscall. + */ return getpagesize(); #else long result = sysconf(_SC_PAGESIZE); @@ -544,6 +577,10 @@ init_thp_state(void) { close(fd); #endif + if (nread < 0) { + goto label_error; + } + if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) { init_system_thp_mode = thp_mode_default; } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) { @@ -588,6 +625,11 @@ pages_boot(void) { init_thp_state(); +#ifdef __FreeBSD__ + /* + * FreeBSD doesn't need the check; madvise(2) is known to work. + */ +#else /* Detect lazy purge runtime support. */ if (pages_can_purge_lazy) { bool committed = false; @@ -601,6 +643,7 @@ pages_boot(void) { } os_pages_unmap(madv_free_page, PAGE); } +#endif return false; } diff --git a/dep/jemalloc/src/prof.c b/dep/jemalloc/src/prof.c index 13df641a030..4d7d65db40c 100644 --- a/dep/jemalloc/src/prof.c +++ b/dep/jemalloc/src/prof.c @@ -7,6 +7,7 @@ #include "jemalloc/internal/hash.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/emitter.h" /******************************************************************************/ @@ -23,7 +24,7 @@ */ #undef _Unwind_Backtrace #include <unwind.h> -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) #endif /******************************************************************************/ @@ -38,6 +39,7 @@ bool opt_prof_gdump = false; bool opt_prof_final = false; bool opt_prof_leak = false; bool opt_prof_accum = false; +bool opt_prof_log = false; char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -70,6 +72,100 @@ uint64_t prof_interval = 0; size_t lg_prof_sample; +typedef enum prof_logging_state_e prof_logging_state_t; +enum prof_logging_state_e { + prof_logging_state_stopped, + prof_logging_state_started, + prof_logging_state_dumping +}; + +/* + * - stopped: log_start never called, or previous log_stop has completed. + * - started: log_start called, log_stop not called yet. Allocations are logged. + * - dumping: log_stop called but not finished; samples are not logged anymore. + */ +prof_logging_state_t prof_logging_state = prof_logging_state_stopped; + +#ifdef JEMALLOC_JET +static bool prof_log_dummy = false; +#endif + +/* Incremented for every log file that is output. */ +static uint64_t log_seq = 0; +static char log_filename[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; + +/* Timestamp for most recent call to log_start(). */ +static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER; + +/* Increment these when adding to the log_bt and log_thr linked lists. */ +static size_t log_bt_index = 0; +static size_t log_thr_index = 0; + +/* Linked list node definitions. These are only used in prof.c. */ +typedef struct prof_bt_node_s prof_bt_node_t; + +struct prof_bt_node_s { + prof_bt_node_t *next; + size_t index; + prof_bt_t bt; + /* Variable size backtrace vector pointed to by bt. */ + void *vec[1]; +}; + +typedef struct prof_thr_node_s prof_thr_node_t; + +struct prof_thr_node_s { + prof_thr_node_t *next; + size_t index; + uint64_t thr_uid; + /* Variable size based on thr_name_sz. */ + char name[1]; +}; + +typedef struct prof_alloc_node_s prof_alloc_node_t; + +/* This is output when logging sampled allocations. */ +struct prof_alloc_node_s { + prof_alloc_node_t *next; + /* Indices into an array of thread data. */ + size_t alloc_thr_ind; + size_t free_thr_ind; + + /* Indices into an array of backtraces. */ + size_t alloc_bt_ind; + size_t free_bt_ind; + + uint64_t alloc_time_ns; + uint64_t free_time_ns; + + size_t usize; +}; + +/* + * Created on the first call to prof_log_start and deleted on prof_log_stop. + * These are the backtraces and threads that have already been logged by an + * allocation. + */ +static bool log_tables_initialized = false; +static ckh_t log_bt_node_set; +static ckh_t log_thr_node_set; + +/* Store linked lists for logged data. */ +static prof_bt_node_t *log_bt_first = NULL; +static prof_bt_node_t *log_bt_last = NULL; +static prof_thr_node_t *log_thr_first = NULL; +static prof_thr_node_t *log_thr_last = NULL; +static prof_alloc_node_t *log_alloc_first = NULL; +static prof_alloc_node_t *log_alloc_last = NULL; + +/* Protects the prof_logging_state and any log_{...} variable. */ +static malloc_mutex_t log_mtx; + /* * Table of mutexes that are shared among gctx's. These are leaf locks, so * there is no problem with using them for more than one gctx at the same time. @@ -145,6 +241,12 @@ static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached); static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); +/* Hashtable functions for log_bt_node_set and log_thr_node_set. */ +static void prof_thr_node_hash(const void *key, size_t r_hash[2]); +static bool prof_thr_node_keycomp(const void *k1, const void *k2); +static void prof_bt_node_hash(const void *key, size_t r_hash[2]); +static bool prof_bt_node_keycomp(const void *k1, const void *k2); + /******************************************************************************/ /* Red-black trees. */ @@ -242,6 +344,12 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) { prof_tctx_set(tsdn, ptr, usize, NULL, tctx); + /* Get the current time and set this in the extent_t. We'll read this + * when free() is called. */ + nstime_t t = NSTIME_ZERO_INITIALIZER; + nstime_update(&t); + prof_alloc_time_set(tsdn, ptr, NULL, t); + malloc_mutex_lock(tsdn, tctx->tdata->lock); tctx->cnts.curobjs++; tctx->cnts.curbytes += usize; @@ -253,14 +361,174 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, malloc_mutex_unlock(tsdn, tctx->tdata->lock); } +static size_t +prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) { + assert(prof_logging_state == prof_logging_state_started); + malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); + + prof_bt_node_t dummy_node; + dummy_node.bt = *bt; + prof_bt_node_t *node; + + /* See if this backtrace is already cached in the table. */ + if (ckh_search(&log_bt_node_set, (void *)(&dummy_node), + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_bt_node_t, vec) + + (bt->len * sizeof(void *)); + prof_bt_node_t *new_node = (prof_bt_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, + true, arena_get(TSDN_NULL, 0, true), true); + if (log_bt_first == NULL) { + log_bt_first = new_node; + log_bt_last = new_node; + } else { + log_bt_last->next = new_node; + log_bt_last = new_node; + } + + new_node->next = NULL; + new_node->index = log_bt_index; + /* + * Copy the backtrace: bt is inside a tdata or gctx, which + * might die before prof_log_stop is called. + */ + new_node->bt.len = bt->len; + memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *)); + new_node->bt.vec = new_node->vec; + + log_bt_index++; + ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL); + return new_node->index; + } else { + return node->index; + } +} +static size_t +prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) { + assert(prof_logging_state == prof_logging_state_started); + malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); + + prof_thr_node_t dummy_node; + dummy_node.thr_uid = thr_uid; + prof_thr_node_t *node; + + /* See if this thread is already cached in the table. */ + if (ckh_search(&log_thr_node_set, (void *)(&dummy_node), + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1; + prof_thr_node_t *new_node = (prof_thr_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, + true, arena_get(TSDN_NULL, 0, true), true); + if (log_thr_first == NULL) { + log_thr_first = new_node; + log_thr_last = new_node; + } else { + log_thr_last->next = new_node; + log_thr_last = new_node; + } + + new_node->next = NULL; + new_node->index = log_thr_index; + new_node->thr_uid = thr_uid; + strcpy(new_node->name, name); + + log_thr_index++; + ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL); + return new_node->index; + } else { + return node->index; + } +} + +static void +prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) { + malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); + + prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false); + if (cons_tdata == NULL) { + /* + * We decide not to log these allocations. cons_tdata will be + * NULL only when the current thread is in a weird state (e.g. + * it's being destroyed). + */ + return; + } + + malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx); + + if (prof_logging_state != prof_logging_state_started) { + goto label_done; + } + + if (!log_tables_initialized) { + bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, + prof_bt_node_hash, prof_bt_node_keycomp); + bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, + prof_thr_node_hash, prof_thr_node_keycomp); + if (err1 || err2) { + goto label_done; + } + log_tables_initialized = true; + } + + nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr, + (alloc_ctx_t *)NULL); + nstime_t free_time = NSTIME_ZERO_INITIALIZER; + nstime_update(&free_time); + + size_t sz = sizeof(prof_alloc_node_t); + prof_alloc_node_t *new_node = (prof_alloc_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, + arena_get(TSDN_NULL, 0, true), true); + + const char *prod_thr_name = (tctx->tdata->thread_name == NULL)? + "" : tctx->tdata->thread_name; + const char *cons_thr_name = prof_thread_name_get(tsd); + + prof_bt_t bt; + /* Initialize the backtrace, using the buffer in tdata to store it. */ + bt_init(&bt, cons_tdata->vec); + prof_backtrace(&bt); + prof_bt_t *cons_bt = &bt; + + /* We haven't destroyed tctx yet, so gctx should be good to read. */ + prof_bt_t *prod_bt = &tctx->gctx->bt; + + new_node->next = NULL; + new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid, + prod_thr_name); + new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid, + cons_thr_name); + new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt); + new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt); + new_node->alloc_time_ns = nstime_ns(&alloc_time); + new_node->free_time_ns = nstime_ns(&free_time); + new_node->usize = usize; + + if (log_alloc_first == NULL) { + log_alloc_first = new_node; + log_alloc_last = new_node; + } else { + log_alloc_last->next = new_node; + log_alloc_last = new_node; + } + +label_done: + malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx); +} + void -prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { +prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx) { malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); + assert(tctx->cnts.curobjs > 0); assert(tctx->cnts.curbytes >= usize); tctx->cnts.curobjs--; tctx->cnts.curbytes -= usize; + prof_try_log(tsd, ptr, usize, tctx); + if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) { prof_tctx_destroy(tsd, tctx); } else { @@ -871,15 +1139,12 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) { void prof_sample_threshold_update(prof_tdata_t *tdata) { #ifdef JEMALLOC_PROF - uint64_t r; - double u; - if (!config_prof) { return; } if (lg_prof_sample == 0) { - tdata->bytes_until_sample = 0; + tsd_bytes_until_sample_set(tsd_fetch(), 0); return; } @@ -901,11 +1166,16 @@ prof_sample_threshold_update(prof_tdata_t *tdata) { * pp 500 * (http://luc.devroye.org/rnbookindex.html) */ - r = prng_lg_range_u64(&tdata->prng_state, 53); - u = (double)r * (1.0/9007199254740992.0L); - tdata->bytes_until_sample = (uint64_t)(log(u) / + uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53); + double u = (double)r * (1.0/9007199254740992.0L); + uint64_t bytes_until_sample = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + (uint64_t)1U; + if (bytes_until_sample > SSIZE_MAX) { + bytes_until_sample = SSIZE_MAX; + } + tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample); + #endif } @@ -1887,6 +2157,33 @@ prof_bt_keycomp(const void *k1, const void *k2) { return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } +static void +prof_bt_node_hash(const void *key, size_t r_hash[2]) { + const prof_bt_node_t *bt_node = (prof_bt_node_t *)key; + prof_bt_hash((void *)(&bt_node->bt), r_hash); +} + +static bool +prof_bt_node_keycomp(const void *k1, const void *k2) { + const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1; + const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2; + return prof_bt_keycomp((void *)(&bt_node1->bt), + (void *)(&bt_node2->bt)); +} + +static void +prof_thr_node_hash(const void *key, size_t r_hash[2]) { + const prof_thr_node_t *thr_node = (prof_thr_node_t *)key; + hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash); +} + +static bool +prof_thr_node_keycomp(const void *k1, const void *k2) { + const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1; + const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2; + return thr_node1->thr_uid == thr_node2->thr_uid; +} + static uint64_t prof_thr_uid_alloc(tsdn_t *tsdn) { uint64_t thr_uid; @@ -2119,6 +2416,368 @@ prof_active_set(tsdn_t *tsdn, bool active) { return prof_active_old; } +#ifdef JEMALLOC_JET +size_t +prof_log_bt_count(void) { + size_t cnt = 0; + prof_bt_node_t *node = log_bt_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +size_t +prof_log_alloc_count(void) { + size_t cnt = 0; + prof_alloc_node_t *node = log_alloc_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +size_t +prof_log_thr_count(void) { + size_t cnt = 0; + prof_thr_node_t *node = log_thr_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +bool +prof_log_is_logging(void) { + return prof_logging_state == prof_logging_state_started; +} + +bool +prof_log_rep_check(void) { + if (prof_logging_state == prof_logging_state_stopped + && log_tables_initialized) { + return true; + } + + if (log_bt_last != NULL && log_bt_last->next != NULL) { + return true; + } + if (log_thr_last != NULL && log_thr_last->next != NULL) { + return true; + } + if (log_alloc_last != NULL && log_alloc_last->next != NULL) { + return true; + } + + size_t bt_count = prof_log_bt_count(); + size_t thr_count = prof_log_thr_count(); + size_t alloc_count = prof_log_alloc_count(); + + + if (prof_logging_state == prof_logging_state_stopped) { + if (bt_count != 0 || thr_count != 0 || alloc_count || 0) { + return true; + } + } + + prof_alloc_node_t *node = log_alloc_first; + while (node != NULL) { + if (node->alloc_bt_ind >= bt_count) { + return true; + } + if (node->free_bt_ind >= bt_count) { + return true; + } + if (node->alloc_thr_ind >= thr_count) { + return true; + } + if (node->free_thr_ind >= thr_count) { + return true; + } + if (node->alloc_time_ns > node->free_time_ns) { + return true; + } + node = node->next; + } + + return false; +} + +void +prof_log_dummy_set(bool new_value) { + prof_log_dummy = new_value; +} +#endif + +bool +prof_log_start(tsdn_t *tsdn, const char *filename) { + if (!opt_prof || !prof_booted) { + return true; + } + + bool ret = false; + size_t buf_size = PATH_MAX + 1; + + malloc_mutex_lock(tsdn, &log_mtx); + + if (prof_logging_state != prof_logging_state_stopped) { + ret = true; + } else if (filename == NULL) { + /* Make default name. */ + malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json", + opt_prof_prefix, prof_getpid(), log_seq); + log_seq++; + prof_logging_state = prof_logging_state_started; + } else if (strlen(filename) >= buf_size) { + ret = true; + } else { + strcpy(log_filename, filename); + prof_logging_state = prof_logging_state_started; + } + + if (!ret) { + nstime_update(&log_start_timestamp); + } + + malloc_mutex_unlock(tsdn, &log_mtx); + + return ret; +} + +/* Used as an atexit function to stop logging on exit. */ +static void +prof_log_stop_final(void) { + tsd_t *tsd = tsd_fetch(); + prof_log_stop(tsd_tsdn(tsd)); +} + +struct prof_emitter_cb_arg_s { + int fd; + ssize_t ret; +}; + +static void +prof_emitter_write_cb(void *opaque, const char *to_write) { + struct prof_emitter_cb_arg_s *arg = + (struct prof_emitter_cb_arg_s *)opaque; + size_t bytes = strlen(to_write); +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + return; + } +#endif + arg->ret = write(arg->fd, (void *)to_write, bytes); +} + +/* + * prof_log_emit_{...} goes through the appropriate linked list, emitting each + * node to the json and deallocating it. + */ +static void +prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "threads"); + prof_thr_node_t *thr_node = log_thr_first; + prof_thr_node_t *thr_old_node; + while (thr_node != NULL) { + emitter_json_object_begin(emitter); + + emitter_json_kv(emitter, "thr_uid", emitter_type_uint64, + &thr_node->thr_uid); + + char *thr_name = thr_node->name; + + emitter_json_kv(emitter, "thr_name", emitter_type_string, + &thr_name); + + emitter_json_object_end(emitter); + thr_old_node = thr_node; + thr_node = thr_node->next; + idalloc(tsd, thr_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "stack_traces"); + prof_bt_node_t *bt_node = log_bt_first; + prof_bt_node_t *bt_old_node; + /* + * Calculate how many hex digits we need: twice number of bytes, two for + * "0x", and then one more for terminating '\0'. + */ + char buf[2 * sizeof(intptr_t) + 3]; + size_t buf_sz = sizeof(buf); + while (bt_node != NULL) { + emitter_json_array_begin(emitter); + size_t i; + for (i = 0; i < bt_node->bt.len; i++) { + malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]); + char *trace_str = buf; + emitter_json_value(emitter, emitter_type_string, + &trace_str); + } + emitter_json_array_end(emitter); + + bt_old_node = bt_node; + bt_node = bt_node->next; + idalloc(tsd, bt_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "allocations"); + prof_alloc_node_t *alloc_node = log_alloc_first; + prof_alloc_node_t *alloc_old_node; + while (alloc_node != NULL) { + emitter_json_object_begin(emitter); + + emitter_json_kv(emitter, "alloc_thread", emitter_type_size, + &alloc_node->alloc_thr_ind); + + emitter_json_kv(emitter, "free_thread", emitter_type_size, + &alloc_node->free_thr_ind); + + emitter_json_kv(emitter, "alloc_trace", emitter_type_size, + &alloc_node->alloc_bt_ind); + + emitter_json_kv(emitter, "free_trace", emitter_type_size, + &alloc_node->free_bt_ind); + + emitter_json_kv(emitter, "alloc_timestamp", + emitter_type_uint64, &alloc_node->alloc_time_ns); + + emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64, + &alloc_node->free_time_ns); + + emitter_json_kv(emitter, "usize", emitter_type_uint64, + &alloc_node->usize); + + emitter_json_object_end(emitter); + + alloc_old_node = alloc_node; + alloc_node = alloc_node->next; + idalloc(tsd, alloc_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_metadata(emitter_t *emitter) { + emitter_json_object_kv_begin(emitter, "info"); + + nstime_t now = NSTIME_ZERO_INITIALIZER; + + nstime_update(&now); + uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp); + emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns); + + char *vers = JEMALLOC_VERSION; + emitter_json_kv(emitter, "version", + emitter_type_string, &vers); + + emitter_json_kv(emitter, "lg_sample_rate", + emitter_type_int, &lg_prof_sample); + + int pid = prof_getpid(); + emitter_json_kv(emitter, "pid", emitter_type_int, &pid); + + emitter_json_object_end(emitter); +} + + +bool +prof_log_stop(tsdn_t *tsdn) { + if (!opt_prof || !prof_booted) { + return true; + } + + tsd_t *tsd = tsdn_tsd(tsdn); + malloc_mutex_lock(tsdn, &log_mtx); + + if (prof_logging_state != prof_logging_state_started) { + malloc_mutex_unlock(tsdn, &log_mtx); + return true; + } + + /* + * Set the state to dumping. We'll set it to stopped when we're done. + * Since other threads won't be able to start/stop/log when the state is + * dumping, we don't have to hold the lock during the whole method. + */ + prof_logging_state = prof_logging_state_dumping; + malloc_mutex_unlock(tsdn, &log_mtx); + + + emitter_t emitter; + + /* Create a file. */ + + int fd; +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + fd = 0; + } else { + fd = creat(log_filename, 0644); + } +#else + fd = creat(log_filename, 0644); +#endif + + if (fd == -1) { + malloc_printf("<jemalloc>: creat() for log file \"%s\" " + " failed with %d\n", log_filename, errno); + if (opt_abort) { + abort(); + } + return true; + } + + /* Emit to json. */ + struct prof_emitter_cb_arg_s arg; + arg.fd = fd; + emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb, + (void *)(&arg)); + + emitter_json_object_begin(&emitter); + prof_log_emit_metadata(&emitter); + prof_log_emit_threads(tsd, &emitter); + prof_log_emit_traces(tsd, &emitter); + prof_log_emit_allocs(tsd, &emitter); + emitter_json_object_end(&emitter); + + /* Reset global state. */ + if (log_tables_initialized) { + ckh_delete(tsd, &log_bt_node_set); + ckh_delete(tsd, &log_thr_node_set); + } + log_tables_initialized = false; + log_bt_index = 0; + log_thr_index = 0; + log_bt_first = NULL; + log_bt_last = NULL; + log_thr_first = NULL; + log_thr_last = NULL; + log_alloc_first = NULL; + log_alloc_last = NULL; + + malloc_mutex_lock(tsdn, &log_mtx); + prof_logging_state = prof_logging_state_stopped; + malloc_mutex_unlock(tsdn, &log_mtx); + +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + return false; + } +#endif + return close(fd); +} + const char * prof_thread_name_get(tsd_t *tsd) { prof_tdata_t *tdata; @@ -2355,6 +3014,35 @@ prof_boot2(tsd_t *tsd) { } } + if (opt_prof_log) { + prof_log_start(tsd_tsdn(tsd), NULL); + } + + if (atexit(prof_log_stop_final) != 0) { + malloc_write("<jemalloc>: Error in atexit() " + "for logging\n"); + if (opt_abort) { + abort(); + } + } + + if (malloc_mutex_init(&log_mtx, "prof_log", + WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) { + return true; + } + + if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, + prof_bt_node_hash, prof_bt_node_keycomp)) { + return true; + } + + if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, + prof_thr_node_hash, prof_thr_node_keycomp)) { + return true; + } + + log_tables_initialized = true; + gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE); @@ -2382,16 +3070,14 @@ prof_boot2(tsd_t *tsd) { return true; } } - } - #ifdef JEMALLOC_PROF_LIBGCC - /* - * Cause the backtracing machinery to allocate its internal state - * before enabling profiling. - */ - _Unwind_Backtrace(prof_unwind_init_callback, NULL); + /* + * Cause the backtracing machinery to allocate its internal + * state before enabling profiling. + */ + _Unwind_Backtrace(prof_unwind_init_callback, NULL); #endif - + } prof_booted = true; return false; diff --git a/dep/jemalloc/src/rtree.c b/dep/jemalloc/src/rtree.c index 53702cf7236..4ae41fe2fec 100644 --- a/dep/jemalloc/src/rtree.c +++ b/dep/jemalloc/src/rtree.c @@ -39,7 +39,7 @@ rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) { /* Nodes are never deleted during normal operation. */ not_reached(); } -UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = +rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = rtree_node_dalloc_impl; static rtree_leaf_elm_t * @@ -54,7 +54,7 @@ rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) { /* Leaves are never deleted during normal operation. */ not_reached(); } -UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = +rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = rtree_leaf_dalloc_impl; #ifdef JEMALLOC_JET diff --git a/dep/jemalloc/src/sc.c b/dep/jemalloc/src/sc.c new file mode 100644 index 00000000000..89ddb6ba6a9 --- /dev/null +++ b/dep/jemalloc/src/sc.c @@ -0,0 +1,313 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bit_util.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/pages.h" +#include "jemalloc/internal/sc.h" + +/* + * This module computes the size classes used to satisfy allocations. The logic + * here was ported more or less line-by-line from a shell script, and because of + * that is not the most idiomatic C. Eventually we should fix this, but for now + * at least the damage is compartmentalized to this file. + */ + +sc_data_t sc_data_global; + +static size_t +reg_size_compute(int lg_base, int lg_delta, int ndelta) { + return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); +} + +/* Returns the number of pages in the slab. */ +static int +slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) { + size_t page = (ZU(1) << lg_page); + size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta); + + size_t try_slab_size = page; + size_t try_nregs = try_slab_size / reg_size; + size_t perfect_slab_size = 0; + bool perfect = false; + /* + * This loop continues until we find the least common multiple of the + * page size and size class size. Size classes are all of the form + * base + ndelta * delta == (ndelta + base/ndelta) * delta, which is + * (ndelta + ngroup) * delta. The way we choose slabbing strategies + * means that delta is at most the page size and ndelta < ngroup. So + * the loop executes for at most 2 * ngroup - 1 iterations, which is + * also the bound on the number of pages in a slab chosen by default. + * With the current default settings, this is at most 7. + */ + while (!perfect) { + perfect_slab_size = try_slab_size; + size_t perfect_nregs = try_nregs; + try_slab_size += page; + try_nregs = try_slab_size / reg_size; + if (perfect_slab_size == perfect_nregs * reg_size) { + perfect = true; + } + } + return (int)(perfect_slab_size / page); +} + +static void +size_class( + /* Output. */ + sc_t *sc, + /* Configuration decisions. */ + int lg_max_lookup, int lg_page, int lg_ngroup, + /* Inputs specific to the size class. */ + int index, int lg_base, int lg_delta, int ndelta) { + sc->index = index; + sc->lg_base = lg_base; + sc->lg_delta = lg_delta; + sc->ndelta = ndelta; + sc->psz = (reg_size_compute(lg_base, lg_delta, ndelta) + % (ZU(1) << lg_page) == 0); + size_t size = (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); + if (index == 0) { + assert(!sc->psz); + } + if (size < (ZU(1) << (lg_page + lg_ngroup))) { + sc->bin = true; + sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta); + } else { + sc->bin = false; + sc->pgs = 0; + } + if (size <= (ZU(1) << lg_max_lookup)) { + sc->lg_delta_lookup = lg_delta; + } else { + sc->lg_delta_lookup = 0; + } +} + +static void +size_classes( + /* Output. */ + sc_data_t *sc_data, + /* Determined by the system. */ + size_t lg_ptr_size, int lg_quantum, + /* Configuration decisions. */ + int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) { + int ptr_bits = (1 << lg_ptr_size) * 8; + int ngroup = (1 << lg_ngroup); + int ntiny = 0; + int nlbins = 0; + int lg_tiny_maxclass = (unsigned)-1; + int nbins = 0; + int npsizes = 0; + + int index = 0; + + int ndelta = 0; + int lg_base = lg_tiny_min; + int lg_delta = lg_base; + + /* Outputs that we update as we go. */ + size_t lookup_maxclass = 0; + size_t small_maxclass = 0; + int lg_large_minclass = 0; + size_t large_maxclass = 0; + + /* Tiny size classes. */ + while (lg_base < lg_quantum) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + if (sc->lg_delta_lookup != 0) { + nlbins = index + 1; + } + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + ntiny++; + /* Final written value is correct. */ + lg_tiny_maxclass = lg_base; + index++; + lg_delta = lg_base; + lg_base++; + } + + /* First non-tiny (pseudo) group. */ + if (ntiny != 0) { + sc_t *sc = &sc_data->sc[index]; + /* + * See the note in sc.h; the first non-tiny size class has an + * unusual encoding. + */ + lg_base--; + ndelta = 1; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + index++; + lg_base++; + lg_delta++; + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + } + while (ndelta < ngroup) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + index++; + ndelta++; + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + } + + /* All remaining groups. */ + lg_base = lg_base + lg_ngroup; + while (lg_base < ptr_bits - 1) { + ndelta = 1; + int ndelta_limit; + if (lg_base == ptr_bits - 2) { + ndelta_limit = ngroup - 1; + } else { + ndelta_limit = ngroup; + } + while (ndelta <= ndelta_limit) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + if (sc->lg_delta_lookup != 0) { + nlbins = index + 1; + /* Final written value is correct. */ + lookup_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + } + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + /* Final written value is correct. */ + small_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + if (lg_ngroup > 0) { + lg_large_minclass = lg_base + 1; + } else { + lg_large_minclass = lg_base + 2; + } + } + large_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + index++; + ndelta++; + } + lg_base++; + lg_delta++; + } + /* Additional outputs. */ + int nsizes = index; + unsigned lg_ceil_nsizes = lg_ceil(nsizes); + + /* Fill in the output data. */ + sc_data->ntiny = ntiny; + sc_data->nlbins = nlbins; + sc_data->nbins = nbins; + sc_data->nsizes = nsizes; + sc_data->lg_ceil_nsizes = lg_ceil_nsizes; + sc_data->npsizes = npsizes; + sc_data->lg_tiny_maxclass = lg_tiny_maxclass; + sc_data->lookup_maxclass = lookup_maxclass; + sc_data->small_maxclass = small_maxclass; + sc_data->lg_large_minclass = lg_large_minclass; + sc_data->large_minclass = (ZU(1) << lg_large_minclass); + sc_data->large_maxclass = large_maxclass; + + /* + * We compute these values in two ways: + * - Incrementally, as above. + * - In macros, in sc.h. + * The computation is easier when done incrementally, but putting it in + * a constant makes it available to the fast paths without having to + * touch the extra global cacheline. We assert, however, that the two + * computations are equivalent. + */ + assert(sc_data->npsizes == SC_NPSIZES); + assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS); + assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS); + assert(sc_data->large_minclass == SC_LARGE_MINCLASS); + assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS); + assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS); + + /* + * In the allocation fastpath, we want to assume that we can + * unconditionally subtract the requested allocation size from + * a ssize_t, and detect passing through 0 correctly. This + * results in optimal generated code. For this to work, the + * maximum allocation size must be less than SSIZE_MAX. + */ + assert(SC_LARGE_MAXCLASS < SSIZE_MAX); +} + +void +sc_data_init(sc_data_t *sc_data) { + assert(!sc_data->initialized); + + int lg_max_lookup = 12; + + size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN, + lg_max_lookup, LG_PAGE, 2); + + sc_data->initialized = true; +} + +static void +sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) { + size_t min_pgs = reg_size / PAGE; + if (reg_size % PAGE != 0) { + min_pgs++; + } + /* + * BITMAP_MAXBITS is actually determined by putting the smallest + * possible size-class on one page, so this can never be 0. + */ + size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE; + + assert(min_pgs <= max_pgs); + assert(min_pgs > 0); + assert(max_pgs >= 1); + if (pgs_guess < min_pgs) { + sc->pgs = (int)min_pgs; + } else if (pgs_guess > max_pgs) { + sc->pgs = (int)max_pgs; + } else { + sc->pgs = (int)pgs_guess; + } +} + +void +sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) { + assert(data->initialized); + for (int i = 0; i < data->nsizes; i++) { + sc_t *sc = &data->sc[i]; + if (!sc->bin) { + break; + } + size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta, + sc->ndelta); + if (begin <= reg_size && reg_size <= end) { + sc_data_update_sc_slab_size(sc, reg_size, pgs); + } + } +} + +void +sc_boot(sc_data_t *data) { + sc_data_init(data); +} diff --git a/dep/jemalloc/src/stats.c b/dep/jemalloc/src/stats.c index 08b9507cfe9..4c427e0d35b 100644 --- a/dep/jemalloc/src/stats.c +++ b/dep/jemalloc/src/stats.c @@ -52,6 +52,20 @@ char opt_stats_print_opts[stats_print_tot_num_options+1] = ""; /******************************************************************************/ +static uint64_t +rate_per_second(uint64_t value, uint64_t uptime_ns) { + uint64_t billion = 1000000000; + if (uptime_ns == 0 || value == 0) { + return 0; + } + if (uptime_ns < billion) { + return value; + } else { + uint64_t uptime_s = uptime_ns / billion; + return value / uptime_s; + } +} + /* Calculate x.yyy and output a string (takes a fixed sized char array). */ static bool get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { @@ -104,24 +118,26 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, #define WIDTH_uint32_t 12 #define WIDTH_uint64_t 16 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ col = &col_##counter_type[k_##counter_type]; \ ++k_##counter_type; \ emitter_col_init(col, row); \ col->justify = emitter_justify_right; \ - col->width = WIDTH_##counter_type; \ + col->width = derived ? 8 : WIDTH_##counter_type; \ col->type = emitter_type_title; \ col->str_val = human; MUTEX_PROF_COUNTERS #undef OP #undef WIDTH_uint32_t #undef WIDTH_uint64_t + col_uint64_t[mutex_counter_total_wait_time_ps].width = 10; } static void mutex_stats_read_global(const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; col_name->str_val = name; @@ -129,12 +145,17 @@ mutex_stats_read_global(const char *name, emitter_col_t *col_name, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "mutexes", name, #counter); \ - CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "mutexes", name, #counter); \ + CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -145,7 +166,8 @@ static void mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; col_name->str_val = name; @@ -153,13 +175,17 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ - CTL_M2_GET(cmd, arena_ind, \ - (counter_type *)&dst->bool_val, counter_type); + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ + CTL_M2_GET(cmd, arena_ind, (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -169,19 +195,25 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, static void mutex_stats_read_arena_bin(unsigned arena_ind, unsigned bin_ind, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human) \ +#define OP(counter, counter_type, human, derived, base_counter) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.bins.0","mutex", #counter); \ - CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ - (counter_type *)&dst->bool_val, counter_type); + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.bins.0","mutex", #counter); \ + CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ + (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -204,19 +236,38 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, type, human) \ - col = &col_##type[k_##type]; \ - ++k_##type; \ - emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ - (const void *)&col->bool_val); +#define OP(counter, type, human, derived, base_counter) \ + if (!derived) { \ + col = &col_##type[k_##type]; \ + ++k_##type; \ + emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ + (const void *)&col->bool_val); \ + } MUTEX_PROF_COUNTERS; #undef OP #undef EMITTER_TYPE_uint32_t #undef EMITTER_TYPE_uint64_t } +#define COL(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_t col_##column_name; \ + emitter_col_init(&col_##column_name, &row_name); \ + col_##column_name.justify = emitter_justify_##left_or_right; \ + col_##column_name.width = col_width; \ + col_##column_name.type = emitter_type_##etype; + +#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ + COL(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_t header_##column_name; \ + emitter_col_init(&header_##column_name, &header_##row_name); \ + header_##column_name.justify = emitter_justify_##left_or_right; \ + header_##column_name.width = col_width; \ + header_##column_name.type = emitter_type_title; \ + header_##column_name.str_val = human ? human : #column_name; + + static void -stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { +stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { size_t page; bool in_gap, in_gap_prev; unsigned nbins, j; @@ -230,43 +281,36 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { emitter_row_t row; emitter_row_init(&row); -#define COL(name, left_or_right, col_width, etype) \ - emitter_col_t col_##name; \ - emitter_col_init(&col_##name, &row); \ - col_##name.justify = emitter_justify_##left_or_right; \ - col_##name.width = col_width; \ - col_##name.type = emitter_type_##etype; \ - emitter_col_t header_col_##name; \ - emitter_col_init(&header_col_##name, &header_row); \ - header_col_##name.justify = emitter_justify_##left_or_right; \ - header_col_##name.width = col_width; \ - header_col_##name.type = emitter_type_title; \ - header_col_##name.str_val = #name; - - COL(size, right, 20, size) - COL(ind, right, 4, unsigned) - COL(allocated, right, 13, uint64) - COL(nmalloc, right, 13, uint64) - COL(ndalloc, right, 13, uint64) - COL(nrequests, right, 13, uint64) - COL(curregs, right, 13, size) - COL(curslabs, right, 13, size) - COL(regs, right, 5, unsigned) - COL(pgs, right, 4, size) + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, allocated, NULL, right, 13, uint64) + COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64) + COL_HDR(row, nshards, NULL, right, 9, unsigned) + COL_HDR(row, curregs, NULL, right, 13, size) + COL_HDR(row, curslabs, NULL, right, 13, size) + COL_HDR(row, regs, NULL, right, 5, unsigned) + COL_HDR(row, pgs, NULL, right, 4, size) /* To buffer a right- and left-justified column. */ - COL(justify_spacer, right, 1, title) - COL(util, right, 6, title) - COL(nfills, right, 13, uint64) - COL(nflushes, right, 13, uint64) - COL(nslabs, right, 13, uint64) - COL(nreslabs, right, 13, uint64) -#undef COL + COL_HDR(row, justify_spacer, NULL, right, 1, title) + COL_HDR(row, util, NULL, right, 6, title) + COL_HDR(row, nfills, NULL, right, 13, uint64) + COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nflushes, NULL, right, 13, uint64) + COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nslabs, NULL, right, 13, uint64) + COL_HDR(row, nreslabs, NULL, right, 13, uint64) + COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64) /* Don't want to actually print the name. */ - header_col_justify_spacer.str_val = " "; + header_justify_spacer.str_val = " "; col_justify_spacer.str_val = " "; - emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters]; emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters]; @@ -284,16 +328,16 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { * We print a "bins:" header as part of the table row; we need to adjust * the header size column to compensate. */ - header_col_size.width -=5; + header_size.width -=5; emitter_table_printf(emitter, "bins:"); emitter_table_row(emitter, &header_row); - emitter_json_arr_begin(emitter, "bins"); + emitter_json_array_kv_begin(emitter, "bins"); for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nslabs; size_t reg_size, slab_size, curregs; size_t curslabs; - uint32_t nregs; + uint32_t nregs, nshards; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nreslabs; @@ -310,6 +354,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { CTL_M2_GET("arenas.bin.0.size", j, ®_size, size_t); CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t); CTL_M2_GET("arenas.bin.0.slab_size", j, &slab_size, size_t); + CTL_M2_GET("arenas.bin.0.nshards", j, &nshards, uint32_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc, uint64_t); @@ -330,10 +375,10 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { if (mutex) { mutex_stats_read_arena_bin(i, j, col_mutex64, - col_mutex32); + col_mutex32, uptime); } - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); emitter_json_kv(emitter, "nmalloc", emitter_type_uint64, &nmalloc); emitter_json_kv(emitter, "ndalloc", emitter_type_uint64, @@ -351,12 +396,12 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { emitter_json_kv(emitter, "curslabs", emitter_type_size, &curslabs); if (mutex) { - emitter_json_dict_begin(emitter, "mutex"); + emitter_json_object_kv_begin(emitter, "mutex"); mutex_stats_emit(emitter, NULL, col_mutex64, col_mutex32); - emitter_json_dict_end(emitter); + emitter_json_object_end(emitter); } - emitter_json_arr_obj_end(emitter); + emitter_json_object_end(emitter); size_t availregs = nregs * curslabs; char util[6]; @@ -381,17 +426,24 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { col_ind.unsigned_val = j; col_allocated.size_val = curregs * reg_size; col_nmalloc.uint64_val = nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); col_ndalloc.uint64_val = ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; + col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); + col_nshards.unsigned_val = nshards; col_curregs.size_val = curregs; col_curslabs.size_val = curslabs; col_regs.unsigned_val = nregs; col_pgs.size_val = slab_size / page; col_util.str_val = util; col_nfills.uint64_val = nfills; + col_nfills_ps.uint64_val = rate_per_second(nfills, uptime); col_nflushes.uint64_val = nflushes; + col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime); col_nslabs.uint64_val = nslabs; col_nreslabs.uint64_val = nreslabs; + col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime); /* * Note that mutex columns were initialized above, if mutex == @@ -400,7 +452,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { emitter_table_row(emitter, &row); } - emitter_json_arr_end(emitter); /* Close "bins". */ + emitter_json_array_end(emitter); /* Close "bins". */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); @@ -408,7 +460,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { } static void -stats_arena_lextents_print(emitter_t *emitter, unsigned i) { +stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { unsigned nbins, nlextents, j; bool in_gap, in_gap_prev; @@ -420,34 +472,22 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i) { emitter_row_t row; emitter_row_init(&row); -#define COL(name, left_or_right, col_width, etype) \ - emitter_col_t header_##name; \ - emitter_col_init(&header_##name, &header_row); \ - header_##name.justify = emitter_justify_##left_or_right; \ - header_##name.width = col_width; \ - header_##name.type = emitter_type_title; \ - header_##name.str_val = #name; \ - \ - emitter_col_t col_##name; \ - emitter_col_init(&col_##name, &row); \ - col_##name.justify = emitter_justify_##left_or_right; \ - col_##name.width = col_width; \ - col_##name.type = emitter_type_##etype; - - COL(size, right, 20, size) - COL(ind, right, 4, unsigned) - COL(allocated, right, 13, size) - COL(nmalloc, right, 13, uint64) - COL(ndalloc, right, 13, uint64) - COL(nrequests, right, 13, uint64) - COL(curlextents, right, 13, size) -#undef COL + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, allocated, NULL, right, 13, size) + COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, curlextents, NULL, right, 13, size) /* As with bins, we label the large extents table. */ header_size.width -= 6; emitter_table_printf(emitter, "large:"); emitter_table_row(emitter, &header_row); - emitter_json_arr_begin(emitter, "lextents"); + emitter_json_array_kv_begin(emitter, "lextents"); for (j = 0, in_gap = false; j < nlextents; j++) { uint64_t nmalloc, ndalloc, nrequests; @@ -471,31 +511,122 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i) { CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents", i, j, &curlextents, size_t); - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); emitter_json_kv(emitter, "curlextents", emitter_type_size, &curlextents); - emitter_json_arr_obj_end(emitter); + emitter_json_object_end(emitter); col_size.size_val = lextent_size; col_ind.unsigned_val = nbins + j; col_allocated.size_val = curlextents * lextent_size; col_nmalloc.uint64_val = nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); col_ndalloc.uint64_val = ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; + col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); col_curlextents.size_val = curlextents; if (!in_gap) { emitter_table_row(emitter, &row); } } - emitter_json_arr_end(emitter); /* Close "lextents". */ + emitter_json_array_end(emitter); /* Close "lextents". */ + if (in_gap) { + emitter_table_printf(emitter, " ---\n"); + } +} + +static void +stats_arena_extents_print(emitter_t *emitter, unsigned i) { + unsigned j; + bool in_gap, in_gap_prev; + emitter_row_t header_row; + emitter_row_init(&header_row); + emitter_row_t row; + emitter_row_init(&row); + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, ndirty, NULL, right, 13, size) + COL_HDR(row, dirty, NULL, right, 13, size) + COL_HDR(row, nmuzzy, NULL, right, 13, size) + COL_HDR(row, muzzy, NULL, right, 13, size) + COL_HDR(row, nretained, NULL, right, 13, size) + COL_HDR(row, retained, NULL, right, 13, size) + COL_HDR(row, ntotal, NULL, right, 13, size) + COL_HDR(row, total, NULL, right, 13, size) + + /* Label this section. */ + header_size.width -= 8; + emitter_table_printf(emitter, "extents:"); + emitter_table_row(emitter, &header_row); + emitter_json_array_kv_begin(emitter, "extents"); + + in_gap = false; + for (j = 0; j < SC_NPSIZES; j++) { + size_t ndirty, nmuzzy, nretained, total, dirty_bytes, + muzzy_bytes, retained_bytes, total_bytes; + CTL_M2_M4_GET("stats.arenas.0.extents.0.ndirty", i, j, + &ndirty, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.nmuzzy", i, j, + &nmuzzy, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.nretained", i, j, + &nretained, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.dirty_bytes", i, j, + &dirty_bytes, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.muzzy_bytes", i, j, + &muzzy_bytes, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.retained_bytes", i, j, + &retained_bytes, size_t); + total = ndirty + nmuzzy + nretained; + total_bytes = dirty_bytes + muzzy_bytes + retained_bytes; + + in_gap_prev = in_gap; + in_gap = (total == 0); + + if (in_gap_prev && !in_gap) { + emitter_table_printf(emitter, + " ---\n"); + } + + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); + emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); + emitter_json_kv(emitter, "nretained", emitter_type_size, + &nretained); + + emitter_json_kv(emitter, "dirty_bytes", emitter_type_size, + &dirty_bytes); + emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size, + &muzzy_bytes); + emitter_json_kv(emitter, "retained_bytes", emitter_type_size, + &retained_bytes); + emitter_json_object_end(emitter); + + col_size.size_val = sz_pind2sz(j); + col_ind.size_val = j; + col_ndirty.size_val = ndirty; + col_dirty.size_val = dirty_bytes; + col_nmuzzy.size_val = nmuzzy; + col_muzzy.size_val = muzzy_bytes; + col_nretained.size_val = nretained; + col_retained.size_val = retained_bytes; + col_ntotal.size_val = total; + col_total.size_val = total_bytes; + + if (!in_gap) { + emitter_table_row(emitter, &row); + } + } + emitter_json_array_end(emitter); /* Close "extents". */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); } } static void -stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) { +stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { emitter_row_t row; emitter_col_t col_name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; @@ -504,29 +635,29 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) { emitter_row_init(&row); mutex_stats_init_cols(&row, "", &col_name, col64, col32); - emitter_json_dict_begin(emitter, "mutexes"); + emitter_json_object_kv_begin(emitter, "mutexes"); emitter_table_row(emitter, &row); for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes; i++) { const char *name = arena_mutex_names[i]; - emitter_json_dict_begin(emitter, name); + emitter_json_object_kv_begin(emitter, name); mutex_stats_read_arena(arena_ind, i, name, &col_name, col64, - col32); + col32, uptime); mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_dict_end(emitter); /* Close the mutex dict. */ + emitter_json_object_end(emitter); /* Close the mutex dict. */ } - emitter_json_dict_end(emitter); /* End "mutexes". */ + emitter_json_object_end(emitter); /* End "mutexes". */ } static void stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, - bool mutex) { + bool mutex, bool extents) { unsigned nthreads; const char *dss; ssize_t dirty_decay_ms, muzzy_decay_ms; size_t page, pactive, pdirty, pmuzzy, mapped, retained; - size_t base, internal, resident, metadata_thp; + size_t base, internal, resident, metadata_thp, extent_avail; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; size_t small_allocated; @@ -594,98 +725,74 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, &muzzy_purged); /* Table-style emission. */ - emitter_col_t decay_type; - emitter_col_init(&decay_type, &decay_row); - decay_type.justify = emitter_justify_right; - decay_type.width = 9; - decay_type.type = emitter_type_title; - decay_type.str_val = "decaying:"; - - emitter_col_t decay_time; - emitter_col_init(&decay_time, &decay_row); - decay_time.justify = emitter_justify_right; - decay_time.width = 6; - decay_time.type = emitter_type_title; - decay_time.str_val = "time"; - - emitter_col_t decay_npages; - emitter_col_init(&decay_npages, &decay_row); - decay_npages.justify = emitter_justify_right; - decay_npages.width = 13; - decay_npages.type = emitter_type_title; - decay_npages.str_val = "npages"; - - emitter_col_t decay_sweeps; - emitter_col_init(&decay_sweeps, &decay_row); - decay_sweeps.justify = emitter_justify_right; - decay_sweeps.width = 13; - decay_sweeps.type = emitter_type_title; - decay_sweeps.str_val = "sweeps"; - - emitter_col_t decay_madvises; - emitter_col_init(&decay_madvises, &decay_row); - decay_madvises.justify = emitter_justify_right; - decay_madvises.width = 13; - decay_madvises.type = emitter_type_title; - decay_madvises.str_val = "madvises"; - - emitter_col_t decay_purged; - emitter_col_init(&decay_purged, &decay_row); - decay_purged.justify = emitter_justify_right; - decay_purged.width = 13; - decay_purged.type = emitter_type_title; - decay_purged.str_val = "purged"; + COL(decay_row, decay_type, right, 9, title); + col_decay_type.str_val = "decaying:"; + + COL(decay_row, decay_time, right, 6, title); + col_decay_time.str_val = "time"; + + COL(decay_row, decay_npages, right, 13, title); + col_decay_npages.str_val = "npages"; + + COL(decay_row, decay_sweeps, right, 13, title); + col_decay_sweeps.str_val = "sweeps"; + + COL(decay_row, decay_madvises, right, 13, title); + col_decay_madvises.str_val = "madvises"; + + COL(decay_row, decay_purged, right, 13, title); + col_decay_purged.str_val = "purged"; /* Title row. */ emitter_table_row(emitter, &decay_row); /* Dirty row. */ - decay_type.str_val = "dirty:"; + col_decay_type.str_val = "dirty:"; if (dirty_decay_ms >= 0) { - decay_time.type = emitter_type_ssize; - decay_time.ssize_val = dirty_decay_ms; + col_decay_time.type = emitter_type_ssize; + col_decay_time.ssize_val = dirty_decay_ms; } else { - decay_time.type = emitter_type_title; - decay_time.str_val = "N/A"; + col_decay_time.type = emitter_type_title; + col_decay_time.str_val = "N/A"; } - decay_npages.type = emitter_type_size; - decay_npages.size_val = pdirty; + col_decay_npages.type = emitter_type_size; + col_decay_npages.size_val = pdirty; - decay_sweeps.type = emitter_type_uint64; - decay_sweeps.uint64_val = dirty_npurge; + col_decay_sweeps.type = emitter_type_uint64; + col_decay_sweeps.uint64_val = dirty_npurge; - decay_madvises.type = emitter_type_uint64; - decay_madvises.uint64_val = dirty_nmadvise; + col_decay_madvises.type = emitter_type_uint64; + col_decay_madvises.uint64_val = dirty_nmadvise; - decay_purged.type = emitter_type_uint64; - decay_purged.uint64_val = dirty_purged; + col_decay_purged.type = emitter_type_uint64; + col_decay_purged.uint64_val = dirty_purged; emitter_table_row(emitter, &decay_row); /* Muzzy row. */ - decay_type.str_val = "muzzy:"; + col_decay_type.str_val = "muzzy:"; if (muzzy_decay_ms >= 0) { - decay_time.type = emitter_type_ssize; - decay_time.ssize_val = muzzy_decay_ms; + col_decay_time.type = emitter_type_ssize; + col_decay_time.ssize_val = muzzy_decay_ms; } else { - decay_time.type = emitter_type_title; - decay_time.str_val = "N/A"; + col_decay_time.type = emitter_type_title; + col_decay_time.str_val = "N/A"; } - decay_npages.type = emitter_type_size; - decay_npages.size_val = pmuzzy; + col_decay_npages.type = emitter_type_size; + col_decay_npages.size_val = pmuzzy; - decay_sweeps.type = emitter_type_uint64; - decay_sweeps.uint64_val = muzzy_npurge; + col_decay_sweeps.type = emitter_type_uint64; + col_decay_sweeps.uint64_val = muzzy_npurge; - decay_madvises.type = emitter_type_uint64; - decay_madvises.uint64_val = muzzy_nmadvise; + col_decay_madvises.type = emitter_type_uint64; + col_decay_madvises.uint64_val = muzzy_nmadvise; - decay_purged.type = emitter_type_uint64; - decay_purged.uint64_val = muzzy_purged; + col_decay_purged.type = emitter_type_uint64; + col_decay_purged.uint64_val = muzzy_purged; emitter_table_row(emitter, &decay_row); @@ -693,81 +800,89 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, emitter_row_t alloc_count_row; emitter_row_init(&alloc_count_row); - emitter_col_t alloc_count_title; - emitter_col_init(&alloc_count_title, &alloc_count_row); - alloc_count_title.justify = emitter_justify_left; - alloc_count_title.width = 25; - alloc_count_title.type = emitter_type_title; - alloc_count_title.str_val = ""; - - emitter_col_t alloc_count_allocated; - emitter_col_init(&alloc_count_allocated, &alloc_count_row); - alloc_count_allocated.justify = emitter_justify_right; - alloc_count_allocated.width = 12; - alloc_count_allocated.type = emitter_type_title; - alloc_count_allocated.str_val = "allocated"; - - emitter_col_t alloc_count_nmalloc; - emitter_col_init(&alloc_count_nmalloc, &alloc_count_row); - alloc_count_nmalloc.justify = emitter_justify_right; - alloc_count_nmalloc.width = 12; - alloc_count_nmalloc.type = emitter_type_title; - alloc_count_nmalloc.str_val = "nmalloc"; - - emitter_col_t alloc_count_ndalloc; - emitter_col_init(&alloc_count_ndalloc, &alloc_count_row); - alloc_count_ndalloc.justify = emitter_justify_right; - alloc_count_ndalloc.width = 12; - alloc_count_ndalloc.type = emitter_type_title; - alloc_count_ndalloc.str_val = "ndalloc"; - - emitter_col_t alloc_count_nrequests; - emitter_col_init(&alloc_count_nrequests, &alloc_count_row); - alloc_count_nrequests.justify = emitter_justify_right; - alloc_count_nrequests.width = 12; - alloc_count_nrequests.type = emitter_type_title; - alloc_count_nrequests.str_val = "nrequests"; + COL(alloc_count_row, count_title, left, 21, title); + col_count_title.str_val = ""; + + COL(alloc_count_row, count_allocated, right, 16, title); + col_count_allocated.str_val = "allocated"; + + COL(alloc_count_row, count_nmalloc, right, 16, title); + col_count_nmalloc.str_val = "nmalloc"; + COL(alloc_count_row, count_nmalloc_ps, right, 8, title); + col_count_nmalloc_ps.str_val = "(#/sec)"; + + COL(alloc_count_row, count_ndalloc, right, 16, title); + col_count_ndalloc.str_val = "ndalloc"; + COL(alloc_count_row, count_ndalloc_ps, right, 8, title); + col_count_ndalloc_ps.str_val = "(#/sec)"; + + COL(alloc_count_row, count_nrequests, right, 16, title); + col_count_nrequests.str_val = "nrequests"; + COL(alloc_count_row, count_nrequests_ps, right, 10, title); + col_count_nrequests_ps.str_val = "(#/sec)"; emitter_table_row(emitter, &alloc_count_row); + col_count_nmalloc_ps.type = emitter_type_uint64; + col_count_ndalloc_ps.type = emitter_type_uint64; + col_count_nrequests_ps.type = emitter_type_uint64; + #define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ &small_or_large##_##name, valtype##_t); \ emitter_json_kv(emitter, #name, emitter_type_##valtype, \ &small_or_large##_##name); \ - alloc_count_##name.type = emitter_type_##valtype; \ - alloc_count_##name.valtype##_val = small_or_large##_##name; + col_count_##name.type = emitter_type_##valtype; \ + col_count_##name.valtype##_val = small_or_large##_##name; - emitter_json_dict_begin(emitter, "small"); - alloc_count_title.str_val = "small:"; + emitter_json_object_kv_begin(emitter, "small"); + col_count_title.str_val = "small:"; GET_AND_EMIT_ALLOC_STAT(small, allocated, size) GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); - emitter_json_dict_end(emitter); /* Close "small". */ + emitter_json_object_end(emitter); /* Close "small". */ - emitter_json_dict_begin(emitter, "large"); - alloc_count_title.str_val = "large:"; + emitter_json_object_kv_begin(emitter, "large"); + col_count_title.str_val = "large:"; GET_AND_EMIT_ALLOC_STAT(large, allocated, size) GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); - emitter_json_dict_end(emitter); /* Close "large". */ + emitter_json_object_end(emitter); /* Close "large". */ #undef GET_AND_EMIT_ALLOC_STAT /* Aggregated small + large stats are emitter only in table mode. */ - alloc_count_title.str_val = "total:"; - alloc_count_allocated.size_val = small_allocated + large_allocated; - alloc_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; - alloc_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; - alloc_count_nrequests.uint64_val = small_nrequests + large_nrequests; + col_count_title.str_val = "total:"; + col_count_allocated.size_val = small_allocated + large_allocated; + col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; + col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; + col_count_nrequests.uint64_val = small_nrequests + large_nrequests; + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); emitter_row_t mem_count_row; @@ -776,14 +891,14 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, emitter_col_t mem_count_title; emitter_col_init(&mem_count_title, &mem_count_row); mem_count_title.justify = emitter_justify_left; - mem_count_title.width = 25; + mem_count_title.width = 21; mem_count_title.type = emitter_type_title; mem_count_title.str_val = ""; emitter_col_t mem_count_val; emitter_col_init(&mem_count_val, &mem_count_row); mem_count_val.justify = emitter_justify_right; - mem_count_val.width = 12; + mem_count_val.width = 16; mem_count_val.type = emitter_type_title; mem_count_val.str_val = ""; @@ -809,16 +924,20 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, GET_AND_EMIT_MEM_STAT(metadata_thp) GET_AND_EMIT_MEM_STAT(tcache_bytes) GET_AND_EMIT_MEM_STAT(resident) + GET_AND_EMIT_MEM_STAT(extent_avail) #undef GET_AND_EMIT_MEM_STAT if (mutex) { - stats_arena_mutexes_print(emitter, i); + stats_arena_mutexes_print(emitter, i, uptime); } if (bins) { - stats_arena_bins_print(emitter, mutex, i); + stats_arena_bins_print(emitter, mutex, i, uptime); } if (large) { - stats_arena_lextents_print(emitter, i); + stats_arena_lextents_print(emitter, i, uptime); + } + if (extents) { + stats_arena_extents_print(emitter, i); } } @@ -891,6 +1010,8 @@ stats_general_print(emitter_t *emitter) { #define OPT_WRITE_UNSIGNED(name) \ OPT_WRITE(name, uv, usz, emitter_type_unsigned) +#define OPT_WRITE_SIZE_T(name) \ + OPT_WRITE(name, sv, ssz, emitter_type_size) #define OPT_WRITE_SSIZE_T(name) \ OPT_WRITE(name, ssv, sssz, emitter_type_ssize) #define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ @@ -908,11 +1029,12 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_CHAR_P("dss") OPT_WRITE_UNSIGNED("narenas") OPT_WRITE_CHAR_P("percpu_arena") + OPT_WRITE_SIZE_T("oversize_threshold") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms") OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms") - OPT_WRITE_UNSIGNED("lg_extent_max_active_fit") + OPT_WRITE_SIZE_T("lg_extent_max_active_fit") OPT_WRITE_CHAR_P("junk") OPT_WRITE_BOOL("zero") OPT_WRITE_BOOL("utrace") @@ -977,7 +1099,7 @@ stats_general_print(emitter_t *emitter) { * The json output sticks arena info into an "arenas" dict; the table * output puts them at the top-level. */ - emitter_json_dict_begin(emitter, "arenas"); + emitter_json_object_kv_begin(emitter, "arenas"); CTL_GET("arenas.narenas", &uv, unsigned); emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv); @@ -1018,9 +1140,9 @@ stats_general_print(emitter_t *emitter) { * (not just omit the printing). */ if (emitter->output == emitter_output_json) { - emitter_json_arr_begin(emitter, "bin"); + emitter_json_array_kv_begin(emitter, "bin"); for (unsigned i = 0; i < nbins; i++) { - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); CTL_M2_GET("arenas.bin.0.size", i, &sv, size_t); emitter_json_kv(emitter, "size", emitter_type_size, @@ -1034,9 +1156,13 @@ stats_general_print(emitter_t *emitter) { emitter_json_kv(emitter, "slab_size", emitter_type_size, &sv); - emitter_json_arr_obj_end(emitter); + CTL_M2_GET("arenas.bin.0.nshards", i, &u32v, uint32_t); + emitter_json_kv(emitter, "nshards", emitter_type_uint32, + &u32v); + + emitter_json_object_end(emitter); } - emitter_json_arr_end(emitter); /* Close "bin". */ + emitter_json_array_end(emitter); /* Close "bin". */ } unsigned nlextents; @@ -1045,25 +1171,25 @@ stats_general_print(emitter_t *emitter) { emitter_type_unsigned, &nlextents); if (emitter->output == emitter_output_json) { - emitter_json_arr_begin(emitter, "lextent"); + emitter_json_array_kv_begin(emitter, "lextent"); for (unsigned i = 0; i < nlextents; i++) { - emitter_json_arr_obj_begin(emitter); + emitter_json_object_begin(emitter); CTL_M2_GET("arenas.lextent.0.size", i, &sv, size_t); emitter_json_kv(emitter, "size", emitter_type_size, &sv); - emitter_json_arr_obj_end(emitter); + emitter_json_object_end(emitter); } - emitter_json_arr_end(emitter); /* Close "lextent". */ + emitter_json_array_end(emitter); /* Close "lextent". */ } - emitter_json_dict_end(emitter); /* Close "arenas" */ + emitter_json_object_end(emitter); /* Close "arenas" */ } static void stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, - bool unmerged, bool bins, bool large, bool mutex) { + bool unmerged, bool bins, bool large, bool mutex, bool extents) { /* * These should be deleted. We keep them around for a while, to aid in * the transition to the emitter code. @@ -1095,7 +1221,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, } /* Generic global stats. */ - emitter_json_dict_begin(emitter, "stats"); + emitter_json_object_kv_begin(emitter, "stats"); emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); emitter_json_kv(emitter, "active", emitter_type_size, &active); emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); @@ -1111,14 +1237,14 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, resident, mapped, retained); /* Background thread stats. */ - emitter_json_dict_begin(emitter, "background_thread"); + emitter_json_object_kv_begin(emitter, "background_thread"); emitter_json_kv(emitter, "num_threads", emitter_type_size, &num_background_threads); emitter_json_kv(emitter, "num_runs", emitter_type_uint64, &background_thread_num_runs); emitter_json_kv(emitter, "run_interval", emitter_type_uint64, &background_thread_run_interval); - emitter_json_dict_end(emitter); /* Close "background_thread". */ + emitter_json_object_end(emitter); /* Close "background_thread". */ emitter_table_printf(emitter, "Background threads: %zu, " "num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n", @@ -1130,30 +1256,33 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_col_t name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; emitter_col_t col32[mutex_prof_num_uint32_t_counters]; + uint64_t uptime; emitter_row_init(&row); mutex_stats_init_cols(&row, "", &name, col64, col32); emitter_table_row(emitter, &row); - emitter_json_dict_begin(emitter, "mutexes"); + emitter_json_object_kv_begin(emitter, "mutexes"); + + CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { mutex_stats_read_global(global_mutex_names[i], &name, - col64, col32); - emitter_json_dict_begin(emitter, global_mutex_names[i]); + col64, col32, uptime); + emitter_json_object_kv_begin(emitter, global_mutex_names[i]); mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_dict_end(emitter); + emitter_json_object_end(emitter); } - emitter_json_dict_end(emitter); /* Close "mutexes". */ + emitter_json_object_end(emitter); /* Close "mutexes". */ } - emitter_json_dict_end(emitter); /* Close "stats". */ + emitter_json_object_end(emitter); /* Close "stats". */ if (merged || destroyed || unmerged) { unsigned narenas; - emitter_json_dict_begin(emitter, "stats.arenas"); + emitter_json_object_kv_begin(emitter, "stats.arenas"); CTL_GET("arenas.narenas", &narenas, unsigned); size_t mib[3]; @@ -1182,10 +1311,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, if (merged && (ninitialized > 1 || !unmerged)) { /* Print merged arena stats. */ emitter_table_printf(emitter, "Merged arenas stats:\n"); - emitter_json_dict_begin(emitter, "merged"); + emitter_json_object_kv_begin(emitter, "merged"); stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, - large, mutex); - emitter_json_dict_end(emitter); /* Close "merged". */ + large, mutex, extents); + emitter_json_object_end(emitter); /* Close "merged". */ } /* Destroyed stats. */ @@ -1193,10 +1322,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, /* Print destroyed arena stats. */ emitter_table_printf(emitter, "Destroyed arenas stats:\n"); - emitter_json_dict_begin(emitter, "destroyed"); + emitter_json_object_kv_begin(emitter, "destroyed"); stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, - bins, large, mutex); - emitter_json_dict_end(emitter); /* Close "destroyed". */ + bins, large, mutex, extents); + emitter_json_object_end(emitter); /* Close "destroyed". */ } /* Unmerged stats. */ @@ -1206,18 +1335,18 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, char arena_ind_str[20]; malloc_snprintf(arena_ind_str, sizeof(arena_ind_str), "%u", i); - emitter_json_dict_begin(emitter, + emitter_json_object_kv_begin(emitter, arena_ind_str); emitter_table_printf(emitter, "arenas[%s]:\n", arena_ind_str); stats_arena_print(emitter, i, bins, - large, mutex); + large, mutex, extents); /* Close "<arena-ind>". */ - emitter_json_dict_end(emitter); + emitter_json_object_end(emitter); } } } - emitter_json_dict_end(emitter); /* Close "stats.arenas". */ + emitter_json_object_end(emitter); /* Close "stats.arenas". */ } } @@ -1270,17 +1399,17 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cbopaque); emitter_begin(&emitter); emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); - emitter_json_dict_begin(&emitter, "jemalloc"); + emitter_json_object_kv_begin(&emitter, "jemalloc"); if (general) { stats_general_print(&emitter); } if (config_stats) { stats_print_helper(&emitter, merged, destroyed, unmerged, - bins, large, mutex); + bins, large, mutex, extents); } - emitter_json_dict_end(&emitter); /* Closes the "jemalloc" dict. */ + emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); emitter_end(&emitter); } diff --git a/dep/jemalloc/src/sz.c b/dep/jemalloc/src/sz.c index 9de77e45fff..8633fb05005 100644 --- a/dep/jemalloc/src/sz.c +++ b/dep/jemalloc/src/sz.c @@ -2,106 +2,63 @@ #include "jemalloc/internal/sz.h" JEMALLOC_ALIGNED(CACHELINE) -const size_t sz_pind2sz_tab[NPSIZES+1] = { -#define PSZ_yes(lg_grp, ndelta, lg_delta) \ - (((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))), -#define PSZ_no(lg_grp, ndelta, lg_delta) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - PSZ_##psz(lg_grp, ndelta, lg_delta) - SIZE_CLASSES -#undef PSZ_yes -#undef PSZ_no -#undef SC - (LARGE_MAXCLASS + PAGE) -}; +size_t sz_pind2sz_tab[SC_NPSIZES+1]; + +static void +sz_boot_pind2sz_tab(const sc_data_t *sc_data) { + int pind = 0; + for (unsigned i = 0; i < SC_NSIZES; i++) { + const sc_t *sc = &sc_data->sc[i]; + if (sc->psz) { + sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + pind++; + } + } + for (int i = pind; i <= (int)SC_NPSIZES; i++) { + sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; + } +} JEMALLOC_ALIGNED(CACHELINE) -const size_t sz_index2size_tab[NSIZES] = { -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - ((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)), - SIZE_CLASSES -#undef SC -}; +size_t sz_index2size_tab[SC_NSIZES]; + +static void +sz_boot_index2size_tab(const sc_data_t *sc_data) { + for (unsigned i = 0; i < SC_NSIZES; i++) { + const sc_t *sc = &sc_data->sc[i]; + sz_index2size_tab[i] = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << (sc->lg_delta)); + } +} +/* + * To keep this table small, we divide sizes by the tiny min size, which gives + * the smallest interval for which the result can change. + */ JEMALLOC_ALIGNED(CACHELINE) -const uint8_t sz_size2index_tab[] = { -#if LG_TINY_MIN == 0 -/* The div module doesn't support division by 1. */ -#error "Unsupported LG_TINY_MIN" -#define S2B_0(i) i, -#elif LG_TINY_MIN == 1 -#warning "Dangerous LG_TINY_MIN" -#define S2B_1(i) i, -#elif LG_TINY_MIN == 2 -#warning "Dangerous LG_TINY_MIN" -#define S2B_2(i) i, -#elif LG_TINY_MIN == 3 -#define S2B_3(i) i, -#elif LG_TINY_MIN == 4 -#define S2B_4(i) i, -#elif LG_TINY_MIN == 5 -#define S2B_5(i) i, -#elif LG_TINY_MIN == 6 -#define S2B_6(i) i, -#elif LG_TINY_MIN == 7 -#define S2B_7(i) i, -#elif LG_TINY_MIN == 8 -#define S2B_8(i) i, -#elif LG_TINY_MIN == 9 -#define S2B_9(i) i, -#elif LG_TINY_MIN == 10 -#define S2B_10(i) i, -#elif LG_TINY_MIN == 11 -#define S2B_11(i) i, -#else -#error "Unsupported LG_TINY_MIN" -#endif -#if LG_TINY_MIN < 1 -#define S2B_1(i) S2B_0(i) S2B_0(i) -#endif -#if LG_TINY_MIN < 2 -#define S2B_2(i) S2B_1(i) S2B_1(i) -#endif -#if LG_TINY_MIN < 3 -#define S2B_3(i) S2B_2(i) S2B_2(i) -#endif -#if LG_TINY_MIN < 4 -#define S2B_4(i) S2B_3(i) S2B_3(i) -#endif -#if LG_TINY_MIN < 5 -#define S2B_5(i) S2B_4(i) S2B_4(i) -#endif -#if LG_TINY_MIN < 6 -#define S2B_6(i) S2B_5(i) S2B_5(i) -#endif -#if LG_TINY_MIN < 7 -#define S2B_7(i) S2B_6(i) S2B_6(i) -#endif -#if LG_TINY_MIN < 8 -#define S2B_8(i) S2B_7(i) S2B_7(i) -#endif -#if LG_TINY_MIN < 9 -#define S2B_9(i) S2B_8(i) S2B_8(i) -#endif -#if LG_TINY_MIN < 10 -#define S2B_10(i) S2B_9(i) S2B_9(i) -#endif -#if LG_TINY_MIN < 11 -#define S2B_11(i) S2B_10(i) S2B_10(i) -#endif -#define S2B_no(i) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - S2B_##lg_delta_lookup(index) - SIZE_CLASSES -#undef S2B_3 -#undef S2B_4 -#undef S2B_5 -#undef S2B_6 -#undef S2B_7 -#undef S2B_8 -#undef S2B_9 -#undef S2B_10 -#undef S2B_11 -#undef S2B_no -#undef SC -}; +uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1]; + +static void +sz_boot_size2index_tab(const sc_data_t *sc_data) { + size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1; + size_t dst_ind = 0; + for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; + sc_ind++) { + const sc_t *sc = &sc_data->sc[sc_ind]; + size_t sz = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1) + >> SC_LG_TINY_MIN); + for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) { + sz_size2index_tab[dst_ind] = sc_ind; + } + } +} + +void +sz_boot(const sc_data_t *sc_data) { + sz_boot_pind2sz_tab(sc_data); + sz_boot_index2size_tab(sc_data); + sz_boot_size2index_tab(sc_data); +} diff --git a/dep/jemalloc/src/tcache.c b/dep/jemalloc/src/tcache.c index a769a6b17bc..e7b970d9045 100644 --- a/dep/jemalloc/src/tcache.c +++ b/dep/jemalloc/src/tcache.c @@ -4,7 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" /******************************************************************************/ /* Data. */ @@ -41,7 +41,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { szind_t binind = tcache->next_gc_bin; cache_bin_t *tbin; - if (binind < NBINS) { + if (binind < SC_NBINS) { tbin = tcache_small_bin_get(tcache, binind); } else { tbin = tcache_large_bin_get(tcache, binind); @@ -50,7 +50,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { /* * Flush (ceiling) 3/4 of the objects below the low water mark. */ - if (binind < NBINS) { + if (binind < SC_NBINS) { tcache_bin_flush_small(tsd, tcache, tbin, binind, tbin->ncached - tbin->low_water + (tbin->low_water >> 2)); @@ -72,7 +72,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { * Increase fill count by 2X for small bins. Make sure * lg_fill_div stays greater than 0. */ - if (binind < NBINS && tcache->lg_fill_div[binind] > 1) { + if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) { tcache->lg_fill_div[binind]--; } } @@ -100,28 +100,68 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, return ret; } +/* Enabled with --enable-extra-size-check. */ +#ifdef JEMALLOC_EXTRA_SIZE_CHECK +static void +tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind, + size_t nflush, extent_t **extents){ + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + /* + * Verify that the items in the tcache all have the correct size; this + * is useful for catching sized deallocation bugs, also to fail early + * instead of corrupting metadata. Since this can be turned on for opt + * builds, avoid the branch in the loop. + */ + szind_t szind; + size_t sz_sum = binind * nflush; + for (unsigned i = 0 ; i < nflush; i++) { + rtree_extent_szind_read(tsdn, &extents_rtree, + rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true, + &extents[i], &szind); + sz_sum -= szind; + } + if (sz_sum != 0) { + malloc_printf("<jemalloc>: size mismatch in thread cache " + "detected, likely caused by sized deallocation bugs by " + "application. Abort.\n"); + abort(); + } +} +#endif + void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, unsigned rem) { bool merged_stats = false; - assert(binind < NBINS); + assert(binind < SC_NBINS); assert((cache_bin_sz_t)rem <= tbin->ncached); arena_t *arena = tcache->arena; assert(arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); + +#ifndef JEMALLOC_EXTRA_SIZE_CHECK /* Look up extent once per item. */ for (unsigned i = 0 ; i < nflush; i++) { item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } - +#else + tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, + item_extent); +#endif while (nflush > 0) { /* Lock the arena bin associated with the first object. */ extent_t *extent = item_extent[0]; - arena_t *bin_arena = extent_arena_get(extent); - bin_t *bin = &bin_arena->bins[binind]; + unsigned bin_arena_ind = extent_arena_ind_get(extent); + arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind, + false); + unsigned binshard = extent_binshard_get(extent); + assert(binshard < bin_infos[binind].n_shards); + bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard]; if (config_prof && bin_arena == arena) { if (arena_prof_accum(tsd_tsdn(tsd), arena, @@ -132,8 +172,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, } malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (config_stats && bin_arena == arena) { - assert(!merged_stats); + if (config_stats && bin_arena == arena && !merged_stats) { merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; @@ -145,9 +184,10 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_get(extent) == bin_arena) { + if (extent_arena_ind_get(extent) == bin_arena_ind + && extent_binshard_get(extent) == binshard) { arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), - bin_arena, extent, ptr); + bin_arena, bin, binind, extent, ptr); } else { /* * This object was allocated via a different @@ -169,8 +209,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - bin_t *bin = &arena->bins[binind]; - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind, + &binshard); bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; @@ -193,50 +234,63 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, assert(binind < nhbins); assert((cache_bin_sz_t)rem <= tbin->ncached); - arena_t *arena = tcache->arena; - assert(arena != NULL); + arena_t *tcache_arena = tcache->arena; + assert(tcache_arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); + +#ifndef JEMALLOC_EXTRA_SIZE_CHECK /* Look up extent once per item. */ for (unsigned i = 0 ; i < nflush; i++) { item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } - +#else + tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, + item_extent); +#endif while (nflush > 0) { /* Lock the arena associated with the first object. */ extent_t *extent = item_extent[0]; - arena_t *locked_arena = extent_arena_get(extent); - UNUSED bool idump; + unsigned locked_arena_ind = extent_arena_ind_get(extent); + arena_t *locked_arena = arena_get(tsd_tsdn(tsd), + locked_arena_ind, false); + bool idump; if (config_prof) { idump = false; } - malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); + bool lock_large = !arena_is_auto(locked_arena); + if (lock_large) { + malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); + } for (unsigned i = 0; i < nflush; i++) { void *ptr = *(tbin->avail - 1 - i); assert(ptr != NULL); extent = item_extent[i]; - if (extent_arena_get(extent) == locked_arena) { + if (extent_arena_ind_get(extent) == locked_arena_ind) { large_dalloc_prep_junked_locked(tsd_tsdn(tsd), extent); } } - if ((config_prof || config_stats) && locked_arena == arena) { + if ((config_prof || config_stats) && + (locked_arena == tcache_arena)) { if (config_prof) { - idump = arena_prof_accum(tsd_tsdn(tsd), arena, - tcache->prof_accumbytes); + idump = arena_prof_accum(tsd_tsdn(tsd), + tcache_arena, tcache->prof_accumbytes); tcache->prof_accumbytes = 0; } if (config_stats) { merged_stats = true; arena_stats_large_nrequests_add(tsd_tsdn(tsd), - &arena->stats, binind, + &tcache_arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } } - malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); + if (lock_large) { + malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); + } unsigned ndeferred = 0; for (unsigned i = 0; i < nflush; i++) { @@ -244,7 +298,7 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_get(extent) == locked_arena) { + if (extent_arena_ind_get(extent) == locked_arena_ind) { large_dalloc_finish(tsd_tsdn(tsd), extent); } else { /* @@ -270,8 +324,8 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats, - binind, tbin->tstats.nrequests); + arena_stats_large_nrequests_add(tsd_tsdn(tsd), + &tcache_arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } @@ -363,10 +417,10 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { size_t stack_offset = 0; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); - memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS); - memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS)); + memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS); + memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS)); unsigned i = 0; - for (; i < NBINS; i++) { + for (; i < SC_NBINS; i++) { tcache->lg_fill_div[i] = 1; stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); /* @@ -458,7 +512,7 @@ static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tcache->arena != NULL); - for (unsigned i = 0; i < NBINS; i++) { + for (unsigned i = 0; i < SC_NBINS; i++) { cache_bin_t *tbin = tcache_small_bin_get(tcache, i); tcache_bin_flush_small(tsd, tcache, tbin, i, 0); @@ -466,7 +520,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tbin->tstats.nrequests == 0); } } - for (unsigned i = NBINS; i < nhbins; i++) { + for (unsigned i = SC_NBINS; i < nhbins; i++) { cache_bin_t *tbin = tcache_large_bin_get(tcache, i); tcache_bin_flush_large(tsd, tbin, i, 0, tcache); @@ -491,6 +545,7 @@ tcache_flush(tsd_t *tsd) { static void tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { tcache_flush_cache(tsd, tcache); + arena_t *arena = tcache->arena; tcache_arena_dissociate(tsd_tsdn(tsd), tcache); if (tsd_tcache) { @@ -503,6 +558,23 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { /* Release both the tcache struct and avail array. */ idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); } + + /* + * The deallocation and tcache flush above may not trigger decay since + * we are on the tcache shutdown path (potentially with non-nominal + * tsd). Manually trigger decay to avoid pathological cases. Also + * include arena 0 because the tcache array is allocated from it. + */ + arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), + false, false); + + if (arena_nthreads_get(arena, false) == 0 && + !background_thread_enabled()) { + /* Force purging when no threads assigned to the arena anymore. */ + arena_decay(tsd_tsdn(tsd), arena, false, true); + } else { + arena_decay(tsd_tsdn(tsd), arena, false, false); + } } /* For auto tcache (embedded in TSD) only. */ @@ -532,10 +604,10 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); /* Merge and reset tcache stats. */ - for (i = 0; i < NBINS; i++) { - bin_t *bin = &arena->bins[i]; + for (i = 0; i < SC_NBINS; i++) { cache_bin_t *tbin = tcache_small_bin_get(tcache, i); - malloc_mutex_lock(tsdn, &bin->lock); + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard); bin->stats.nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(tsdn, &bin->lock); tbin->tstats.nrequests = 0; @@ -614,23 +686,32 @@ label_return: } static tcache_t * -tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) { +tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); if (elm->tcache == NULL) { return NULL; } tcache_t *tcache = elm->tcache; - elm->tcache = NULL; + if (allow_reinit) { + elm->tcache = TCACHES_ELM_NEED_REINIT; + } else { + elm->tcache = NULL; + } + + if (tcache == TCACHES_ELM_NEED_REINIT) { + return NULL; + } return tcache; } void tcaches_flush(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); - tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); + tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); if (tcache != NULL) { + /* Destroy the tcache; recreate in tcaches_get() if needed. */ tcache_destroy(tsd, tcache, false); } } @@ -639,7 +720,7 @@ void tcaches_destroy(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_t *elm = &tcaches[ind]; - tcache_t *tcache = tcaches_elm_remove(tsd, elm); + tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); elm->next = tcaches_avail; tcaches_avail = elm; malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); @@ -652,8 +733,8 @@ bool tcache_boot(tsdn_t *tsdn) { /* If necessary, clamp opt_lg_tcache_max. */ if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < - SMALL_MAXCLASS) { - tcache_maxclass = SMALL_MAXCLASS; + SC_SMALL_MAXCLASS) { + tcache_maxclass = SC_SMALL_MAXCLASS; } else { tcache_maxclass = (ZU(1) << opt_lg_tcache_max); } @@ -673,7 +754,7 @@ tcache_boot(tsdn_t *tsdn) { } stack_nelms = 0; unsigned i; - for (i = 0; i < NBINS; i++) { + for (i = 0; i < SC_NBINS; i++) { if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_SMALL_MIN; diff --git a/dep/jemalloc/src/hooks.c b/dep/jemalloc/src/test_hooks.c index 6266ecd47fe..ace00d9c469 100644 --- a/dep/jemalloc/src/hooks.c +++ b/dep/jemalloc/src/test_hooks.c @@ -6,7 +6,7 @@ * from outside the generated library, so that we can use them in test code. */ JEMALLOC_EXPORT -void (*hooks_arena_new_hook)() = NULL; +void (*test_hooks_arena_new_hook)() = NULL; JEMALLOC_EXPORT -void (*hooks_libc_hook)() = NULL; +void (*test_hooks_libc_hook)() = NULL; diff --git a/dep/jemalloc/src/tsd.c b/dep/jemalloc/src/tsd.c index c1430682dd5..d5fb4d6f82f 100644 --- a/dep/jemalloc/src/tsd.c +++ b/dep/jemalloc/src/tsd.c @@ -12,6 +12,10 @@ static unsigned ncleanups; static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; +/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP __thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; __thread bool JEMALLOC_TLS_MODEL tsd_initialized = false; @@ -41,6 +45,7 @@ tsd_init_head_t tsd_init_head = { ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER }; + tsd_wrapper_t tsd_boot_wrapper = { false, TSD_INITIALIZER @@ -48,17 +53,164 @@ tsd_wrapper_t tsd_boot_wrapper = { bool tsd_booted = false; #endif +JEMALLOC_DIAGNOSTIC_POP + +/******************************************************************************/ + +/* A list of all the tsds in the nominal state. */ +typedef ql_head(tsd_t) tsd_list_t; +static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); +static malloc_mutex_t tsd_nominal_tsds_lock; + +/* How many slow-path-enabling features are turned on. */ +static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0); + +static bool +tsd_in_nominal_list(tsd_t *tsd) { + tsd_t *tsd_list; + bool found = false; + /* + * We don't know that tsd is nominal; it might not be safe to get data + * out of it here. + */ + malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); + ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + if (tsd == tsd_list) { + found = true; + break; + } + } + malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock); + return found; +} + +static void +tsd_add_nominal(tsd_t *tsd) { + assert(!tsd_in_nominal_list(tsd)); + assert(tsd_state_get(tsd) <= tsd_state_nominal_max); + ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +static void +tsd_remove_nominal(tsd_t *tsd) { + assert(tsd_in_nominal_list(tsd)); + assert(tsd_state_get(tsd) <= tsd_state_nominal_max); + malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +static void +tsd_force_recompute(tsdn_t *tsdn) { + /* + * The stores to tsd->state here need to synchronize with the exchange + * in tsd_slow_update. + */ + atomic_fence(ATOMIC_RELEASE); + malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); + tsd_t *remote_tsd; + ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) + <= tsd_state_nominal_max); + tsd_atomic_store(&remote_tsd->state, tsd_state_nominal_recompute, + ATOMIC_RELAXED); + } + malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); +} + +void +tsd_global_slow_inc(tsdn_t *tsdn) { + atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); + /* + * We unconditionally force a recompute, even if the global slow count + * was already positive. If we didn't, then it would be possible for us + * to return to the user, have the user synchronize externally with some + * other thread, and then have that other thread not have picked up the + * update yet (since the original incrementing thread might still be + * making its way through the tsd list). + */ + tsd_force_recompute(tsdn); +} + +void tsd_global_slow_dec(tsdn_t *tsdn) { + atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); + /* See the note in ..._inc(). */ + tsd_force_recompute(tsdn); +} + +static bool +tsd_local_slow(tsd_t *tsd) { + return !tsd_tcache_enabled_get(tsd) + || tsd_reentrancy_level_get(tsd) > 0; +} + +bool +tsd_global_slow() { + return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0; +} /******************************************************************************/ +static uint8_t +tsd_state_compute(tsd_t *tsd) { + if (!tsd_nominal(tsd)) { + return tsd_state_get(tsd); + } + /* We're in *a* nominal state; but which one? */ + if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) { + return tsd_state_nominal_slow; + } else { + return tsd_state_nominal; + } +} + void tsd_slow_update(tsd_t *tsd) { - if (tsd_nominal(tsd)) { - if (malloc_slow || !tsd_tcache_enabled_get(tsd) || - tsd_reentrancy_level_get(tsd) > 0) { - tsd->state = tsd_state_nominal_slow; + uint8_t old_state; + do { + uint8_t new_state = tsd_state_compute(tsd); + old_state = tsd_atomic_exchange(&tsd->state, new_state, + ATOMIC_ACQUIRE); + } while (old_state == tsd_state_nominal_recompute); +} + +void +tsd_state_set(tsd_t *tsd, uint8_t new_state) { + /* Only the tsd module can change the state *to* recompute. */ + assert(new_state != tsd_state_nominal_recompute); + uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED); + if (old_state > tsd_state_nominal_max) { + /* + * Not currently in the nominal list, but it might need to be + * inserted there. + */ + assert(!tsd_in_nominal_list(tsd)); + tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED); + if (new_state <= tsd_state_nominal_max) { + tsd_add_nominal(tsd); + } + } else { + /* + * We're currently nominal. If the new state is non-nominal, + * great; we take ourselves off the list and just enter the new + * state. + */ + assert(tsd_in_nominal_list(tsd)); + if (new_state > tsd_state_nominal_max) { + tsd_remove_nominal(tsd); + tsd_atomic_store(&tsd->state, new_state, + ATOMIC_RELAXED); } else { - tsd->state = tsd_state_nominal; + /* + * This is the tricky case. We're transitioning from + * one nominal state to another. The caller can't know + * about any races that are occuring at the same time, + * so we always have to recompute no matter what. + */ + tsd_slow_update(tsd); } } } @@ -87,6 +239,7 @@ tsd_data_init(tsd_t *tsd) { static void assert_tsd_data_cleanup_done(tsd_t *tsd) { assert(!tsd_nominal(tsd)); + assert(!tsd_in_nominal_list(tsd)); assert(*tsd_arenap_get_unsafe(tsd) == NULL); assert(*tsd_iarenap_get_unsafe(tsd) == NULL); assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); @@ -97,8 +250,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) { static bool tsd_data_init_nocleanup(tsd_t *tsd) { - assert(tsd->state == tsd_state_reincarnated || - tsd->state == tsd_state_minimal_initialized); + assert(tsd_state_get(tsd) == tsd_state_reincarnated || + tsd_state_get(tsd) == tsd_state_minimal_initialized); /* * During reincarnation, there is no guarantee that the cleanup function * will be called (deallocation may happen after all tsd destructors). @@ -117,27 +270,33 @@ tsd_t * tsd_fetch_slow(tsd_t *tsd, bool minimal) { assert(!tsd_fast(tsd)); - if (tsd->state == tsd_state_nominal_slow) { - /* On slow path but no work needed. */ - assert(malloc_slow || !tsd_tcache_enabled_get(tsd) || - tsd_reentrancy_level_get(tsd) > 0 || - *tsd_arenas_tdata_bypassp_get(tsd)); - } else if (tsd->state == tsd_state_uninitialized) { + if (tsd_state_get(tsd) == tsd_state_nominal_slow) { + /* + * On slow path but no work needed. Note that we can't + * necessarily *assert* that we're slow, because we might be + * slow because of an asynchronous modification to global state, + * which might be asynchronously modified *back*. + */ + } else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) { + tsd_slow_update(tsd); + } else if (tsd_state_get(tsd) == tsd_state_uninitialized) { if (!minimal) { - tsd->state = tsd_state_nominal; - tsd_slow_update(tsd); - /* Trigger cleanup handler registration. */ - tsd_set(tsd); - tsd_data_init(tsd); + if (tsd_booted) { + tsd_state_set(tsd, tsd_state_nominal); + tsd_slow_update(tsd); + /* Trigger cleanup handler registration. */ + tsd_set(tsd); + tsd_data_init(tsd); + } } else { - tsd->state = tsd_state_minimal_initialized; + tsd_state_set(tsd, tsd_state_minimal_initialized); tsd_set(tsd); tsd_data_init_nocleanup(tsd); } - } else if (tsd->state == tsd_state_minimal_initialized) { + } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) { if (!minimal) { /* Switch to fully initialized. */ - tsd->state = tsd_state_nominal; + tsd_state_set(tsd, tsd_state_nominal); assert(*tsd_reentrancy_levelp_get(tsd) >= 1); (*tsd_reentrancy_levelp_get(tsd))--; tsd_slow_update(tsd); @@ -145,12 +304,12 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) { } else { assert_tsd_data_cleanup_done(tsd); } - } else if (tsd->state == tsd_state_purgatory) { - tsd->state = tsd_state_reincarnated; + } else if (tsd_state_get(tsd) == tsd_state_purgatory) { + tsd_state_set(tsd, tsd_state_reincarnated); tsd_set(tsd); tsd_data_init_nocleanup(tsd); } else { - assert(tsd->state == tsd_state_reincarnated); + assert(tsd_state_get(tsd) == tsd_state_reincarnated); } return tsd; @@ -214,7 +373,7 @@ void tsd_cleanup(void *arg) { tsd_t *tsd = (tsd_t *)arg; - switch (tsd->state) { + switch (tsd_state_get(tsd)) { case tsd_state_uninitialized: /* Do nothing. */ break; @@ -232,7 +391,7 @@ tsd_cleanup(void *arg) { case tsd_state_nominal: case tsd_state_nominal_slow: tsd_do_data_cleanup(tsd); - tsd->state = tsd_state_purgatory; + tsd_state_set(tsd, tsd_state_purgatory); tsd_set(tsd); break; case tsd_state_purgatory: @@ -260,6 +419,10 @@ malloc_tsd_boot0(void) { tsd_t *tsd; ncleanups = 0; + if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock", + WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { + return NULL; + } if (tsd_boot0()) { return NULL; } @@ -310,7 +473,7 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { # pragma comment(linker, "/INCLUDE:_tls_callback") # else # pragma comment(linker, "/INCLUDE:_tls_used") -# pragma comment(linker, "/INCLUDE:tls_callback") +# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) ) # endif # pragma section(".CRT$XLY",long,read) #endif @@ -349,3 +512,23 @@ tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { malloc_mutex_unlock(TSDN_NULL, &head->lock); } #endif + +void +tsd_prefork(tsd_t *tsd) { + malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +void +tsd_postfork_parent(tsd_t *tsd) { + malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +void +tsd_postfork_child(tsd_t *tsd) { + malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_new(&tsd_nominal_tsds); + + if (tsd_state_get(tsd) <= tsd_state_nominal_max) { + tsd_add_nominal(tsd); + } +} |