yuqei/jemalloc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the --enable-swap configure option.

Browse Source 
Add malloc_swap_enable().

Add the O/o JEMALLOC_OPTIONS flags, which control memory overcommit.

Fix mapped memory stats reporting for arenas.
This commit is contained in:
Jason Evans
2010-01-24 02:53:40 -08:00
parent ed1bf457fb
commit 4201af0542
23 changed files with 1112 additions and 578 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jemalloc/src/internal/jemalloc_base.h
-3
View File
@@ -9,9 +9,6 @@
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
#ifdef JEMALLOC_STATS
extern size_t base_mapped;
#endif
extern malloc_mutex_t base_mtx;
void *base_alloc(size_t size);
jemalloc/src/internal/jemalloc_chunk.h
+9 -16
View File
@@ -27,33 +27,22 @@
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
extern size_t opt_lg_chunk;
#ifdef JEMALLOC_SWAP
extern bool opt_overcommit;
#endif
#ifdef JEMALLOC_STATS
/* Chunk statistics. */
extern chunk_stats_t stats_chunks;
#endif
extern size_t opt_lg_chunk;
extern size_t chunksize;
extern size_t chunksize_mask; /* (chunksize - 1). */
extern size_t chunk_npages;
extern size_t arena_chunk_header_npages;
extern size_t arena_maxclass; /* Max size class for arenas. */
#ifdef JEMALLOC_DSS
/*
* Protects sbrk() calls. This avoids malloc races among threads, though it
* does not protect against races with threads that call sbrk() directly.
*/
extern malloc_mutex_t dss_mtx;
/* Base address of the DSS. */
extern void *dss_base;
/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
extern void *dss_prev;
/* Current upper limit on DSS addresses. */
extern void *dss_max;
#endif
void *pages_map(void *addr, size_t size);
void *chunk_alloc(size_t size, bool zero);
void chunk_dealloc(void *chunk, size_t size);
bool chunk_boot(void);
@@ -64,3 +53,7 @@ bool chunk_boot(void);
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#include "internal/jemalloc_chunk_swap.h"
#include "internal/jemalloc_chunk_dss.h"
#include "internal/jemalloc_chunk_mmap.h"
jemalloc/src/internal/jemalloc_chunk_dss.h
+29
View File
@@ -0,0 +1,29 @@
#ifdef JEMALLOC_DSS
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
/*
* Protects sbrk() calls. This avoids malloc races among threads, though it
* does not protect against races with threads that call sbrk() directly.
*/
extern malloc_mutex_t dss_mtx;
void *chunk_alloc_dss(size_t size, bool zero);
bool chunk_dealloc_dss(void *chunk, size_t size);
bool chunk_dss_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* JEMALLOC_DSS */
jemalloc/src/internal/jemalloc_chunk_mmap.h
+20
View File
@@ -0,0 +1,20 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
void *chunk_alloc_mmap(size_t size);
void chunk_dealloc_mmap(void *chunk, size_t size);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
jemalloc/src/internal/jemalloc_chunk_swap.h
+30
View File
@@ -0,0 +1,30 @@
#ifdef JEMALLOC_SWAP
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
extern malloc_mutex_t swap_mtx;
extern bool swap_enabled;
#ifdef JEMALLOC_STATS
extern size_t swap_avail;
#endif
void *chunk_alloc_swap(size_t size, bool zero);
bool chunk_dealloc_swap(void *chunk, size_t size);
bool chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed);
bool chunk_swap_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#endif /* JEMALLOC_SWAP */
jemalloc/src/internal/jemalloc_extent.h
+2 -2
View File
@@ -9,7 +9,7 @@ typedef struct extent_node_s extent_node_t;
/* Tree of extents. */
struct extent_node_s {
#ifdef JEMALLOC_DSS
#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
/* Linkage for the size/address-ordered tree. */
rb_node(extent_node_t) link_szad;
#endif
@@ -29,7 +29,7 @@ typedef rb_tree(extent_node_t) extent_tree_t;
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
#ifdef JEMALLOC_DSS
#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t)
#endif
jemalloc/src/internal/jemalloc_stats.h
+2
View File
@@ -135,6 +135,8 @@ void malloc_cprintf(void (*write4)(void *, const char *, const char *,
void malloc_printf(const char *format, ...)
JEMALLOC_ATTR(format(printf, 1, 2));
#endif
void stats_print(void (*write4)(void *, const char *, const char *,
const char *, const char *), void *w4opaque, const char *opts);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
jemalloc/src/jemalloc.c
+42
View File
@@ -632,6 +632,14 @@ MALLOC_OUT:
case 'N':
opt_narenas_lshift++;
break;
#ifdef JEMALLOC_SWAP
case 'o':
opt_overcommit = false;
break;
case 'O':
opt_overcommit = true;
break;
#endif
case 'p':
opt_stats_print = false;
break;
@@ -1197,6 +1205,23 @@ JEMALLOC_P(malloc_usable_size)(const void *ptr)
return (ret);
}
#ifdef JEMALLOC_SWAP
JEMALLOC_ATTR(visibility("default"))
int
JEMALLOC_P(malloc_swap_enable)(const int *fds, unsigned nfds, int prezeroed)
{
/*
* Make sure malloc is initialized, because we need page size, chunk
* size, etc.
*/
if (malloc_init())
return (-1);
return (chunk_swap_enable(fds, nfds, (prezeroed != 0)) ? -1 : 0);
}
#endif
#ifdef JEMALLOC_TCACHE
JEMALLOC_ATTR(visibility("default"))
void
@@ -1213,6 +1238,15 @@ JEMALLOC_P(malloc_tcache_flush)(void)
}
#endif
JEMALLOC_ATTR(visibility("default"))
void
JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
const char *, const char *, const char *), void *w4opaque, const char *opts)
{
stats_print(write4, w4opaque, opts);
}
/*
* End non-standard functions.
*/
@@ -1271,6 +1305,10 @@ jemalloc_prefork(void)
#ifdef JEMALLOC_DSS
malloc_mutex_lock(&dss_mtx);
#endif
#ifdef JEMALLOC_SWAP
malloc_mutex_lock(&swap_mtx);
#endif
}
static void
@@ -1281,6 +1319,10 @@ jemalloc_postfork(void)
/* Release all mutexes, now that fork() has completed. */
#ifdef JEMALLOC_SWAP
malloc_mutex_unlock(&swap_mtx);
#endif
#ifdef JEMALLOC_DSS
malloc_mutex_unlock(&dss_mtx);
#endif
jemalloc/src/jemalloc.h.in
+4
View File
@@ -21,6 +21,10 @@ void *JEMALLOC_P(realloc)(void *ptr, size_t size);
void JEMALLOC_P(free)(void *ptr);
size_t JEMALLOC_P(malloc_usable_size)(const void *ptr);
#ifdef JEMALLOC_SWAP
int JEMALLOC_P(malloc_swap_enable)(const int *fds, unsigned nfds,
int prezeroed);
#endif
#ifdef JEMALLOC_TCACHE
void JEMALLOC_P(malloc_tcache_flush)(void);
#endif
jemalloc/src/jemalloc_arena.c
+1
View File
@@ -1570,6 +1570,7 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
*nactive += arena->nactive;
*ndirty += arena->ndirty;
astats->mapped += arena->stats.mapped;
astats->npurge += arena->stats.npurge;
astats->nmadvise += arena->stats.nmadvise;
astats->purged += arena->stats.purged;
jemalloc/src/jemalloc_base.c
+15 -97
View File
@@ -1,9 +1,8 @@
#define JEMALLOC_BASE_C_
#include "internal/jemalloc_internal.h"
#ifdef JEMALLOC_STATS
size_t base_mapped;
#endif
/******************************************************************************/
/* Data. */
malloc_mutex_t base_mtx;
@@ -17,97 +16,27 @@ static void *base_next_addr;
static void *base_past_addr; /* Addr immediately past base_pages. */
static extent_node_t *base_nodes;
#ifdef JEMALLOC_DSS
static bool base_pages_alloc_dss(size_t minsize);
#endif
static bool base_pages_alloc_mmap(size_t minsize);
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static bool base_pages_alloc(size_t minsize);
#ifdef JEMALLOC_DSS
static bool
base_pages_alloc_dss(size_t minsize)
{
/*
* Do special DSS allocation here, since base allocations don't need to
* be chunk-aligned.
*/
malloc_mutex_lock(&dss_mtx);
if (dss_prev != (void *)-1) {
intptr_t incr;
size_t csize = CHUNK_CEILING(minsize);
do {
/* Get the current end of the DSS. */
dss_max = sbrk(0);
/*
* Calculate how much padding is necessary to
* chunk-align the end of the DSS. Don't worry about
* dss_max not being chunk-aligned though.
*/
incr = (intptr_t)chunksize
- (intptr_t)CHUNK_ADDR2OFFSET(dss_max);
assert(incr >= 0);
if ((size_t)incr < minsize)
incr += csize;
dss_prev = sbrk(incr);
if (dss_prev == dss_max) {
/* Success. */
dss_max = (void *)((intptr_t)dss_prev + incr);
base_pages = dss_prev;
base_next_addr = base_pages;
base_past_addr = dss_max;
#ifdef JEMALLOC_STATS
base_mapped += incr;
#endif
malloc_mutex_unlock(&dss_mtx);
return (false);
}
} while (dss_prev != (void *)-1);
}
malloc_mutex_unlock(&dss_mtx);
return (true);
}
#endif
static bool
base_pages_alloc_mmap(size_t minsize)
{
size_t csize;
assert(minsize != 0);
csize = PAGE_CEILING(minsize);
base_pages = pages_map(NULL, csize);
if (base_pages == NULL)
return (true);
base_next_addr = base_pages;
base_past_addr = (void *)((uintptr_t)base_pages + csize);
#ifdef JEMALLOC_STATS
base_mapped += csize;
#endif
return (false);
}
/******************************************************************************/
static bool
base_pages_alloc(size_t minsize)
{
size_t csize;
#ifdef JEMALLOC_DSS
if (base_pages_alloc_dss(minsize) == false)
return (false);
assert(minsize != 0);
csize = CHUNK_CEILING(minsize);
base_pages = chunk_alloc(csize, false);
if (base_pages == NULL)
return (true);
base_next_addr = base_pages;
base_past_addr = (void *)((uintptr_t)base_pages + csize);
if (minsize != 0)
#endif
{
if (base_pages_alloc_mmap(minsize) == false)
return (false);
}
return (true);
return (false);
}
void *
@@ -167,17 +96,6 @@ bool
base_boot(void)
{
#ifdef JEMALLOC_STATS
base_mapped = 0;
#endif
#ifdef JEMALLOC_DSS
/*
* Allocate a base chunk here, since it doesn't actually have to be
* chunk-aligned. Doing this before allocating any other chunks allows
* the use of space that would otherwise be wasted.
*/
base_pages_alloc(0);
#endif
base_nodes = NULL;
if (malloc_mutex_init(&base_mtx))
return (true);
jemalloc/src/jemalloc_chunk.c
+30 -437
View File
@@ -5,6 +5,9 @@
/* Data. */
size_t opt_lg_chunk = LG_CHUNK_DEFAULT;
#ifdef JEMALLOC_SWAP
bool opt_overcommit = true;
#endif
#ifdef JEMALLOC_STATS
chunk_stats_t stats_chunks;
@@ -17,309 +20,7 @@ size_t chunk_npages;
size_t arena_chunk_header_npages;
size_t arena_maxclass; /* Max size class for arenas. */
#ifdef JEMALLOC_DSS
malloc_mutex_t dss_mtx;
void *dss_base;
void *dss_prev;
void *dss_max;
/*
* Trees of chunks that were previously allocated (trees differ only in node
* ordering). These are used when allocating chunks, in an attempt to re-use
* address space. Depending on function, different tree orderings are needed,
* which is why there are two trees with the same contents.
*/
static extent_tree_t dss_chunks_szad;
static extent_tree_t dss_chunks_ad;
#endif
/*
* Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
* potentially avoid some system calls. We can get away without TLS here,
* since the state of mmap_unaligned only affects performance, rather than
* correct function.
*/
static
#ifndef NO_TLS
__thread
#endif
bool mmap_unaligned
#ifndef NO_TLS
JEMALLOC_ATTR(tls_model("initial-exec"))
#endif
;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void pages_unmap(void *addr, size_t size);
#ifdef JEMALLOC_DSS
static void *chunk_alloc_dss(size_t size);
static void *chunk_recycle_dss(size_t size, bool zero);
#endif
static void *chunk_alloc_mmap_slow(size_t size, bool unaligned);
static void *chunk_alloc_mmap(size_t size);
#ifdef JEMALLOC_DSS
static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size);
static bool chunk_dealloc_dss(void *chunk, size_t size);
#endif
static void chunk_dealloc_mmap(void *chunk, size_t size);
/******************************************************************************/
void *
pages_map(void *addr, size_t size)
{
void *ret;
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
-1, 0);
assert(ret != NULL);
if (ret == MAP_FAILED)
ret = NULL;
else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
if (munmap(ret, size) == -1) {
char buf[STRERROR_BUF];
strerror_r(errno, buf, sizeof(buf));
malloc_write4("<jemalloc>", ": Error in munmap(): ",
buf, "\n");
if (opt_abort)
abort();
}
ret = NULL;
}
assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr));
return (ret);
}
static void
pages_unmap(void *addr, size_t size)
{
if (munmap(addr, size) == -1) {
char buf[STRERROR_BUF];
strerror_r(errno, buf, sizeof(buf));
malloc_write4("<jemalloc>", ": Error in munmap(): ", buf, "\n");
if (opt_abort)
abort();
}
}
#ifdef JEMALLOC_DSS
static void *
chunk_alloc_dss(size_t size)
{
/*
* sbrk() uses a signed increment argument, so take care not to
* interpret a huge allocation request as a negative increment.
*/
if ((intptr_t)size < 0)
return (NULL);
malloc_mutex_lock(&dss_mtx);
if (dss_prev != (void *)-1) {
intptr_t incr;
/*
* The loop is necessary to recover from races with other
* threads that are using the DSS for something other than
* malloc.
*/
do {
void *ret;
/* Get the current end of the DSS. */
dss_max = sbrk(0);
/*
* Calculate how much padding is necessary to
* chunk-align the end of the DSS.
*/
incr = (intptr_t)size
- (intptr_t)CHUNK_ADDR2OFFSET(dss_max);
if (incr == (intptr_t)size)
ret = dss_max;
else {
ret = (void *)((intptr_t)dss_max + incr);
incr += size;
}
dss_prev = sbrk(incr);
if (dss_prev == dss_max) {
/* Success. */
dss_max = (void *)((intptr_t)dss_prev + incr);
malloc_mutex_unlock(&dss_mtx);
return (ret);
}
} while (dss_prev != (void *)-1);
}
malloc_mutex_unlock(&dss_mtx);
return (NULL);
}
static void *
chunk_recycle_dss(size_t size, bool zero)
{
extent_node_t *node, key;
key.addr = NULL;
key.size = size;
malloc_mutex_lock(&dss_mtx);
node = extent_tree_szad_nsearch(&dss_chunks_szad, &key);
if (node != NULL) {
void *ret = node->addr;
/* Remove node from the tree. */
extent_tree_szad_remove(&dss_chunks_szad, node);
if (node->size == size) {
extent_tree_ad_remove(&dss_chunks_ad, node);
base_node_dealloc(node);
} else {
/*
* Insert the remainder of node's address range as a
* smaller chunk. Its position within dss_chunks_ad
* does not change.
*/
assert(node->size > size);
node->addr = (void *)((uintptr_t)node->addr + size);
node->size -= size;
extent_tree_szad_insert(&dss_chunks_szad, node);
}
malloc_mutex_unlock(&dss_mtx);
if (zero)
memset(ret, 0, size);
return (ret);
}
malloc_mutex_unlock(&dss_mtx);
return (NULL);
}
#endif
static void *
chunk_alloc_mmap_slow(size_t size, bool unaligned)
{
void *ret;
size_t offset;
/* Beware size_t wrap-around. */
if (size + chunksize <= size)
return (NULL);
ret = pages_map(NULL, size + chunksize);
if (ret == NULL)
return (NULL);
/* Clean up unneeded leading/trailing space. */
offset = CHUNK_ADDR2OFFSET(ret);
if (offset != 0) {
/* Note that mmap() returned an unaligned mapping. */
unaligned = true;
/* Leading space. */
pages_unmap(ret, chunksize - offset);
ret = (void *)((uintptr_t)ret +
(chunksize - offset));
/* Trailing space. */
pages_unmap((void *)((uintptr_t)ret + size),
offset);
} else {
/* Trailing space only. */
pages_unmap((void *)((uintptr_t)ret + size),
chunksize);
}
/*
* If mmap() returned an aligned mapping, reset mmap_unaligned so that
* the next chunk_alloc_mmap() execution tries the fast allocation
* method.
*/
if (unaligned == false)
mmap_unaligned = false;
return (ret);
}
static void *
chunk_alloc_mmap(size_t size)
{
void *ret;
/*
* Ideally, there would be a way to specify alignment to mmap() (like
* NetBSD has), but in the absence of such a feature, we have to work
* hard to efficiently create aligned mappings. The reliable, but
* slow method is to create a mapping that is over-sized, then trim the
* excess. However, that always results in at least one call to
* pages_unmap().
*
* A more optimistic approach is to try mapping precisely the right
* amount, then try to append another mapping if alignment is off. In
* practice, this works out well as long as the application is not
* interleaving mappings via direct mmap() calls. If we do run into a
* situation where there is an interleaved mapping and we are unable to
* extend an unaligned mapping, our best option is to switch to the
* slow method until mmap() returns another aligned mapping. This will
* tend to leave a gap in the memory map that is too small to cause
* later problems for the optimistic method.
*
* Another possible confounding factor is address space layout
* randomization (ASLR), which causes mmap(2) to disregard the
* requested address. mmap_unaligned tracks whether the previous
* chunk_alloc_mmap() execution received any unaligned or relocated
* mappings, and if so, the current execution will immediately fall
* back to the slow method. However, we keep track of whether the fast
* method would have succeeded, and if so, we make a note to try the
* fast method next time.
*/
if (mmap_unaligned == false) {
size_t offset;
ret = pages_map(NULL, size);
if (ret == NULL)
return (NULL);
offset = CHUNK_ADDR2OFFSET(ret);
if (offset != 0) {
mmap_unaligned = true;
/* Try to extend chunk boundary. */
if (pages_map((void *)((uintptr_t)ret + size),
chunksize - offset) == NULL) {
/*
* Extension failed. Clean up, then revert to
* the reliable-but-expensive method.
*/
pages_unmap(ret, size);
ret = chunk_alloc_mmap_slow(size, true);
} else {
/* Clean up unneeded leading space. */
pages_unmap(ret, chunksize - offset);
ret = (void *)((uintptr_t)ret + (chunksize -
offset));
}
}
}
ret = chunk_alloc_mmap_slow(size, false);
return (ret);
}
void *
chunk_alloc(size_t size, bool zero)
@@ -329,20 +30,26 @@ chunk_alloc(size_t size, bool zero)
assert(size != 0);
assert((size & chunksize_mask) == 0);
#ifdef JEMALLOC_DSS
ret = chunk_recycle_dss(size, zero);
if (ret != NULL) {
goto RETURN;
#ifdef JEMALLOC_SWAP
if (swap_enabled) {
ret = chunk_alloc_swap(size, zero);
if (ret != NULL)
goto RETURN;
}
ret = chunk_alloc_dss(size);
if (ret != NULL)
goto RETURN;
if (swap_enabled == false || opt_overcommit) {
#endif
#ifdef JEMALLOC_DSS
ret = chunk_alloc_dss(size, zero);
if (ret != NULL)
goto RETURN;
#endif
ret = chunk_alloc_mmap(size);
if (ret != NULL)
goto RETURN;
#ifdef JEMALLOC_SWAP
}
#endif
ret = chunk_alloc_mmap(size);
if (ret != NULL)
goto RETURN;
/* All strategies for allocation failed. */
ret = NULL;
@@ -360,122 +67,6 @@ RETURN:
return (ret);
}
#ifdef JEMALLOC_DSS
static extent_node_t *
chunk_dealloc_dss_record(void *chunk, size_t size)
{
extent_node_t *node, *prev, key;
key.addr = (void *)((uintptr_t)chunk + size);
node = extent_tree_ad_nsearch(&dss_chunks_ad, &key);
/* Try to coalesce forward. */
if (node != NULL && node->addr == key.addr) {
/*
* Coalesce chunk with the following address range. This does
* not change the position within dss_chunks_ad, so only
* remove/insert from/into dss_chunks_szad.
*/
extent_tree_szad_remove(&dss_chunks_szad, node);
node->addr = chunk;
node->size += size;
extent_tree_szad_insert(&dss_chunks_szad, node);
} else {
/*
* Coalescing forward failed, so insert a new node. Drop
* dss_mtx during node allocation, since it is possible that a
* new base chunk will be allocated.
*/
malloc_mutex_unlock(&dss_mtx);
node = base_node_alloc();
malloc_mutex_lock(&dss_mtx);
if (node == NULL)
return (NULL);
node->addr = chunk;
node->size = size;
extent_tree_ad_insert(&dss_chunks_ad, node);
extent_tree_szad_insert(&dss_chunks_szad, node);
}
/* Try to coalesce backward. */
prev = extent_tree_ad_prev(&dss_chunks_ad, node);
if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
chunk) {
/*
* Coalesce chunk with the previous address range. This does
* not change the position within dss_chunks_ad, so only
* remove/insert node from/into dss_chunks_szad.
*/
extent_tree_szad_remove(&dss_chunks_szad, prev);
extent_tree_ad_remove(&dss_chunks_ad, prev);
extent_tree_szad_remove(&dss_chunks_szad, node);
node->addr = prev->addr;
node->size += prev->size;
extent_tree_szad_insert(&dss_chunks_szad, node);
base_node_dealloc(prev);
}
return (node);
}
static bool
chunk_dealloc_dss(void *chunk, size_t size)
{
malloc_mutex_lock(&dss_mtx);
if ((uintptr_t)chunk >= (uintptr_t)dss_base
&& (uintptr_t)chunk < (uintptr_t)dss_max) {
extent_node_t *node;
/* Try to coalesce with other unused chunks. */
node = chunk_dealloc_dss_record(chunk, size);
if (node != NULL) {
chunk = node->addr;
size = node->size;
}
/* Get the current end of the DSS. */
dss_max = sbrk(0);
/*
* Try to shrink the DSS if this chunk is at the end of the
* DSS. The sbrk() call here is subject to a race condition
* with threads that use brk(2) or sbrk(2) directly, but the
* alternative would be to leak memory for the sake of poorly
* designed multi-threaded programs.
*/
if ((void *)((uintptr_t)chunk + size) == dss_max
&& (dss_prev = sbrk(-(intptr_t)size)) == dss_max) {
/* Success. */
dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size);
if (node != NULL) {
extent_tree_szad_remove(&dss_chunks_szad, node);
extent_tree_ad_remove(&dss_chunks_ad, node);
base_node_dealloc(node);
}
malloc_mutex_unlock(&dss_mtx);
} else {
malloc_mutex_unlock(&dss_mtx);
madvise(chunk, size, MADV_DONTNEED);
}
return (false);
}
malloc_mutex_unlock(&dss_mtx);
return (true);
}
#endif
static void
chunk_dealloc_mmap(void *chunk, size_t size)
{
pages_unmap(chunk, size);
}
void
chunk_dealloc(void *chunk, size_t size)
{
@@ -489,10 +80,13 @@ chunk_dealloc(void *chunk, size_t size)
stats_chunks.curchunks -= (size / chunksize);
#endif
#ifdef JEMALLOC_SWAP
if (swap_enabled && chunk_dealloc_swap(chunk, size) == false)
return;
#endif
#ifdef JEMALLOC_DSS
if (chunk_dealloc_dss(chunk, size) == false)
return;
#endif
chunk_dealloc_mmap(chunk, size);
}
@@ -511,14 +105,13 @@ chunk_boot(void)
memset(&stats_chunks, 0, sizeof(chunk_stats_t));
#endif
#ifdef JEMALLOC_DSS
if (malloc_mutex_init(&dss_mtx))
#ifdef JEMALLOC_SWAP
if (chunk_swap_boot())
return (true);
#endif
#ifdef JEMALLOC_DSS
if (chunk_dss_boot())
return (true);
dss_base = sbrk(0);
dss_prev = dss_base;
dss_max = dss_base;
extent_tree_szad_new(&dss_chunks_szad);
extent_tree_ad_new(&dss_chunks_ad);
#endif
return (false);
jemalloc/src/jemalloc_chunk_dss.c
+267
View File
@@ -0,0 +1,267 @@
#define JEMALLOC_CHUNK_DSS_C_
#include "internal/jemalloc_internal.h"
#ifdef JEMALLOC_DSS
/******************************************************************************/
/* Data. */
malloc_mutex_t dss_mtx;
/* Base address of the DSS. */
static void *dss_base;
/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
static void *dss_prev;
/* Current upper limit on DSS addresses. */
static void *dss_max;
/*
* Trees of chunks that were previously allocated (trees differ only in node
* ordering). These are used when allocating chunks, in an attempt to re-use
* address space. Depending on function, different tree orderings are needed,
* which is why there are two trees with the same contents.
*/
static extent_tree_t dss_chunks_szad;
static extent_tree_t dss_chunks_ad;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void *chunk_recycle_dss(size_t size, bool zero);
static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size);
/******************************************************************************/
static void *
chunk_recycle_dss(size_t size, bool zero)
{
extent_node_t *node, key;
key.addr = NULL;
key.size = size;
malloc_mutex_lock(&dss_mtx);
node = extent_tree_szad_nsearch(&dss_chunks_szad, &key);
if (node != NULL) {
void *ret = node->addr;
/* Remove node from the tree. */
extent_tree_szad_remove(&dss_chunks_szad, node);
if (node->size == size) {
extent_tree_ad_remove(&dss_chunks_ad, node);
base_node_dealloc(node);
} else {
/*
* Insert the remainder of node's address range as a
* smaller chunk. Its position within dss_chunks_ad
* does not change.
*/
assert(node->size > size);
node->addr = (void *)((uintptr_t)node->addr + size);
node->size -= size;
extent_tree_szad_insert(&dss_chunks_szad, node);
}
malloc_mutex_unlock(&dss_mtx);
if (zero)
memset(ret, 0, size);
return (ret);
}
malloc_mutex_unlock(&dss_mtx);
return (NULL);
}
void *
chunk_alloc_dss(size_t size, bool zero)
{
void *ret;
ret = chunk_recycle_dss(size, zero);
if (ret != NULL)
return (ret);
/*
* sbrk() uses a signed increment argument, so take care not to
* interpret a huge allocation request as a negative increment.
*/
if ((intptr_t)size < 0)
return (NULL);
malloc_mutex_lock(&dss_mtx);
if (dss_prev != (void *)-1) {
intptr_t incr;
/*
* The loop is necessary to recover from races with other
* threads that are using the DSS for something other than
* malloc.
*/
do {
/* Get the current end of the DSS. */
dss_max = sbrk(0);
/*
* Calculate how much padding is necessary to
* chunk-align the end of the DSS.
*/
incr = (intptr_t)size
- (intptr_t)CHUNK_ADDR2OFFSET(dss_max);
if (incr == (intptr_t)size)
ret = dss_max;
else {
ret = (void *)((intptr_t)dss_max + incr);
incr += size;
}
dss_prev = sbrk(incr);
if (dss_prev == dss_max) {
/* Success. */
dss_max = (void *)((intptr_t)dss_prev + incr);
malloc_mutex_unlock(&dss_mtx);
return (ret);
}
} while (dss_prev != (void *)-1);
}
malloc_mutex_unlock(&dss_mtx);
return (NULL);
}
static extent_node_t *
chunk_dealloc_dss_record(void *chunk, size_t size)
{
extent_node_t *xnode, *node, *prev, key;
xnode = NULL;
while (true) {
key.addr = (void *)((uintptr_t)chunk + size);
node = extent_tree_ad_nsearch(&dss_chunks_ad, &key);
/* Try to coalesce forward. */
if (node != NULL && node->addr == key.addr) {
/*
* Coalesce chunk with the following address range.
* This does not change the position within
* dss_chunks_ad, so only remove/insert from/into
* dss_chunks_szad.
*/
extent_tree_szad_remove(&dss_chunks_szad, node);
node->addr = chunk;
node->size += size;
extent_tree_szad_insert(&dss_chunks_szad, node);
break;
} else if (xnode == NULL) {
/*
* It is possible that base_node_alloc() will cause a
* new base chunk to be allocated, so take care not to
* deadlock on dss_mtx, and recover if another thread
* deallocates an adjacent chunk while this one is busy
* allocating xnode.
*/
malloc_mutex_unlock(&dss_mtx);
xnode = base_node_alloc();
malloc_mutex_lock(&dss_mtx);
if (xnode == NULL)
return (NULL);
} else {
/* Coalescing forward failed, so insert a new node. */
node = xnode;
xnode = NULL;
node->addr = chunk;
node->size = size;
extent_tree_ad_insert(&dss_chunks_ad, node);
extent_tree_szad_insert(&dss_chunks_szad, node);
break;
}
}
/* Discard xnode if it ended up unused do to a race. */
if (xnode != NULL)
base_node_dealloc(xnode);
/* Try to coalesce backward. */
prev = extent_tree_ad_prev(&dss_chunks_ad, node);
if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
chunk) {
/*
* Coalesce chunk with the previous address range. This does
* not change the position within dss_chunks_ad, so only
* remove/insert node from/into dss_chunks_szad.
*/
extent_tree_szad_remove(&dss_chunks_szad, prev);
extent_tree_ad_remove(&dss_chunks_ad, prev);
extent_tree_szad_remove(&dss_chunks_szad, node);
node->addr = prev->addr;
node->size += prev->size;
extent_tree_szad_insert(&dss_chunks_szad, node);
base_node_dealloc(prev);
}
return (node);
}
bool
chunk_dealloc_dss(void *chunk, size_t size)
{
bool ret;
malloc_mutex_lock(&dss_mtx);
if ((uintptr_t)chunk >= (uintptr_t)dss_base
&& (uintptr_t)chunk < (uintptr_t)dss_max) {
extent_node_t *node;
/* Try to coalesce with other unused chunks. */
node = chunk_dealloc_dss_record(chunk, size);
if (node != NULL) {
chunk = node->addr;
size = node->size;
}
/* Get the current end of the DSS. */
dss_max = sbrk(0);
/*
* Try to shrink the DSS if this chunk is at the end of the
* DSS. The sbrk() call here is subject to a race condition
* with threads that use brk(2) or sbrk(2) directly, but the
* alternative would be to leak memory for the sake of poorly
* designed multi-threaded programs.
*/
if ((void *)((uintptr_t)chunk + size) == dss_max
&& (dss_prev = sbrk(-(intptr_t)size)) == dss_max) {
/* Success. */
dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size);
if (node != NULL) {
extent_tree_szad_remove(&dss_chunks_szad, node);
extent_tree_ad_remove(&dss_chunks_ad, node);
base_node_dealloc(node);
}
} else
madvise(chunk, size, MADV_DONTNEED);
ret = false;
goto RETURN;
}
ret = true
RETURN:
malloc_mutex_unlock(&dss_mtx);
return (ret);
}
bool
chunk_dss_boot(void)
{
if (malloc_mutex_init(&dss_mtx))
return (true);
dss_base = sbrk(0);
dss_prev = dss_base;
dss_max = dss_base;
extent_tree_szad_new(&dss_chunks_szad);
extent_tree_ad_new(&dss_chunks_ad);
return (false);
}
/******************************************************************************/
#endif /* JEMALLOC_DSS */
jemalloc/src/jemalloc_chunk_mmap.c
+198
View File
@@ -0,0 +1,198 @@
#define JEMALLOC_CHUNK_MMAP_C_
#include "internal/jemalloc_internal.h"
/******************************************************************************/
/* Data. */
/*
* Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
* potentially avoid some system calls. We can get away without TLS here,
* since the state of mmap_unaligned only affects performance, rather than
* correct function.
*/
static
#ifndef NO_TLS
__thread
#endif
bool mmap_unaligned
#ifndef NO_TLS
JEMALLOC_ATTR(tls_model("initial-exec"))
#endif
;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void *pages_map(void *addr, size_t size);
static void pages_unmap(void *addr, size_t size);
static void *chunk_alloc_mmap_slow(size_t size, bool unaligned);
/******************************************************************************/
static void *
pages_map(void *addr, size_t size)
{
void *ret;
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
-1, 0);
assert(ret != NULL);
if (ret == MAP_FAILED)
ret = NULL;
else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
if (munmap(ret, size) == -1) {
char buf[STRERROR_BUF];
strerror_r(errno, buf, sizeof(buf));
malloc_write4("<jemalloc>", ": Error in munmap(): ",
buf, "\n");
if (opt_abort)
abort();
}
ret = NULL;
}
assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr));
return (ret);
}
static void
pages_unmap(void *addr, size_t size)
{
if (munmap(addr, size) == -1) {
char buf[STRERROR_BUF];
strerror_r(errno, buf, sizeof(buf));
malloc_write4("<jemalloc>", ": Error in munmap(): ", buf, "\n");
if (opt_abort)
abort();
}
}
static void *
chunk_alloc_mmap_slow(size_t size, bool unaligned)
{
void *ret;
size_t offset;
/* Beware size_t wrap-around. */
if (size + chunksize <= size)
return (NULL);
ret = pages_map(NULL, size + chunksize);
if (ret == NULL)
return (NULL);
/* Clean up unneeded leading/trailing space. */
offset = CHUNK_ADDR2OFFSET(ret);
if (offset != 0) {
/* Note that mmap() returned an unaligned mapping. */
unaligned = true;
/* Leading space. */
pages_unmap(ret, chunksize - offset);
ret = (void *)((uintptr_t)ret +
(chunksize - offset));
/* Trailing space. */
pages_unmap((void *)((uintptr_t)ret + size),
offset);
} else {
/* Trailing space only. */
pages_unmap((void *)((uintptr_t)ret + size),
chunksize);
}
/*
* If mmap() returned an aligned mapping, reset mmap_unaligned so that
* the next chunk_alloc_mmap() execution tries the fast allocation
* method.
*/
if (unaligned == false)
mmap_unaligned = false;
return (ret);
}
void *
chunk_alloc_mmap(size_t size)
{
void *ret;
/*
* Ideally, there would be a way to specify alignment to mmap() (like
* NetBSD has), but in the absence of such a feature, we have to work
* hard to efficiently create aligned mappings. The reliable, but
* slow method is to create a mapping that is over-sized, then trim the
* excess. However, that always results in at least one call to
* pages_unmap().
*
* A more optimistic approach is to try mapping precisely the right
* amount, then try to append another mapping if alignment is off. In
* practice, this works out well as long as the application is not
* interleaving mappings via direct mmap() calls. If we do run into a
* situation where there is an interleaved mapping and we are unable to
* extend an unaligned mapping, our best option is to switch to the
* slow method until mmap() returns another aligned mapping. This will
* tend to leave a gap in the memory map that is too small to cause
* later problems for the optimistic method.
*
* Another possible confounding factor is address space layout
* randomization (ASLR), which causes mmap(2) to disregard the
* requested address. mmap_unaligned tracks whether the previous
* chunk_alloc_mmap() execution received any unaligned or relocated
* mappings, and if so, the current execution will immediately fall
* back to the slow method. However, we keep track of whether the fast
* method would have succeeded, and if so, we make a note to try the
* fast method next time.
*/
if (mmap_unaligned == false) {
size_t offset;
ret = pages_map(NULL, size);
if (ret == NULL)
return (NULL);
offset = CHUNK_ADDR2OFFSET(ret);
if (offset != 0) {
mmap_unaligned = true;
/* Try to extend chunk boundary. */
if (pages_map((void *)((uintptr_t)ret + size),
chunksize - offset) == NULL) {
/*
* Extension failed. Clean up, then revert to
* the reliable-but-expensive method.
*/
pages_unmap(ret, size);
ret = chunk_alloc_mmap_slow(size, true);
} else {
/* Clean up unneeded leading space. */
pages_unmap(ret, chunksize - offset);
ret = (void *)((uintptr_t)ret + (chunksize -
offset));
}
}
}
ret = chunk_alloc_mmap_slow(size, false);
return (ret);
}
void
chunk_dealloc_mmap(void *chunk, size_t size)
{
pages_unmap(chunk, size);
}
jemalloc/src/jemalloc_chunk_swap.c
+354
View File
@@ -0,0 +1,354 @@
#define JEMALLOC_CHUNK_SWAP_C_
#include "internal/jemalloc_internal.h"
#ifdef JEMALLOC_SWAP
/******************************************************************************/
/* Data. */
malloc_mutex_t swap_mtx;
bool swap_enabled;
#ifdef JEMALLOC_STATS
size_t swap_avail;
#endif
static bool swap_prezeroed;
/* Base address of the mmap()ed file(s). */
static void *swap_base;
/* Current end of the space in use (<= swap_max). */
static void *swap_end;
/* Absolute upper limit on file-backed addresses. */
static void *swap_max;
/*
* Trees of chunks that were previously allocated (trees differ only in node
* ordering). These are used when allocating chunks, in an attempt to re-use
* address space. Depending on function, different tree orderings are needed,
* which is why there are two trees with the same contents.
*/
static extent_tree_t swap_chunks_szad;
static extent_tree_t swap_chunks_ad;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static void *chunk_recycle_swap(size_t size, bool zero);
static extent_node_t *chunk_dealloc_swap_record(void *chunk, size_t size);
/******************************************************************************/
static void *
chunk_recycle_swap(size_t size, bool zero)
{
extent_node_t *node, key;
key.addr = NULL;
key.size = size;
malloc_mutex_lock(&swap_mtx);
node = extent_tree_szad_nsearch(&swap_chunks_szad, &key);
if (node != NULL) {
void *ret = node->addr;
/* Remove node from the tree. */
extent_tree_szad_remove(&swap_chunks_szad, node);
if (node->size == size) {
extent_tree_ad_remove(&swap_chunks_ad, node);
base_node_dealloc(node);
} else {
/*
* Insert the remainder of node's address range as a
* smaller chunk. Its position within swap_chunks_ad
* does not change.
*/
assert(node->size > size);
node->addr = (void *)((uintptr_t)node->addr + size);
node->size -= size;
extent_tree_szad_insert(&swap_chunks_szad, node);
}
#ifdef JEMALLOC_STATS
swap_avail -= size;
#endif
malloc_mutex_unlock(&swap_mtx);
if (zero)
memset(ret, 0, size);
return (ret);
}
malloc_mutex_unlock(&swap_mtx);
return (NULL);
}
void *
chunk_alloc_swap(size_t size, bool zero)
{
void *ret;
assert(swap_enabled);
ret = chunk_recycle_swap(size, zero);
if (ret != NULL)
return (ret);
malloc_mutex_lock(&swap_mtx);
if ((uintptr_t)swap_end + size <= (uintptr_t)swap_max) {
ret = swap_end;
swap_end = (void *)((uintptr_t)swap_end + size);
#ifdef JEMALLOC_STATS
swap_avail -= size;
#endif
malloc_mutex_unlock(&swap_mtx);
if (zero && swap_prezeroed == false)
memset(ret, 0, size);
} else {
malloc_mutex_unlock(&swap_mtx);
return (NULL);
}
return (ret);
}
static extent_node_t *
chunk_dealloc_swap_record(void *chunk, size_t size)
{
extent_node_t *xnode, *node, *prev, key;
xnode = NULL;
while (true) {
key.addr = (void *)((uintptr_t)chunk + size);
node = extent_tree_ad_nsearch(&swap_chunks_ad, &key);
/* Try to coalesce forward. */
if (node != NULL && node->addr == key.addr) {
/*
* Coalesce chunk with the following address range.
* This does not change the position within
* swap_chunks_ad, so only remove/insert from/into
* swap_chunks_szad.
*/
extent_tree_szad_remove(&swap_chunks_szad, node);
node->addr = chunk;
node->size += size;
extent_tree_szad_insert(&swap_chunks_szad, node);
break;
} else if (xnode == NULL) {
/*
* It is possible that base_node_alloc() will cause a
* new base chunk to be allocated, so take care not to
* deadlock on swap_mtx, and recover if another thread
* deallocates an adjacent chunk while this one is busy
* allocating xnode.
*/
malloc_mutex_unlock(&swap_mtx);
xnode = base_node_alloc();
malloc_mutex_lock(&swap_mtx);
if (xnode == NULL)
return (NULL);
} else {
/* Coalescing forward failed, so insert a new node. */
node = xnode;
xnode = NULL;
node->addr = chunk;
node->size = size;
extent_tree_ad_insert(&swap_chunks_ad, node);
extent_tree_szad_insert(&swap_chunks_szad, node);
break;
}
}
/* Discard xnode if it ended up unused do to a race. */
if (xnode != NULL)
base_node_dealloc(xnode);
/* Try to coalesce backward. */
prev = extent_tree_ad_prev(&swap_chunks_ad, node);
if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
chunk) {
/*
* Coalesce chunk with the previous address range. This does
* not change the position within swap_chunks_ad, so only
* remove/insert node from/into swap_chunks_szad.
*/
extent_tree_szad_remove(&swap_chunks_szad, prev);
extent_tree_ad_remove(&swap_chunks_ad, prev);
extent_tree_szad_remove(&swap_chunks_szad, node);
node->addr = prev->addr;
node->size += prev->size;
extent_tree_szad_insert(&swap_chunks_szad, node);
base_node_dealloc(prev);
}
return (node);
}
bool
chunk_dealloc_swap(void *chunk, size_t size)
{
bool ret;
assert(swap_enabled);
malloc_mutex_lock(&swap_mtx);
if ((uintptr_t)chunk >= (uintptr_t)swap_base
&& (uintptr_t)chunk < (uintptr_t)swap_max) {
extent_node_t *node;
/* Try to coalesce with other unused chunks. */
node = chunk_dealloc_swap_record(chunk, size);
if (node != NULL) {
chunk = node->addr;
size = node->size;
}
/*
* Try to shrink the in-use memory if this chunk is at the end
* of the in-use memory.
*/
if ((void *)((uintptr_t)chunk + size) == swap_end) {
swap_end = (void *)((uintptr_t)swap_end - size);
if (node != NULL) {
extent_tree_szad_remove(&swap_chunks_szad,
node);
extent_tree_ad_remove(&swap_chunks_ad, node);
base_node_dealloc(node);
}
} else
madvise(chunk, size, MADV_DONTNEED);
ret = false;
goto RETURN;
}
ret = true;
RETURN:
#ifdef JEMALLOC_STATS
swap_avail += size;
#endif
malloc_mutex_unlock(&swap_mtx);
return (ret);
}
bool
chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed)
{
bool ret;
unsigned i;
off_t off;
void *vaddr;
size_t cumsize, voff;
size_t sizes[nfds];
malloc_mutex_lock(&swap_mtx);
/* Get file sizes. */
for (i = 0, cumsize = 0; i < nfds; i++) {
off = lseek(fds[i], 0, SEEK_END);
if (off == ((off_t)-1)) {
ret = true;
goto RETURN;
}
if (PAGE_CEILING(off) != off) {
/* Truncate to a multiple of the page size. */
off &= ~PAGE_MASK;
if (ftruncate(fds[i], off) != 0) {
ret = true;
goto RETURN;
}
}
sizes[i] = off;
if (cumsize + off < cumsize) {
/*
* Cumulative file size is greater than the total
* address space. Bail out while it's still obvious
* what the problem is.
*/
ret = true;
goto RETURN;
}
cumsize += off;
}
/* Round down to a multiple of the chunk size. */
cumsize &= ~chunksize_mask;
if (cumsize == 0) {
ret = true;
goto RETURN;
}
/*
* Allocate a chunk-aligned region of anonymous memory, which will
* be the final location for the memory-mapped files.
*/
vaddr = chunk_alloc_mmap(cumsize);
if (vaddr == NULL) {
ret = true;
goto RETURN;
}
/* Overlay the files onto the anonymous mapping. */
for (i = 0, voff = 0; i < nfds; i++) {
void *addr = mmap((void *)((uintptr_t)vaddr + voff), sizes[i],
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fds[i], 0);
if (addr == MAP_FAILED) {
char buf[STRERROR_BUF];
strerror_r(errno, buf, sizeof(buf));
malloc_write4("<jemalloc>",
": Error in mmap(..., MAP_FIXED, ...): ",
buf, "\n");
if (opt_abort)
abort();
if (munmap(vaddr, voff) == -1) {
strerror_r(errno, buf, sizeof(buf));
malloc_write4("<jemalloc>",
": Error in munmap(): ", buf, "\n");
}
ret = true;
goto RETURN;
}
assert(addr == (void *)((uintptr_t)vaddr + voff));
voff += sizes[i];
}
swap_prezeroed = prezeroed;
swap_base = vaddr;
swap_end = swap_base;
swap_max = (void *)((uintptr_t)vaddr + cumsize);
swap_enabled = true;
#ifdef JEMALLOC_STATS
swap_avail = cumsize;
#endif
ret = false;
RETURN:
malloc_mutex_unlock(&swap_mtx);
return (ret);
}
bool
chunk_swap_boot(void)
{
if (malloc_mutex_init(&swap_mtx))
return (true);
swap_enabled = false;
#ifdef JEMALLOC_STATS
swap_avail = 0;
#endif
swap_prezeroed = false;
swap_base = NULL;
swap_end = NULL;
swap_max = NULL;
extent_tree_szad_new(&swap_chunks_szad);
extent_tree_ad_new(&swap_chunks_ad);
return (false);
}
/******************************************************************************/
#endif /* JEMALLOC_SWAP */
jemalloc/src/jemalloc_defs.h.in
+3
View File
@@ -67,6 +67,9 @@
*/
#undef JEMALLOC_DSS
/* JEMALLOC_SWAP enables mmap()ed swap file support. */
#undef JEMALLOC_SWAP
/* Support memory filling (junk/zero). */
#undef JEMALLOC_FILL
jemalloc/src/jemalloc_extent.c
+1 -1
View File
@@ -3,7 +3,7 @@
/******************************************************************************/
#ifdef JEMALLOC_DSS
#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
static inline int
extent_szad_comp(extent_node_t *a, extent_node_t *b)
{
jemalloc/src/jemalloc_huge.c
+1 -1
View File
@@ -208,7 +208,7 @@ huge_dalloc(void *ptr)
/* Unmap chunk. */
#ifdef JEMALLOC_FILL
#ifdef JEMALLOC_DSS
#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS))
if (opt_junk)
memset(node->addr, 0x5a, node->size);
#endif
jemalloc/src/jemalloc_stats.c
+20 -11
View File
@@ -104,13 +104,11 @@ malloc_printf(const char *format, ...)
malloc_vcprintf(NULL, NULL, format, ap);
va_end(ap);
}
#endif
JEMALLOC_ATTR(visibility("default"))
void
JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
const char *, const char *, const char *), void *w4opaque, const char *opts)
stats_print(void (*write4)(void *, const char *, const char *, const char *,
const char *), void *w4opaque, const char *opts)
{
char s[UMAX2S_BUFSIZE];
bool general = true;
@@ -167,6 +165,9 @@ JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
opt_abort ? "A" : "a", "", "");
#ifdef JEMALLOC_FILL
write4(w4opaque, opt_junk ? "J" : "j", "", "", "");
#endif
#ifdef JEMALLOC_SWAP
write4(w4opaque, opt_overcommit ? "O" : "o", "", "", "");
#endif
write4(w4opaque, "P", "", "", "");
#ifdef JEMALLOC_TCACHE
@@ -271,10 +272,6 @@ JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
mapped = stats_chunks.curchunks * chunksize;
malloc_mutex_unlock(&huge_mtx);
malloc_mutex_lock(&base_mtx);
mapped += base_mapped;
malloc_mutex_unlock(&base_mtx);
malloc_cprintf(write4, w4opaque,
"Allocated: %zu, mapped: %zu\n", allocated, mapped);
@@ -287,10 +284,22 @@ JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
malloc_mutex_unlock(&huge_mtx);
malloc_cprintf(write4, w4opaque, "chunks: nchunks "
"highchunks curchunks\n");
malloc_cprintf(write4, w4opaque, " %13llu%13lu%13lu\n",
"highchunks curchunks"
#ifdef JEMALLOC_SWAP
" swap_avail"
#endif
"\n");
malloc_cprintf(write4, w4opaque, " %13llu%13lu%13lu"
#ifdef JEMALLOC_SWAP
"%13zu"
#endif
"\n",
chunks_stats.nchunks, chunks_stats.highchunks,
chunks_stats.curchunks);
chunks_stats.curchunks
#ifdef JEMALLOC_SWAP
, (swap_avail >> opt_lg_chunk)
#endif
);
}
/* Print chunk stats. */