1#define JEMALLOC_BASE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7static malloc_mutex_t base_mtx;
8static extent_tree_t base_avail_szad;
9static extent_node_t *base_nodes;
10static size_t base_allocated;
11static size_t base_resident;
12static size_t base_mapped;
13
14/******************************************************************************/
15
16static extent_node_t *
17base_node_try_alloc(tsdn_t *tsdn)
18{
19 extent_node_t *node;
20
21 malloc_mutex_assert_owner(tsdn, &base_mtx);
22
23 if (base_nodes == NULL)
24 return (NULL);
25 node = base_nodes;
26 base_nodes = *(extent_node_t **)node;
27 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
28 return (node);
29}
30
31static void
32base_node_dalloc(tsdn_t *tsdn, extent_node_t *node)
33{
34
35 malloc_mutex_assert_owner(tsdn, &base_mtx);
36
37 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
38 *(extent_node_t **)node = base_nodes;
39 base_nodes = node;
40}
41
42static extent_node_t *
43base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
44{
45 extent_node_t *node;
46 size_t csize, nsize;
47 void *addr;
48
49 malloc_mutex_assert_owner(tsdn, &base_mtx);
50 assert(minsize != 0);
51 node = base_node_try_alloc(tsdn);
52 /* Allocate enough space to also carve a node out if necessary. */
53 nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0;
54 csize = CHUNK_CEILING(minsize + nsize);
55 addr = chunk_alloc_base(csize);
56 if (addr == NULL) {
57 if (node != NULL)
58 base_node_dalloc(tsdn, node);
59 return (NULL);
60 }
61 base_mapped += csize;
62 if (node == NULL) {
63 node = (extent_node_t *)addr;
64 addr = (void *)((uintptr_t)addr + nsize);
65 csize -= nsize;
66 if (config_stats) {
67 base_allocated += nsize;
68 base_resident += PAGE_CEILING(nsize);
69 }
70 }
71 extent_node_init(node, NULL, addr, csize, true, true);
72 return (node);
73}
74
75/*
76 * base_alloc() guarantees demand-zeroed memory, in order to make multi-page
77 * sparse data structures such as radix tree nodes efficient with respect to
78 * physical memory usage.
79 */
80void *
81base_alloc(tsdn_t *tsdn, size_t size)
82{
83 void *ret;
84 size_t csize, usize;
85 extent_node_t *node;
86 extent_node_t key;
87
88 /*
89 * Round size up to nearest multiple of the cacheline size, so that
90 * there is no chance of false cache line sharing.
91 */
92 csize = CACHELINE_CEILING(size);
93
94 usize = s2u(csize);
95 extent_node_init(&key, NULL, NULL, usize, false, false);
96 malloc_mutex_lock(tsdn, &base_mtx);
97 node = extent_tree_szad_nsearch(&base_avail_szad, &key);
98 if (node != NULL) {
99 /* Use existing space. */
100 extent_tree_szad_remove(&base_avail_szad, node);
101 } else {
102 /* Try to allocate more space. */
103 node = base_chunk_alloc(tsdn, csize);
104 }
105 if (node == NULL) {
106 ret = NULL;
107 goto label_return;
108 }
109
110 ret = extent_node_addr_get(node);
111 if (extent_node_size_get(node) > csize) {
112 extent_node_addr_set(node, (void *)((uintptr_t)ret + csize));
113 extent_node_size_set(node, extent_node_size_get(node) - csize);
114 extent_tree_szad_insert(&base_avail_szad, node);
115 } else
116 base_node_dalloc(tsdn, node);
117 if (config_stats) {
118 base_allocated += csize;
119 /*
120 * Add one PAGE to base_resident for every page boundary that is
121 * crossed by the new allocation.
122 */
123 base_resident += PAGE_CEILING((uintptr_t)ret + csize) -
124 PAGE_CEILING((uintptr_t)ret);
125 }
126 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize);
127label_return:
128 malloc_mutex_unlock(tsdn, &base_mtx);
129 return (ret);
130}
131
132void
133base_stats_get(tsdn_t *tsdn, size_t *allocated, size_t *resident,
134 size_t *mapped)
135{
136
137 malloc_mutex_lock(tsdn, &base_mtx);
138 assert(base_allocated <= base_resident);
139 assert(base_resident <= base_mapped);
140 *allocated = base_allocated;
141 *resident = base_resident;
142 *mapped = base_mapped;
143 malloc_mutex_unlock(tsdn, &base_mtx);
144}
145
146bool
147base_boot(void)
148{
149
150 if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE))
151 return (true);
152 extent_tree_szad_new(&base_avail_szad);
153 base_nodes = NULL;
154
155 return (false);
156}
157
158void
159base_prefork(tsdn_t *tsdn)
160{
161
162 malloc_mutex_prefork(tsdn, &base_mtx);
163}
164
165void
166base_postfork_parent(tsdn_t *tsdn)
167{
168
169 malloc_mutex_postfork_parent(tsdn, &base_mtx);
170}
171
172void
173base_postfork_child(tsdn_t *tsdn)
174{
175
176 malloc_mutex_postfork_child(tsdn, &base_mtx);
177}
178