1#define JEMALLOC_HUGE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5
6static extent_node_t *
7huge_node_get(const void *ptr)
8{
9 extent_node_t *node;
10
11 node = chunk_lookup(ptr, true);
12 assert(!extent_node_achunk_get(node));
13
14 return (node);
15}
16
17static bool
18huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
19{
20
21 assert(extent_node_addr_get(node) == ptr);
22 assert(!extent_node_achunk_get(node));
23 return (chunk_register(tsdn, ptr, node));
24}
25
26static void
27huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
28{
29 bool err;
30
31 err = huge_node_set(tsdn, ptr, node);
32 assert(!err);
33}
34
35static void
36huge_node_unset(const void *ptr, const extent_node_t *node)
37{
38
39 chunk_deregister(ptr, node);
40}
41
42void *
43huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
44{
45
46 assert(usize == s2u(usize));
47
48 return (huge_palloc(tsdn, arena, usize, chunksize, zero));
49}
50
51void *
52huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
53 bool zero)
54{
55 void *ret;
56 size_t ausize;
57 extent_node_t *node;
58 bool is_zeroed;
59
60 /* Allocate one or more contiguous chunks for this request. */
61
62 assert(!tsdn_null(tsdn) || arena != NULL);
63
64 ausize = sa2u(usize, alignment);
65 if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
66 return (NULL);
67 assert(ausize >= chunksize);
68
69 /* Allocate an extent node with which to track the chunk. */
70 node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
71 CACHELINE, false, NULL, true, arena_ichoose(tsdn, arena));
72 if (node == NULL)
73 return (NULL);
74
75 /*
76 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
77 * it is possible to make correct junk/zero fill decisions below.
78 */
79 is_zeroed = zero;
80 if (likely(!tsdn_null(tsdn)))
81 arena = arena_choose(tsdn_tsd(tsdn), arena);
82 if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
83 arena, usize, alignment, &is_zeroed)) == NULL) {
84 idalloctm(tsdn, node, NULL, true, true);
85 return (NULL);
86 }
87
88 extent_node_init(node, arena, ret, usize, is_zeroed, true);
89
90 if (huge_node_set(tsdn, ret, node)) {
91 arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
92 idalloctm(tsdn, node, NULL, true, true);
93 return (NULL);
94 }
95
96 /* Insert node into huge. */
97 malloc_mutex_lock(tsdn, &arena->huge_mtx);
98 ql_elm_new(node, ql_link);
99 ql_tail_insert(&arena->huge, node, ql_link);
100 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
101
102 if (zero || (config_fill && unlikely(opt_zero))) {
103 if (!is_zeroed)
104 memset(ret, 0, usize);
105 } else if (config_fill && unlikely(opt_junk_alloc))
106 memset(ret, JEMALLOC_ALLOC_JUNK, usize);
107
108 arena_decay_tick(tsdn, arena);
109 return (ret);
110}
111
112#ifdef JEMALLOC_JET
113#undef huge_dalloc_junk
114#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
115#endif
116static void
117huge_dalloc_junk(tsdn_t *tsdn, void *ptr, size_t usize)
118{
119
120 if (config_fill && have_dss && unlikely(opt_junk_free)) {
121 /*
122 * Only bother junk filling if the chunk isn't about to be
123 * unmapped.
124 */
125 if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
126 memset(ptr, JEMALLOC_FREE_JUNK, usize);
127 }
128}
129#ifdef JEMALLOC_JET
130#undef huge_dalloc_junk
131#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk)
132huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
133#endif
134
135static void
136huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
137 size_t usize_min, size_t usize_max, bool zero)
138{
139 size_t usize, usize_next;
140 extent_node_t *node;
141 arena_t *arena;
142 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
143 bool pre_zeroed, post_zeroed;
144
145 /* Increase usize to incorporate extra. */
146 for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
147 <= oldsize; usize = usize_next)
148 ; /* Do nothing. */
149
150 if (oldsize == usize)
151 return;
152
153 node = huge_node_get(ptr);
154 arena = extent_node_arena_get(node);
155 pre_zeroed = extent_node_zeroed_get(node);
156
157 /* Fill if necessary (shrinking). */
158 if (oldsize > usize) {
159 size_t sdiff = oldsize - usize;
160 if (config_fill && unlikely(opt_junk_free)) {
161 memset((void *)((uintptr_t)ptr + usize),
162 JEMALLOC_FREE_JUNK, sdiff);
163 post_zeroed = false;
164 } else {
165 post_zeroed = !chunk_purge_wrapper(tsdn, arena,
166 &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
167 sdiff);
168 }
169 } else
170 post_zeroed = pre_zeroed;
171
172 malloc_mutex_lock(tsdn, &arena->huge_mtx);
173 /* Update the size of the huge allocation. */
174 huge_node_unset(ptr, node);
175 assert(extent_node_size_get(node) != usize);
176 extent_node_size_set(node, usize);
177 huge_node_reset(tsdn, ptr, node);
178 /* Update zeroed. */
179 extent_node_zeroed_set(node, post_zeroed);
180 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
181
182 arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
183
184 /* Fill if necessary (growing). */
185 if (oldsize < usize) {
186 if (zero || (config_fill && unlikely(opt_zero))) {
187 if (!pre_zeroed) {
188 memset((void *)((uintptr_t)ptr + oldsize), 0,
189 usize - oldsize);
190 }
191 } else if (config_fill && unlikely(opt_junk_alloc)) {
192 memset((void *)((uintptr_t)ptr + oldsize),
193 JEMALLOC_ALLOC_JUNK, usize - oldsize);
194 }
195 }
196}
197
198static bool
199huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
200 size_t usize)
201{
202 extent_node_t *node;
203 arena_t *arena;
204 chunk_hooks_t chunk_hooks;
205 size_t cdiff;
206 bool pre_zeroed, post_zeroed;
207
208 node = huge_node_get(ptr);
209 arena = extent_node_arena_get(node);
210 pre_zeroed = extent_node_zeroed_get(node);
211 chunk_hooks = chunk_hooks_get(tsdn, arena);
212
213 assert(oldsize > usize);
214
215 /* Split excess chunks. */
216 cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
217 if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
218 CHUNK_CEILING(usize), cdiff, true, arena->ind))
219 return (true);
220
221 if (oldsize > usize) {
222 size_t sdiff = oldsize - usize;
223 if (config_fill && unlikely(opt_junk_free)) {
224 huge_dalloc_junk(tsdn, (void *)((uintptr_t)ptr + usize),
225 sdiff);
226 post_zeroed = false;
227 } else {
228 post_zeroed = !chunk_purge_wrapper(tsdn, arena,
229 &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
230 usize), CHUNK_CEILING(oldsize),
231 CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
232 }
233 } else
234 post_zeroed = pre_zeroed;
235
236 malloc_mutex_lock(tsdn, &arena->huge_mtx);
237 /* Update the size of the huge allocation. */
238 huge_node_unset(ptr, node);
239 extent_node_size_set(node, usize);
240 huge_node_reset(tsdn, ptr, node);
241 /* Update zeroed. */
242 extent_node_zeroed_set(node, post_zeroed);
243 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
244
245 /* Zap the excess chunks. */
246 arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
247
248 return (false);
249}
250
251static bool
252huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
253 size_t usize, bool zero) {
254 extent_node_t *node;
255 arena_t *arena;
256 bool is_zeroed_subchunk, is_zeroed_chunk;
257
258 node = huge_node_get(ptr);
259 arena = extent_node_arena_get(node);
260 malloc_mutex_lock(tsdn, &arena->huge_mtx);
261 is_zeroed_subchunk = extent_node_zeroed_get(node);
262 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
263
264 /*
265 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
266 * update extent's zeroed field, and zero as necessary.
267 */
268 is_zeroed_chunk = false;
269 if (arena_chunk_ralloc_huge_expand(tsdn, arena, ptr, oldsize, usize,
270 &is_zeroed_chunk))
271 return (true);
272
273 malloc_mutex_lock(tsdn, &arena->huge_mtx);
274 huge_node_unset(ptr, node);
275 extent_node_size_set(node, usize);
276 extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
277 is_zeroed_chunk);
278 huge_node_reset(tsdn, ptr, node);
279 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
280
281 if (zero || (config_fill && unlikely(opt_zero))) {
282 if (!is_zeroed_subchunk) {
283 memset((void *)((uintptr_t)ptr + oldsize), 0,
284 CHUNK_CEILING(oldsize) - oldsize);
285 }
286 if (!is_zeroed_chunk) {
287 memset((void *)((uintptr_t)ptr +
288 CHUNK_CEILING(oldsize)), 0, usize -
289 CHUNK_CEILING(oldsize));
290 }
291 } else if (config_fill && unlikely(opt_junk_alloc)) {
292 memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
293 usize - oldsize);
294 }
295
296 return (false);
297}
298
299bool
300huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
301 size_t usize_max, bool zero)
302{
303
304 assert(s2u(oldsize) == oldsize);
305 /* The following should have been caught by callers. */
306 assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
307
308 /* Both allocations must be huge to avoid a move. */
309 if (oldsize < chunksize || usize_max < chunksize)
310 return (true);
311
312 if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
313 /* Attempt to expand the allocation in-place. */
314 if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
315 zero)) {
316 arena_decay_tick(tsdn, huge_aalloc(ptr));
317 return (false);
318 }
319 /* Try again, this time with usize_min. */
320 if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
321 CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
322 ptr, oldsize, usize_min, zero)) {
323 arena_decay_tick(tsdn, huge_aalloc(ptr));
324 return (false);
325 }
326 }
327
328 /*
329 * Avoid moving the allocation if the existing chunk size accommodates
330 * the new size.
331 */
332 if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
333 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
334 huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
335 usize_max, zero);
336 arena_decay_tick(tsdn, huge_aalloc(ptr));
337 return (false);
338 }
339
340 /* Attempt to shrink the allocation in-place. */
341 if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
342 if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
343 usize_max)) {
344 arena_decay_tick(tsdn, huge_aalloc(ptr));
345 return (false);
346 }
347 }
348 return (true);
349}
350
351static void *
352huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
353 size_t alignment, bool zero)
354{
355
356 if (alignment <= chunksize)
357 return (huge_malloc(tsdn, arena, usize, zero));
358 return (huge_palloc(tsdn, arena, usize, alignment, zero));
359}
360
361void *
362huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
363 size_t usize, size_t alignment, bool zero, tcache_t *tcache)
364{
365 void *ret;
366 size_t copysize;
367
368 /* The following should have been caught by callers. */
369 assert(usize > 0 && usize <= HUGE_MAXCLASS);
370
371 /* Try to avoid moving the allocation. */
372 if (!huge_ralloc_no_move(tsd_tsdn(tsd), ptr, oldsize, usize, usize,
373 zero))
374 return (ptr);
375
376 /*
377 * usize and oldsize are different enough that we need to use a
378 * different size class. In that case, fall back to allocating new
379 * space and copying.
380 */
381 ret = huge_ralloc_move_helper(tsd_tsdn(tsd), arena, usize, alignment,
382 zero);
383 if (ret == NULL)
384 return (NULL);
385
386 copysize = (usize < oldsize) ? usize : oldsize;
387 memcpy(ret, ptr, copysize);
388 isqalloc(tsd, ptr, oldsize, tcache, true);
389 return (ret);
390}
391
392void
393huge_dalloc(tsdn_t *tsdn, void *ptr)
394{
395 extent_node_t *node;
396 arena_t *arena;
397
398 node = huge_node_get(ptr);
399 arena = extent_node_arena_get(node);
400 huge_node_unset(ptr, node);
401 malloc_mutex_lock(tsdn, &arena->huge_mtx);
402 ql_remove(&arena->huge, node, ql_link);
403 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
404
405 huge_dalloc_junk(tsdn, extent_node_addr_get(node),
406 extent_node_size_get(node));
407 arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
408 extent_node_addr_get(node), extent_node_size_get(node));
409 idalloctm(tsdn, node, NULL, true, true);
410
411 arena_decay_tick(tsdn, arena);
412}
413
414arena_t *
415huge_aalloc(const void *ptr)
416{
417
418 return (extent_node_arena_get(huge_node_get(ptr)));
419}
420
421size_t
422huge_salloc(tsdn_t *tsdn, const void *ptr)
423{
424 size_t size;
425 extent_node_t *node;
426 arena_t *arena;
427
428 node = huge_node_get(ptr);
429 arena = extent_node_arena_get(node);
430 malloc_mutex_lock(tsdn, &arena->huge_mtx);
431 size = extent_node_size_get(node);
432 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
433
434 return (size);
435}
436
437prof_tctx_t *
438huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
439{
440 prof_tctx_t *tctx;
441 extent_node_t *node;
442 arena_t *arena;
443
444 node = huge_node_get(ptr);
445 arena = extent_node_arena_get(node);
446 malloc_mutex_lock(tsdn, &arena->huge_mtx);
447 tctx = extent_node_prof_tctx_get(node);
448 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
449
450 return (tctx);
451}
452
453void
454huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
455{
456 extent_node_t *node;
457 arena_t *arena;
458
459 node = huge_node_get(ptr);
460 arena = extent_node_arena_get(node);
461 malloc_mutex_lock(tsdn, &arena->huge_mtx);
462 extent_node_prof_tctx_set(node, tctx);
463 malloc_mutex_unlock(tsdn, &arena->huge_mtx);
464}
465
466void
467huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
468{
469
470 huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
471}
472