1/* ----------------------------------------------------------------------------
2Copyright (c) 2018, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#include "mimalloc.h"
8#include "mimalloc-internal.h"
9#include "mimalloc-atomic.h"
10
11#include <stdio.h>
12#include <stdlib.h> // strtol
13#include <string.h> // strncpy, strncat, strlen, strstr
14#include <ctype.h> // toupper
15#include <stdarg.h>
16
17static uintptr_t mi_max_error_count = 16; // stop outputting errors after this
18
19static void mi_add_stderr_output();
20
21int mi_version(void) mi_attr_noexcept {
22 return MI_MALLOC_VERSION;
23}
24
25#ifdef _WIN32
26#include <conio.h>
27#endif
28
29// --------------------------------------------------------
30// Options
31// These can be accessed by multiple threads and may be
32// concurrently initialized, but an initializing data race
33// is ok since they resolve to the same value.
34// --------------------------------------------------------
35typedef enum mi_init_e {
36 UNINIT, // not yet initialized
37 DEFAULTED, // not found in the environment, use default value
38 INITIALIZED // found in environment or set explicitly
39} mi_init_t;
40
41typedef struct mi_option_desc_s {
42 long value; // the value
43 mi_init_t init; // is it initialized yet? (from the environment)
44 mi_option_t option; // for debugging: the option index should match the option
45 const char* name; // option name without `mimalloc_` prefix
46} mi_option_desc_t;
47
48#define MI_OPTION(opt) mi_option_##opt, #opt
49#define MI_OPTION_DESC(opt) {0, UNINIT, MI_OPTION(opt) }
50
51static mi_option_desc_t options[_mi_option_last] =
52{
53 // stable options
54 { MI_DEBUG, UNINIT, MI_OPTION(show_errors) },
55 { 0, UNINIT, MI_OPTION(show_stats) },
56 { 0, UNINIT, MI_OPTION(verbose) },
57
58 // the following options are experimental and not all combinations make sense.
59 { 1, UNINIT, MI_OPTION(eager_commit) }, // note: needs to be on when eager_region_commit is enabled
60 #ifdef _WIN32 // and BSD?
61 { 0, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...)
62 #else
63 { 1, UNINIT, MI_OPTION(eager_region_commit) },
64 #endif
65 { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
66 { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
67 { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread
68 { 0, UNINIT, MI_OPTION(page_reset) },
69 { 0, UNINIT, MI_OPTION(cache_reset) },
70 { 0, UNINIT, MI_OPTION(reset_decommits) }, // note: cannot enable this if secure is on
71 { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
72 { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit)
73 { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
74 { 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output
75};
76
77static void mi_option_init(mi_option_desc_t* desc);
78
79void _mi_options_init(void) {
80 // called on process load; should not be called before the CRT is initialized!
81 // (e.g. do not call this from process_init as that may run before CRT initialization)
82 mi_add_stderr_output(); // now it safe to use stderr for output
83 for(int i = 0; i < _mi_option_last; i++ ) {
84 mi_option_t option = (mi_option_t)i;
85 mi_option_get(option); // initialize
86 if (option != mi_option_verbose) {
87 mi_option_desc_t* desc = &options[option];
88 _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value);
89 }
90 }
91 mi_max_error_count = mi_option_get(mi_option_max_errors);
92}
93
94long mi_option_get(mi_option_t option) {
95 mi_assert(option >= 0 && option < _mi_option_last);
96 mi_option_desc_t* desc = &options[option];
97 mi_assert(desc->option == option); // index should match the option
98 if (mi_unlikely(desc->init == UNINIT)) {
99 mi_option_init(desc);
100 }
101 return desc->value;
102}
103
104void mi_option_set(mi_option_t option, long value) {
105 mi_assert(option >= 0 && option < _mi_option_last);
106 mi_option_desc_t* desc = &options[option];
107 mi_assert(desc->option == option); // index should match the option
108 desc->value = value;
109 desc->init = INITIALIZED;
110}
111
112void mi_option_set_default(mi_option_t option, long value) {
113 mi_assert(option >= 0 && option < _mi_option_last);
114 mi_option_desc_t* desc = &options[option];
115 if (desc->init != INITIALIZED) {
116 desc->value = value;
117 }
118}
119
120bool mi_option_is_enabled(mi_option_t option) {
121 return (mi_option_get(option) != 0);
122}
123
124void mi_option_set_enabled(mi_option_t option, bool enable) {
125 mi_option_set(option, (enable ? 1 : 0));
126}
127
128void mi_option_set_enabled_default(mi_option_t option, bool enable) {
129 mi_option_set_default(option, (enable ? 1 : 0));
130}
131
132void mi_option_enable(mi_option_t option) {
133 mi_option_set_enabled(option,true);
134}
135
136void mi_option_disable(mi_option_t option) {
137 mi_option_set_enabled(option,false);
138}
139
140
141static void mi_out_stderr(const char* msg) {
142 #ifdef _WIN32
143 // on windows with redirection, the C runtime cannot handle locale dependent output
144 // after the main thread closes so we use direct console output.
145 if (!_mi_preloading()) { _cputs(msg); }
146 #else
147 fputs(msg, stderr);
148 #endif
149}
150
151// Since an output function can be registered earliest in the `main`
152// function we also buffer output that happens earlier. When
153// an output function is registered it is called immediately with
154// the output up to that point.
155#ifndef MI_MAX_DELAY_OUTPUT
156#define MI_MAX_DELAY_OUTPUT (32*1024)
157#endif
158static char out_buf[MI_MAX_DELAY_OUTPUT+1];
159static _Atomic(uintptr_t) out_len;
160
161static void mi_out_buf(const char* msg) {
162 if (msg==NULL) return;
163 if (mi_atomic_read_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return;
164 size_t n = strlen(msg);
165 if (n==0) return;
166 // claim space
167 uintptr_t start = mi_atomic_addu(&out_len, n);
168 if (start >= MI_MAX_DELAY_OUTPUT) return;
169 // check bound
170 if (start+n >= MI_MAX_DELAY_OUTPUT) {
171 n = MI_MAX_DELAY_OUTPUT-start-1;
172 }
173 memcpy(&out_buf[start], msg, n);
174}
175
176static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf) {
177 if (out==NULL) return;
178 // claim (if `no_more_buf == true`, no more output will be added after this point)
179 size_t count = mi_atomic_addu(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));
180 // and output the current contents
181 if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
182 out_buf[count] = 0;
183 out(out_buf);
184 if (!no_more_buf) {
185 out_buf[count] = '\n'; // if continue with the buffer, insert a newline
186 }
187}
188
189
190// Once this module is loaded, switch to this routine
191// which outputs to stderr and the delayed output buffer.
192static void mi_out_buf_stderr(const char* msg) {
193 mi_out_stderr(msg);
194 mi_out_buf(msg);
195}
196
197
198
199// --------------------------------------------------------
200// Default output handler
201// --------------------------------------------------------
202
203// Should be atomic but gives errors on many platforms as generally we cannot cast a function pointer to a uintptr_t.
204// For now, don't register output from multiple threads.
205#pragma warning(suppress:4180)
206static mi_output_fun* volatile mi_out_default; // = NULL
207
208static mi_output_fun* mi_out_get_default(void) {
209 mi_output_fun* out = mi_out_default;
210 return (out == NULL ? &mi_out_buf : out);
211}
212
213void mi_register_output(mi_output_fun* out) mi_attr_noexcept {
214 mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer
215 if (out!=NULL) mi_out_buf_flush(out,true); // output all the delayed output now
216}
217
218// add stderr to the delayed output after the module is loaded
219static void mi_add_stderr_output() {
220 mi_out_buf_flush(&mi_out_stderr, false); // flush current contents to stderr
221 mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output
222}
223
224// --------------------------------------------------------
225// Messages, all end up calling `_mi_fputs`.
226// --------------------------------------------------------
227#define MAX_ERROR_COUNT (10)
228static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings
229
230// When overriding malloc, we may recurse into mi_vfprintf if an allocation
231// inside the C runtime causes another message.
232static mi_decl_thread bool recurse = false;
233
234void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message) {
235 if (recurse) return;
236 if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) out = mi_out_get_default();
237 recurse = true;
238 if (prefix != NULL) out(prefix);
239 out(message);
240 recurse = false;
241 return;
242}
243
244// Define our own limited `fprintf` that avoids memory allocation.
245// We do this using `snprintf` with a limited buffer.
246static void mi_vfprintf( mi_output_fun* out, const char* prefix, const char* fmt, va_list args ) {
247 char buf[512];
248 if (fmt==NULL) return;
249 if (recurse) return;
250 recurse = true;
251 vsnprintf(buf,sizeof(buf)-1,fmt,args);
252 recurse = false;
253 _mi_fputs(out,prefix,buf);
254}
255
256
257void _mi_fprintf( mi_output_fun* out, const char* fmt, ... ) {
258 va_list args;
259 va_start(args,fmt);
260 mi_vfprintf(out,NULL,fmt,args);
261 va_end(args);
262}
263
264void _mi_trace_message(const char* fmt, ...) {
265 if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher
266 va_list args;
267 va_start(args, fmt);
268 mi_vfprintf(NULL, "mimalloc: ", fmt, args);
269 va_end(args);
270}
271
272void _mi_verbose_message(const char* fmt, ...) {
273 if (!mi_option_is_enabled(mi_option_verbose)) return;
274 va_list args;
275 va_start(args,fmt);
276 mi_vfprintf(NULL, "mimalloc: ", fmt, args);
277 va_end(args);
278}
279
280void _mi_error_message(const char* fmt, ...) {
281 if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
282 if (mi_atomic_increment(&error_count) > mi_max_error_count) return;
283 va_list args;
284 va_start(args,fmt);
285 mi_vfprintf(NULL, "mimalloc: error: ", fmt, args);
286 va_end(args);
287 mi_assert(false);
288}
289
290void _mi_warning_message(const char* fmt, ...) {
291 if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return;
292 if (mi_atomic_increment(&error_count) > mi_max_error_count) return;
293 va_list args;
294 va_start(args,fmt);
295 mi_vfprintf(NULL, "mimalloc: warning: ", fmt, args);
296 va_end(args);
297}
298
299
300#if MI_DEBUG
301void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) {
302 _mi_fprintf(NULL,"mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
303 abort();
304}
305#endif
306
307mi_attr_noreturn void _mi_fatal_error(const char* fmt, ...) {
308 va_list args;
309 va_start(args, fmt);
310 mi_vfprintf(NULL, "mimalloc: fatal: ", fmt, args);
311 va_end(args);
312 #if (MI_SECURE>=0)
313 abort();
314 #endif
315}
316
317// --------------------------------------------------------
318// Initialize options by checking the environment
319// --------------------------------------------------------
320
321static void mi_strlcpy(char* dest, const char* src, size_t dest_size) {
322 dest[0] = 0;
323 #pragma warning(suppress:4996)
324 strncpy(dest, src, dest_size - 1);
325 dest[dest_size - 1] = 0;
326}
327
328static void mi_strlcat(char* dest, const char* src, size_t dest_size) {
329 #pragma warning(suppress:4996)
330 strncat(dest, src, dest_size - 1);
331 dest[dest_size - 1] = 0;
332}
333
334#if defined _WIN32
335// On Windows use GetEnvironmentVariable instead of getenv to work
336// reliably even when this is invoked before the C runtime is initialized.
337// i.e. when `_mi_preloading() == true`.
338// Note: on windows, environment names are not case sensitive.
339#include <windows.h>
340static bool mi_getenv(const char* name, char* result, size_t result_size) {
341 result[0] = 0;
342 size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size);
343 return (len > 0 && len < result_size);
344}
345#else
346static bool mi_getenv(const char* name, char* result, size_t result_size) {
347 const char* s = getenv(name);
348 if (s == NULL) {
349 // in unix environments we check the upper case name too.
350 char buf[64+1];
351 size_t len = strlen(name);
352 if (len >= sizeof(buf)) len = sizeof(buf) - 1;
353 for (size_t i = 0; i < len; i++) {
354 buf[i] = toupper(name[i]);
355 }
356 buf[len] = 0;
357 s = getenv(buf);
358 }
359 if (s != NULL && strlen(s) < result_size) {
360 mi_strlcpy(result, s, result_size);
361 return true;
362 }
363 else {
364 return false;
365 }
366}
367#endif
368static void mi_option_init(mi_option_desc_t* desc) {
369 // Read option value from the environment
370 char buf[64+1];
371 mi_strlcpy(buf, "mimalloc_", sizeof(buf));
372 mi_strlcat(buf, desc->name, sizeof(buf));
373 char s[64+1];
374 if (mi_getenv(buf, s, sizeof(s))) {
375 size_t len = strlen(s);
376 if (len >= sizeof(buf)) len = sizeof(buf) - 1;
377 for (size_t i = 0; i < len; i++) {
378 buf[i] = (char)toupper(s[i]);
379 }
380 buf[len] = 0;
381 if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) {
382 desc->value = 1;
383 desc->init = INITIALIZED;
384 }
385 else if (strstr("0;FALSE;NO;OFF", buf) != NULL) {
386 desc->value = 0;
387 desc->init = INITIALIZED;
388 }
389 else {
390 char* end = buf;
391 long value = strtol(buf, &end, 10);
392 if (*end == 0) {
393 desc->value = value;
394 desc->init = INITIALIZED;
395 }
396 else {
397 _mi_warning_message("environment option mimalloc_%s has an invalid value: %s\n", desc->name, buf);
398 desc->init = DEFAULTED;
399 }
400 }
401 }
402 else {
403 desc->init = DEFAULTED;
404 }
405 mi_assert_internal(desc->init != UNINIT);
406}
407