1 | /* ---------------------------------------------------------------------------- |
2 | Copyright (c) 2018, Microsoft Research, Daan Leijen |
3 | This is free software; you can redistribute it and/or modify it under the |
4 | terms of the MIT license. A copy of the license can be found in the file |
5 | "LICENSE" at the root of this distribution. |
6 | -----------------------------------------------------------------------------*/ |
7 | #include "mimalloc.h" |
8 | #include "mimalloc-internal.h" |
9 | #include "mimalloc-atomic.h" |
10 | |
11 | #include <stdio.h> |
12 | #include <stdlib.h> // strtol |
13 | #include <string.h> // strncpy, strncat, strlen, strstr |
14 | #include <ctype.h> // toupper |
15 | #include <stdarg.h> |
16 | |
17 | static uintptr_t mi_max_error_count = 16; // stop outputting errors after this |
18 | |
19 | static void mi_add_stderr_output(); |
20 | |
21 | int mi_version(void) mi_attr_noexcept { |
22 | return MI_MALLOC_VERSION; |
23 | } |
24 | |
25 | #ifdef _WIN32 |
26 | #include <conio.h> |
27 | #endif |
28 | |
29 | // -------------------------------------------------------- |
30 | // Options |
31 | // These can be accessed by multiple threads and may be |
32 | // concurrently initialized, but an initializing data race |
33 | // is ok since they resolve to the same value. |
34 | // -------------------------------------------------------- |
35 | typedef enum mi_init_e { |
36 | UNINIT, // not yet initialized |
37 | DEFAULTED, // not found in the environment, use default value |
38 | INITIALIZED // found in environment or set explicitly |
39 | } mi_init_t; |
40 | |
41 | typedef struct mi_option_desc_s { |
42 | long value; // the value |
43 | mi_init_t init; // is it initialized yet? (from the environment) |
44 | mi_option_t option; // for debugging: the option index should match the option |
45 | const char* name; // option name without `mimalloc_` prefix |
46 | } mi_option_desc_t; |
47 | |
48 | #define MI_OPTION(opt) mi_option_##opt, #opt |
49 | #define MI_OPTION_DESC(opt) {0, UNINIT, MI_OPTION(opt) } |
50 | |
51 | static mi_option_desc_t options[_mi_option_last] = |
52 | { |
53 | // stable options |
54 | { MI_DEBUG, UNINIT, MI_OPTION(show_errors) }, |
55 | { 0, UNINIT, MI_OPTION(show_stats) }, |
56 | { 0, UNINIT, MI_OPTION(verbose) }, |
57 | |
58 | // the following options are experimental and not all combinations make sense. |
59 | { 1, UNINIT, MI_OPTION(eager_commit) }, // note: needs to be on when eager_region_commit is enabled |
60 | #ifdef _WIN32 // and BSD? |
61 | { 0, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...) |
62 | #else |
63 | { 1, UNINIT, MI_OPTION(eager_region_commit) }, |
64 | #endif |
65 | { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's |
66 | { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, |
67 | { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread |
68 | { 0, UNINIT, MI_OPTION(page_reset) }, |
69 | { 0, UNINIT, MI_OPTION(cache_reset) }, |
70 | { 0, UNINIT, MI_OPTION(reset_decommits) }, // note: cannot enable this if secure is on |
71 | { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed |
72 | { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit) |
73 | { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose |
74 | { 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output |
75 | }; |
76 | |
77 | static void mi_option_init(mi_option_desc_t* desc); |
78 | |
79 | void _mi_options_init(void) { |
80 | // called on process load; should not be called before the CRT is initialized! |
81 | // (e.g. do not call this from process_init as that may run before CRT initialization) |
82 | mi_add_stderr_output(); // now it safe to use stderr for output |
83 | for(int i = 0; i < _mi_option_last; i++ ) { |
84 | mi_option_t option = (mi_option_t)i; |
85 | mi_option_get(option); // initialize |
86 | if (option != mi_option_verbose) { |
87 | mi_option_desc_t* desc = &options[option]; |
88 | _mi_verbose_message("option '%s': %ld\n" , desc->name, desc->value); |
89 | } |
90 | } |
91 | mi_max_error_count = mi_option_get(mi_option_max_errors); |
92 | } |
93 | |
94 | long mi_option_get(mi_option_t option) { |
95 | mi_assert(option >= 0 && option < _mi_option_last); |
96 | mi_option_desc_t* desc = &options[option]; |
97 | mi_assert(desc->option == option); // index should match the option |
98 | if (mi_unlikely(desc->init == UNINIT)) { |
99 | mi_option_init(desc); |
100 | } |
101 | return desc->value; |
102 | } |
103 | |
104 | void mi_option_set(mi_option_t option, long value) { |
105 | mi_assert(option >= 0 && option < _mi_option_last); |
106 | mi_option_desc_t* desc = &options[option]; |
107 | mi_assert(desc->option == option); // index should match the option |
108 | desc->value = value; |
109 | desc->init = INITIALIZED; |
110 | } |
111 | |
112 | void mi_option_set_default(mi_option_t option, long value) { |
113 | mi_assert(option >= 0 && option < _mi_option_last); |
114 | mi_option_desc_t* desc = &options[option]; |
115 | if (desc->init != INITIALIZED) { |
116 | desc->value = value; |
117 | } |
118 | } |
119 | |
120 | bool mi_option_is_enabled(mi_option_t option) { |
121 | return (mi_option_get(option) != 0); |
122 | } |
123 | |
124 | void mi_option_set_enabled(mi_option_t option, bool enable) { |
125 | mi_option_set(option, (enable ? 1 : 0)); |
126 | } |
127 | |
128 | void mi_option_set_enabled_default(mi_option_t option, bool enable) { |
129 | mi_option_set_default(option, (enable ? 1 : 0)); |
130 | } |
131 | |
132 | void mi_option_enable(mi_option_t option) { |
133 | mi_option_set_enabled(option,true); |
134 | } |
135 | |
136 | void mi_option_disable(mi_option_t option) { |
137 | mi_option_set_enabled(option,false); |
138 | } |
139 | |
140 | |
141 | static void mi_out_stderr(const char* msg) { |
142 | #ifdef _WIN32 |
143 | // on windows with redirection, the C runtime cannot handle locale dependent output |
144 | // after the main thread closes so we use direct console output. |
145 | if (!_mi_preloading()) { _cputs(msg); } |
146 | #else |
147 | fputs(msg, stderr); |
148 | #endif |
149 | } |
150 | |
151 | // Since an output function can be registered earliest in the `main` |
152 | // function we also buffer output that happens earlier. When |
153 | // an output function is registered it is called immediately with |
154 | // the output up to that point. |
155 | #ifndef MI_MAX_DELAY_OUTPUT |
156 | #define MI_MAX_DELAY_OUTPUT (32*1024) |
157 | #endif |
158 | static char out_buf[MI_MAX_DELAY_OUTPUT+1]; |
159 | static _Atomic(uintptr_t) out_len; |
160 | |
161 | static void mi_out_buf(const char* msg) { |
162 | if (msg==NULL) return; |
163 | if (mi_atomic_read_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; |
164 | size_t n = strlen(msg); |
165 | if (n==0) return; |
166 | // claim space |
167 | uintptr_t start = mi_atomic_addu(&out_len, n); |
168 | if (start >= MI_MAX_DELAY_OUTPUT) return; |
169 | // check bound |
170 | if (start+n >= MI_MAX_DELAY_OUTPUT) { |
171 | n = MI_MAX_DELAY_OUTPUT-start-1; |
172 | } |
173 | memcpy(&out_buf[start], msg, n); |
174 | } |
175 | |
176 | static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf) { |
177 | if (out==NULL) return; |
178 | // claim (if `no_more_buf == true`, no more output will be added after this point) |
179 | size_t count = mi_atomic_addu(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1)); |
180 | // and output the current contents |
181 | if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT; |
182 | out_buf[count] = 0; |
183 | out(out_buf); |
184 | if (!no_more_buf) { |
185 | out_buf[count] = '\n'; // if continue with the buffer, insert a newline |
186 | } |
187 | } |
188 | |
189 | |
190 | // Once this module is loaded, switch to this routine |
191 | // which outputs to stderr and the delayed output buffer. |
192 | static void mi_out_buf_stderr(const char* msg) { |
193 | mi_out_stderr(msg); |
194 | mi_out_buf(msg); |
195 | } |
196 | |
197 | |
198 | |
199 | // -------------------------------------------------------- |
200 | // Default output handler |
201 | // -------------------------------------------------------- |
202 | |
203 | // Should be atomic but gives errors on many platforms as generally we cannot cast a function pointer to a uintptr_t. |
204 | // For now, don't register output from multiple threads. |
205 | #pragma warning(suppress:4180) |
206 | static mi_output_fun* volatile mi_out_default; // = NULL |
207 | |
208 | static mi_output_fun* mi_out_get_default(void) { |
209 | mi_output_fun* out = mi_out_default; |
210 | return (out == NULL ? &mi_out_buf : out); |
211 | } |
212 | |
213 | void mi_register_output(mi_output_fun* out) mi_attr_noexcept { |
214 | mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer |
215 | if (out!=NULL) mi_out_buf_flush(out,true); // output all the delayed output now |
216 | } |
217 | |
218 | // add stderr to the delayed output after the module is loaded |
219 | static void mi_add_stderr_output() { |
220 | mi_out_buf_flush(&mi_out_stderr, false); // flush current contents to stderr |
221 | mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output |
222 | } |
223 | |
224 | // -------------------------------------------------------- |
225 | // Messages, all end up calling `_mi_fputs`. |
226 | // -------------------------------------------------------- |
227 | #define MAX_ERROR_COUNT (10) |
228 | static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings |
229 | |
230 | // When overriding malloc, we may recurse into mi_vfprintf if an allocation |
231 | // inside the C runtime causes another message. |
232 | static mi_decl_thread bool recurse = false; |
233 | |
234 | void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message) { |
235 | if (recurse) return; |
236 | if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) out = mi_out_get_default(); |
237 | recurse = true; |
238 | if (prefix != NULL) out(prefix); |
239 | out(message); |
240 | recurse = false; |
241 | return; |
242 | } |
243 | |
244 | // Define our own limited `fprintf` that avoids memory allocation. |
245 | // We do this using `snprintf` with a limited buffer. |
246 | static void mi_vfprintf( mi_output_fun* out, const char* prefix, const char* fmt, va_list args ) { |
247 | char buf[512]; |
248 | if (fmt==NULL) return; |
249 | if (recurse) return; |
250 | recurse = true; |
251 | vsnprintf(buf,sizeof(buf)-1,fmt,args); |
252 | recurse = false; |
253 | _mi_fputs(out,prefix,buf); |
254 | } |
255 | |
256 | |
257 | void _mi_fprintf( mi_output_fun* out, const char* fmt, ... ) { |
258 | va_list args; |
259 | va_start(args,fmt); |
260 | mi_vfprintf(out,NULL,fmt,args); |
261 | va_end(args); |
262 | } |
263 | |
264 | void _mi_trace_message(const char* fmt, ...) { |
265 | if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher |
266 | va_list args; |
267 | va_start(args, fmt); |
268 | mi_vfprintf(NULL, "mimalloc: " , fmt, args); |
269 | va_end(args); |
270 | } |
271 | |
272 | void _mi_verbose_message(const char* fmt, ...) { |
273 | if (!mi_option_is_enabled(mi_option_verbose)) return; |
274 | va_list args; |
275 | va_start(args,fmt); |
276 | mi_vfprintf(NULL, "mimalloc: " , fmt, args); |
277 | va_end(args); |
278 | } |
279 | |
280 | void _mi_error_message(const char* fmt, ...) { |
281 | if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; |
282 | if (mi_atomic_increment(&error_count) > mi_max_error_count) return; |
283 | va_list args; |
284 | va_start(args,fmt); |
285 | mi_vfprintf(NULL, "mimalloc: error: " , fmt, args); |
286 | va_end(args); |
287 | mi_assert(false); |
288 | } |
289 | |
290 | void _mi_warning_message(const char* fmt, ...) { |
291 | if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; |
292 | if (mi_atomic_increment(&error_count) > mi_max_error_count) return; |
293 | va_list args; |
294 | va_start(args,fmt); |
295 | mi_vfprintf(NULL, "mimalloc: warning: " , fmt, args); |
296 | va_end(args); |
297 | } |
298 | |
299 | |
300 | #if MI_DEBUG |
301 | void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) { |
302 | _mi_fprintf(NULL,"mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n" , fname, line, (func==NULL?"" :func), assertion); |
303 | abort(); |
304 | } |
305 | #endif |
306 | |
307 | mi_attr_noreturn void _mi_fatal_error(const char* fmt, ...) { |
308 | va_list args; |
309 | va_start(args, fmt); |
310 | mi_vfprintf(NULL, "mimalloc: fatal: " , fmt, args); |
311 | va_end(args); |
312 | #if (MI_SECURE>=0) |
313 | abort(); |
314 | #endif |
315 | } |
316 | |
317 | // -------------------------------------------------------- |
318 | // Initialize options by checking the environment |
319 | // -------------------------------------------------------- |
320 | |
321 | static void mi_strlcpy(char* dest, const char* src, size_t dest_size) { |
322 | dest[0] = 0; |
323 | #pragma warning(suppress:4996) |
324 | strncpy(dest, src, dest_size - 1); |
325 | dest[dest_size - 1] = 0; |
326 | } |
327 | |
328 | static void mi_strlcat(char* dest, const char* src, size_t dest_size) { |
329 | #pragma warning(suppress:4996) |
330 | strncat(dest, src, dest_size - 1); |
331 | dest[dest_size - 1] = 0; |
332 | } |
333 | |
334 | #if defined _WIN32 |
335 | // On Windows use GetEnvironmentVariable instead of getenv to work |
336 | // reliably even when this is invoked before the C runtime is initialized. |
337 | // i.e. when `_mi_preloading() == true`. |
338 | // Note: on windows, environment names are not case sensitive. |
339 | #include <windows.h> |
340 | static bool mi_getenv(const char* name, char* result, size_t result_size) { |
341 | result[0] = 0; |
342 | size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); |
343 | return (len > 0 && len < result_size); |
344 | } |
345 | #else |
346 | static bool mi_getenv(const char* name, char* result, size_t result_size) { |
347 | const char* s = getenv(name); |
348 | if (s == NULL) { |
349 | // in unix environments we check the upper case name too. |
350 | char buf[64+1]; |
351 | size_t len = strlen(name); |
352 | if (len >= sizeof(buf)) len = sizeof(buf) - 1; |
353 | for (size_t i = 0; i < len; i++) { |
354 | buf[i] = toupper(name[i]); |
355 | } |
356 | buf[len] = 0; |
357 | s = getenv(buf); |
358 | } |
359 | if (s != NULL && strlen(s) < result_size) { |
360 | mi_strlcpy(result, s, result_size); |
361 | return true; |
362 | } |
363 | else { |
364 | return false; |
365 | } |
366 | } |
367 | #endif |
368 | static void mi_option_init(mi_option_desc_t* desc) { |
369 | // Read option value from the environment |
370 | char buf[64+1]; |
371 | mi_strlcpy(buf, "mimalloc_" , sizeof(buf)); |
372 | mi_strlcat(buf, desc->name, sizeof(buf)); |
373 | char s[64+1]; |
374 | if (mi_getenv(buf, s, sizeof(s))) { |
375 | size_t len = strlen(s); |
376 | if (len >= sizeof(buf)) len = sizeof(buf) - 1; |
377 | for (size_t i = 0; i < len; i++) { |
378 | buf[i] = (char)toupper(s[i]); |
379 | } |
380 | buf[len] = 0; |
381 | if (buf[0]==0 || strstr("1;TRUE;YES;ON" , buf) != NULL) { |
382 | desc->value = 1; |
383 | desc->init = INITIALIZED; |
384 | } |
385 | else if (strstr("0;FALSE;NO;OFF" , buf) != NULL) { |
386 | desc->value = 0; |
387 | desc->init = INITIALIZED; |
388 | } |
389 | else { |
390 | char* end = buf; |
391 | long value = strtol(buf, &end, 10); |
392 | if (*end == 0) { |
393 | desc->value = value; |
394 | desc->init = INITIALIZED; |
395 | } |
396 | else { |
397 | _mi_warning_message("environment option mimalloc_%s has an invalid value: %s\n" , desc->name, buf); |
398 | desc->init = DEFAULTED; |
399 | } |
400 | } |
401 | } |
402 | else { |
403 | desc->init = DEFAULTED; |
404 | } |
405 | mi_assert_internal(desc->init != UNINIT); |
406 | } |
407 | |