1 | /* ---------------------------------------------------------------------------- |
2 | Copyright (c) 2018, Microsoft Research, Daan Leijen |
3 | This is free software; you can redistribute it and/or modify it under the |
4 | terms of the MIT license. A copy of the license can be found in the file |
5 | "LICENSE" at the root of this distribution. |
6 | -----------------------------------------------------------------------------*/ |
7 | #include "mimalloc.h" |
8 | #include "mimalloc-internal.h" |
9 | #include "mimalloc-atomic.h" |
10 | |
11 | #include <stdio.h> // fputs, stderr |
12 | #include <string.h> // memset |
13 | |
14 | |
15 | /* ----------------------------------------------------------- |
16 | Statistics operations |
17 | ----------------------------------------------------------- */ |
18 | |
19 | static bool mi_is_in_main(void* stat) { |
20 | return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main |
21 | && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t))); |
22 | } |
23 | |
24 | static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { |
25 | if (amount == 0) return; |
26 | if (mi_is_in_main(stat)) |
27 | { |
28 | // add atomically (for abandoned pages) |
29 | mi_atomic_add64(&stat->current,amount); |
30 | if (stat->current > stat->peak) stat->peak = stat->current; // racing.. it's ok |
31 | if (amount > 0) { |
32 | mi_atomic_add64(&stat->allocated,amount); |
33 | } |
34 | else { |
35 | mi_atomic_add64(&stat->freed, -amount); |
36 | } |
37 | } |
38 | else { |
39 | // add thread local |
40 | stat->current += amount; |
41 | if (stat->current > stat->peak) stat->peak = stat->current; |
42 | if (amount > 0) { |
43 | stat->allocated += amount; |
44 | } |
45 | else { |
46 | stat->freed += -amount; |
47 | } |
48 | } |
49 | } |
50 | |
51 | void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { |
52 | if (mi_is_in_main(stat)) { |
53 | mi_atomic_add64( &stat->count, 1 ); |
54 | mi_atomic_add64( &stat->total, (int64_t)amount ); |
55 | } |
56 | else { |
57 | stat->count++; |
58 | stat->total += amount; |
59 | } |
60 | } |
61 | |
62 | void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) { |
63 | mi_stat_update(stat, (int64_t)amount); |
64 | } |
65 | |
66 | void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { |
67 | mi_stat_update(stat, -((int64_t)amount)); |
68 | } |
69 | |
70 | // must be thread safe as it is called from stats_merge |
71 | static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) { |
72 | if (stat==src) return; |
73 | mi_atomic_add64( &stat->allocated, src->allocated * unit); |
74 | mi_atomic_add64( &stat->current, src->current * unit); |
75 | mi_atomic_add64( &stat->freed, src->freed * unit); |
76 | // peak scores do not work across threads.. |
77 | mi_atomic_add64( &stat->peak, src->peak * unit); |
78 | } |
79 | |
80 | static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t* src, int64_t unit) { |
81 | if (stat==src) return; |
82 | mi_atomic_add64( &stat->total, src->total * unit); |
83 | mi_atomic_add64( &stat->count, src->count * unit); |
84 | } |
85 | |
86 | // must be thread safe as it is called from stats_merge |
87 | static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { |
88 | if (stats==src) return; |
89 | mi_stat_add(&stats->segments, &src->segments,1); |
90 | mi_stat_add(&stats->pages, &src->pages,1); |
91 | mi_stat_add(&stats->reserved, &src->reserved, 1); |
92 | mi_stat_add(&stats->committed, &src->committed, 1); |
93 | mi_stat_add(&stats->reset, &src->reset, 1); |
94 | mi_stat_add(&stats->page_committed, &src->page_committed, 1); |
95 | |
96 | mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); |
97 | mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); |
98 | mi_stat_add(&stats->threads, &src->threads, 1); |
99 | |
100 | mi_stat_add(&stats->malloc, &src->malloc, 1); |
101 | mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); |
102 | mi_stat_add(&stats->huge, &src->huge, 1); |
103 | mi_stat_add(&stats->giant, &src->giant, 1); |
104 | |
105 | mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1); |
106 | mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls, 1); |
107 | mi_stat_counter_add(&stats->commit_calls, &src->commit_calls, 1); |
108 | |
109 | mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); |
110 | mi_stat_counter_add(&stats->searches, &src->searches, 1); |
111 | mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); |
112 | mi_stat_counter_add(&stats->giant_count, &src->giant_count, 1); |
113 | #if MI_STAT>1 |
114 | for (size_t i = 0; i <= MI_BIN_HUGE; i++) { |
115 | if (src->normal[i].allocated > 0 || src->normal[i].freed > 0) { |
116 | mi_stat_add(&stats->normal[i], &src->normal[i], 1); |
117 | } |
118 | } |
119 | #endif |
120 | } |
121 | |
122 | /* ----------------------------------------------------------- |
123 | Display statistics |
124 | ----------------------------------------------------------- */ |
125 | |
126 | // unit > 0 : size in binary bytes |
127 | // unit == 0: count as decimal |
128 | // unit < 0 : count in binary |
129 | static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, const char* fmt) { |
130 | char buf[32]; |
131 | int len = 32; |
132 | const char* suffix = (unit <= 0 ? " " : "b" ); |
133 | double base = (unit == 0 ? 1000.0 : 1024.0); |
134 | if (unit>0) n *= unit; |
135 | |
136 | double pos = (double)(n < 0 ? -n : n); |
137 | if (pos < base) |
138 | snprintf(buf,len, "%d %s " , (int)n, suffix); |
139 | else if (pos < base*base) |
140 | snprintf(buf, len, "%.1f k%s" , (double)n / base, suffix); |
141 | else if (pos < base*base*base) |
142 | snprintf(buf, len, "%.1f m%s" , (double)n / (base*base), suffix); |
143 | else |
144 | snprintf(buf, len, "%.1f g%s" , (double)n / (base*base*base), suffix); |
145 | |
146 | _mi_fprintf(out, (fmt==NULL ? "%11s" : fmt), buf); |
147 | } |
148 | |
149 | |
150 | static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out) { |
151 | mi_printf_amount(n,unit,out,NULL); |
152 | } |
153 | |
154 | static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out) { |
155 | if (unit==1) _mi_fprintf(out,"%11s" ," " ); |
156 | else mi_print_amount(n,0,out); |
157 | } |
158 | |
159 | static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out ) { |
160 | _mi_fprintf(out,"%10s:" , msg); |
161 | if (unit>0) { |
162 | mi_print_amount(stat->peak, unit, out); |
163 | mi_print_amount(stat->allocated, unit, out); |
164 | mi_print_amount(stat->freed, unit, out); |
165 | mi_print_amount(unit, 1, out); |
166 | mi_print_count(stat->allocated, unit, out); |
167 | if (stat->allocated > stat->freed) |
168 | _mi_fprintf(out, " not all freed!\n" ); |
169 | else |
170 | _mi_fprintf(out, " ok\n" ); |
171 | } |
172 | else if (unit<0) { |
173 | mi_print_amount(stat->peak, -1, out); |
174 | mi_print_amount(stat->allocated, -1, out); |
175 | mi_print_amount(stat->freed, -1, out); |
176 | if (unit==-1) { |
177 | _mi_fprintf(out, "%22s" , "" ); |
178 | } |
179 | else { |
180 | mi_print_amount(-unit, 1, out); |
181 | mi_print_count((stat->allocated / -unit), 0, out); |
182 | } |
183 | if (stat->allocated > stat->freed) |
184 | _mi_fprintf(out, " not all freed!\n" ); |
185 | else |
186 | _mi_fprintf(out, " ok\n" ); |
187 | } |
188 | else { |
189 | mi_print_amount(stat->peak, 1, out); |
190 | mi_print_amount(stat->allocated, 1, out); |
191 | _mi_fprintf(out, "\n" ); |
192 | } |
193 | } |
194 | |
195 | static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out ) { |
196 | _mi_fprintf(out, "%10s:" , msg); |
197 | mi_print_amount(stat->total, -1, out); |
198 | _mi_fprintf(out, "\n" ); |
199 | } |
200 | |
201 | static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out) { |
202 | double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); |
203 | _mi_fprintf(out, "%10s: %7.1f avg\n" , msg, avg); |
204 | } |
205 | |
206 | |
207 | static void (mi_output_fun* out ) { |
208 | _mi_fprintf(out,"%10s: %10s %10s %10s %10s %10s\n" , "heap stats" , "peak " , "total " , "freed " , "unit " , "count " ); |
209 | } |
210 | |
211 | #if MI_STAT>1 |
212 | static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out) { |
213 | bool found = false; |
214 | char buf[64]; |
215 | for (size_t i = 0; i <= max; i++) { |
216 | if (bins[i].allocated > 0) { |
217 | found = true; |
218 | int64_t unit = _mi_bin_size((uint8_t)i); |
219 | snprintf(buf, 64, "%s %3zu" , fmt, i); |
220 | mi_stat_add(all, &bins[i], unit); |
221 | mi_stat_print(&bins[i], buf, unit, out); |
222 | } |
223 | } |
224 | //snprintf(buf, 64, "%s all", fmt); |
225 | //mi_stat_print(all, buf, 1); |
226 | if (found) { |
227 | _mi_fprintf(out, "\n" ); |
228 | mi_print_header(out); |
229 | } |
230 | } |
231 | #endif |
232 | |
233 | |
234 | static void mi_process_info(double* utime, double* stime, size_t* , size_t* page_faults, size_t* page_reclaim, size_t* peak_commit); |
235 | |
236 | static void _mi_stats_print(mi_stats_t* stats, double secs, mi_output_fun* out) mi_attr_noexcept { |
237 | mi_print_header(out); |
238 | #if MI_STAT>1 |
239 | mi_stat_count_t normal = { 0,0,0,0 }; |
240 | mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal" ,out); |
241 | mi_stat_print(&normal, "normal" , 1, out); |
242 | mi_stat_print(&stats->huge, "huge" , (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out); |
243 | mi_stat_print(&stats->giant, "giant" , (stats->giant_count.count == 0 ? 1 : -(stats->giant.allocated / stats->giant_count.count)), out); |
244 | mi_stat_count_t total = { 0,0,0,0 }; |
245 | mi_stat_add(&total, &normal, 1); |
246 | mi_stat_add(&total, &stats->huge, 1); |
247 | mi_stat_add(&total, &stats->giant, 1); |
248 | mi_stat_print(&total, "total" , 1, out); |
249 | _mi_fprintf(out, "malloc requested: " ); |
250 | mi_print_amount(stats->malloc.allocated, 1, out); |
251 | _mi_fprintf(out, "\n\n" ); |
252 | #endif |
253 | mi_stat_print(&stats->reserved, "reserved" , 1, out); |
254 | mi_stat_print(&stats->committed, "committed" , 1, out); |
255 | mi_stat_print(&stats->reset, "reset" , 1, out); |
256 | mi_stat_print(&stats->page_committed, "touched" , 1, out); |
257 | mi_stat_print(&stats->segments, "segments" , -1, out); |
258 | mi_stat_print(&stats->segments_abandoned, "-abandoned" , -1, out); |
259 | mi_stat_print(&stats->segments_cache, "-cached" , -1, out); |
260 | mi_stat_print(&stats->pages, "pages" , -1, out); |
261 | mi_stat_print(&stats->pages_abandoned, "-abandoned" , -1, out); |
262 | mi_stat_counter_print(&stats->pages_extended, "-extended" , out); |
263 | mi_stat_counter_print(&stats->page_no_retire, "-noretire" , out); |
264 | mi_stat_counter_print(&stats->mmap_calls, "mmaps" , out); |
265 | mi_stat_counter_print(&stats->commit_calls, "commits" , out); |
266 | mi_stat_print(&stats->threads, "threads" , -1, out); |
267 | mi_stat_counter_print_avg(&stats->searches, "searches" , out); |
268 | |
269 | if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n" , "elapsed" , secs); |
270 | |
271 | double user_time; |
272 | double sys_time; |
273 | size_t ; |
274 | size_t page_faults; |
275 | size_t page_reclaim; |
276 | size_t peak_commit; |
277 | mi_process_info(&user_time, &sys_time, &peak_rss, &page_faults, &page_reclaim, &peak_commit); |
278 | _mi_fprintf(out,"%10s: user: %.3f s, system: %.3f s, faults: %lu, reclaims: %lu, rss: " , "process" , user_time, sys_time, (unsigned long)page_faults, (unsigned long)page_reclaim ); |
279 | mi_printf_amount((int64_t)peak_rss, 1, out, "%s" ); |
280 | if (peak_commit > 0) { |
281 | _mi_fprintf(out,", commit charge: " ); |
282 | mi_printf_amount((int64_t)peak_commit, 1, out, "%s" ); |
283 | } |
284 | _mi_fprintf(out,"\n" ); |
285 | } |
286 | |
287 | double _mi_clock_end(double start); |
288 | double _mi_clock_start(void); |
289 | static double mi_time_start = 0.0; |
290 | |
291 | static mi_stats_t* mi_stats_get_default(void) { |
292 | mi_heap_t* heap = mi_heap_get_default(); |
293 | return &heap->tld->stats; |
294 | } |
295 | |
296 | static void mi_stats_merge_from(mi_stats_t* stats) { |
297 | if (stats != &_mi_stats_main) { |
298 | mi_stats_add(&_mi_stats_main, stats); |
299 | memset(stats, 0, sizeof(mi_stats_t)); |
300 | } |
301 | } |
302 | |
303 | void mi_stats_reset(void) mi_attr_noexcept { |
304 | mi_stats_t* stats = mi_stats_get_default(); |
305 | if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); } |
306 | memset(&_mi_stats_main, 0, sizeof(mi_stats_t)); |
307 | mi_time_start = _mi_clock_start(); |
308 | } |
309 | |
310 | void mi_stats_merge(void) mi_attr_noexcept { |
311 | mi_stats_merge_from( mi_stats_get_default() ); |
312 | } |
313 | |
314 | void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done` |
315 | mi_stats_merge_from(stats); |
316 | } |
317 | |
318 | |
319 | static void mi_stats_print_ex(mi_stats_t* stats, double secs, mi_output_fun* out) { |
320 | mi_stats_merge_from(stats); |
321 | _mi_stats_print(&_mi_stats_main, secs, out); |
322 | } |
323 | |
324 | void mi_stats_print(mi_output_fun* out) mi_attr_noexcept { |
325 | mi_stats_print_ex(mi_stats_get_default(),_mi_clock_end(mi_time_start),out); |
326 | } |
327 | |
328 | void mi_thread_stats_print(mi_output_fun* out) mi_attr_noexcept { |
329 | _mi_stats_print(mi_stats_get_default(), _mi_clock_end(mi_time_start), out); |
330 | } |
331 | |
332 | |
333 | |
334 | // -------------------------------------------------------- |
335 | // Basic timer for convenience |
336 | // -------------------------------------------------------- |
337 | |
338 | #ifdef _WIN32 |
339 | #include <windows.h> |
340 | static double mi_to_seconds(LARGE_INTEGER t) { |
341 | static double freq = 0.0; |
342 | if (freq <= 0.0) { |
343 | LARGE_INTEGER f; |
344 | QueryPerformanceFrequency(&f); |
345 | freq = (double)(f.QuadPart); |
346 | } |
347 | return ((double)(t.QuadPart) / freq); |
348 | } |
349 | |
350 | static double mi_clock_now(void) { |
351 | LARGE_INTEGER t; |
352 | QueryPerformanceCounter(&t); |
353 | return mi_to_seconds(t); |
354 | } |
355 | #else |
356 | #include <time.h> |
357 | #ifdef CLOCK_REALTIME |
358 | static double mi_clock_now(void) { |
359 | struct timespec t; |
360 | clock_gettime(CLOCK_REALTIME, &t); |
361 | return (double)t.tv_sec + (1.0e-9 * (double)t.tv_nsec); |
362 | } |
363 | #else |
364 | // low resolution timer |
365 | static double mi_clock_now(void) { |
366 | return ((double)clock() / (double)CLOCKS_PER_SEC); |
367 | } |
368 | #endif |
369 | #endif |
370 | |
371 | |
372 | static double mi_clock_diff = 0.0; |
373 | |
374 | double _mi_clock_start(void) { |
375 | if (mi_clock_diff == 0.0) { |
376 | double t0 = mi_clock_now(); |
377 | mi_clock_diff = mi_clock_now() - t0; |
378 | } |
379 | return mi_clock_now(); |
380 | } |
381 | |
382 | double _mi_clock_end(double start) { |
383 | double end = mi_clock_now(); |
384 | return (end - start - mi_clock_diff); |
385 | } |
386 | |
387 | |
388 | // -------------------------------------------------------- |
389 | // Basic process statistics |
390 | // -------------------------------------------------------- |
391 | |
392 | #if defined(_WIN32) |
393 | #include <windows.h> |
394 | #include <psapi.h> |
395 | #pragma comment(lib,"psapi.lib") |
396 | |
397 | static double filetime_secs(const FILETIME* ftime) { |
398 | ULARGE_INTEGER i; |
399 | i.LowPart = ftime->dwLowDateTime; |
400 | i.HighPart = ftime->dwHighDateTime; |
401 | double secs = (double)(i.QuadPart) * 1.0e-7; // FILETIME is in 100 nano seconds |
402 | return secs; |
403 | } |
404 | static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { |
405 | FILETIME ct; |
406 | FILETIME ut; |
407 | FILETIME st; |
408 | FILETIME et; |
409 | GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); |
410 | *utime = filetime_secs(&ut); |
411 | *stime = filetime_secs(&st); |
412 | |
413 | PROCESS_MEMORY_COUNTERS info; |
414 | GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); |
415 | *peak_rss = (size_t)info.PeakWorkingSetSize; |
416 | *page_faults = (size_t)info.PageFaultCount; |
417 | *peak_commit = (size_t)info.PeakPagefileUsage; |
418 | *page_reclaim = 0; |
419 | } |
420 | |
421 | #elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) |
422 | #include <stdio.h> |
423 | #include <unistd.h> |
424 | #include <sys/resource.h> |
425 | |
426 | #if defined(__APPLE__) && defined(__MACH__) |
427 | #include <mach/mach.h> |
428 | #endif |
429 | |
430 | static double timeval_secs(const struct timeval* tv) { |
431 | return (double)tv->tv_sec + ((double)tv->tv_usec * 1.0e-6); |
432 | } |
433 | |
434 | static void mi_process_info(double* utime, double* stime, size_t* , size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { |
435 | struct rusage rusage; |
436 | getrusage(RUSAGE_SELF, &rusage); |
437 | #if defined(__APPLE__) && defined(__MACH__) |
438 | *peak_rss = rusage.ru_maxrss; |
439 | #else |
440 | *peak_rss = rusage.ru_maxrss * 1024; |
441 | #endif |
442 | *page_faults = rusage.ru_majflt; |
443 | *page_reclaim = rusage.ru_minflt; |
444 | *peak_commit = 0; |
445 | *utime = timeval_secs(&rusage.ru_utime); |
446 | *stime = timeval_secs(&rusage.ru_stime); |
447 | } |
448 | |
449 | #else |
450 | #ifndef __wasi__ |
451 | // WebAssembly instances are not processes |
452 | #pragma message("define a way to get process info") |
453 | #endif |
454 | |
455 | static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { |
456 | *peak_rss = 0; |
457 | *page_faults = 0; |
458 | *page_reclaim = 0; |
459 | *peak_commit = 0; |
460 | *utime = 0.0; |
461 | *stime = 0.0; |
462 | } |
463 | #endif |
464 | |