1/* ----------------------------------------------------------------------------
2Copyright (c) 2018, Microsoft Research, Daan Leijen
3This is free software; you can redistribute it and/or modify it under the
4terms of the MIT license. A copy of the license can be found in the file
5"LICENSE" at the root of this distribution.
6-----------------------------------------------------------------------------*/
7#include "mimalloc.h"
8#include "mimalloc-internal.h"
9#include "mimalloc-atomic.h"
10
11#include <stdio.h> // fputs, stderr
12#include <string.h> // memset
13
14
15/* -----------------------------------------------------------
16 Statistics operations
17----------------------------------------------------------- */
18
19static bool mi_is_in_main(void* stat) {
20 return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main
21 && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));
22}
23
24static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
25 if (amount == 0) return;
26 if (mi_is_in_main(stat))
27 {
28 // add atomically (for abandoned pages)
29 mi_atomic_add64(&stat->current,amount);
30 if (stat->current > stat->peak) stat->peak = stat->current; // racing.. it's ok
31 if (amount > 0) {
32 mi_atomic_add64(&stat->allocated,amount);
33 }
34 else {
35 mi_atomic_add64(&stat->freed, -amount);
36 }
37 }
38 else {
39 // add thread local
40 stat->current += amount;
41 if (stat->current > stat->peak) stat->peak = stat->current;
42 if (amount > 0) {
43 stat->allocated += amount;
44 }
45 else {
46 stat->freed += -amount;
47 }
48 }
49}
50
51void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
52 if (mi_is_in_main(stat)) {
53 mi_atomic_add64( &stat->count, 1 );
54 mi_atomic_add64( &stat->total, (int64_t)amount );
55 }
56 else {
57 stat->count++;
58 stat->total += amount;
59 }
60}
61
62void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
63 mi_stat_update(stat, (int64_t)amount);
64}
65
66void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
67 mi_stat_update(stat, -((int64_t)amount));
68}
69
70// must be thread safe as it is called from stats_merge
71static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) {
72 if (stat==src) return;
73 mi_atomic_add64( &stat->allocated, src->allocated * unit);
74 mi_atomic_add64( &stat->current, src->current * unit);
75 mi_atomic_add64( &stat->freed, src->freed * unit);
76 // peak scores do not work across threads..
77 mi_atomic_add64( &stat->peak, src->peak * unit);
78}
79
80static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t* src, int64_t unit) {
81 if (stat==src) return;
82 mi_atomic_add64( &stat->total, src->total * unit);
83 mi_atomic_add64( &stat->count, src->count * unit);
84}
85
86// must be thread safe as it is called from stats_merge
87static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
88 if (stats==src) return;
89 mi_stat_add(&stats->segments, &src->segments,1);
90 mi_stat_add(&stats->pages, &src->pages,1);
91 mi_stat_add(&stats->reserved, &src->reserved, 1);
92 mi_stat_add(&stats->committed, &src->committed, 1);
93 mi_stat_add(&stats->reset, &src->reset, 1);
94 mi_stat_add(&stats->page_committed, &src->page_committed, 1);
95
96 mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1);
97 mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1);
98 mi_stat_add(&stats->threads, &src->threads, 1);
99
100 mi_stat_add(&stats->malloc, &src->malloc, 1);
101 mi_stat_add(&stats->segments_cache, &src->segments_cache, 1);
102 mi_stat_add(&stats->huge, &src->huge, 1);
103 mi_stat_add(&stats->giant, &src->giant, 1);
104
105 mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1);
106 mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls, 1);
107 mi_stat_counter_add(&stats->commit_calls, &src->commit_calls, 1);
108
109 mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1);
110 mi_stat_counter_add(&stats->searches, &src->searches, 1);
111 mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1);
112 mi_stat_counter_add(&stats->giant_count, &src->giant_count, 1);
113#if MI_STAT>1
114 for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
115 if (src->normal[i].allocated > 0 || src->normal[i].freed > 0) {
116 mi_stat_add(&stats->normal[i], &src->normal[i], 1);
117 }
118 }
119#endif
120}
121
122/* -----------------------------------------------------------
123 Display statistics
124----------------------------------------------------------- */
125
126// unit > 0 : size in binary bytes
127// unit == 0: count as decimal
128// unit < 0 : count in binary
129static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, const char* fmt) {
130 char buf[32];
131 int len = 32;
132 const char* suffix = (unit <= 0 ? " " : "b");
133 double base = (unit == 0 ? 1000.0 : 1024.0);
134 if (unit>0) n *= unit;
135
136 double pos = (double)(n < 0 ? -n : n);
137 if (pos < base)
138 snprintf(buf,len, "%d %s ", (int)n, suffix);
139 else if (pos < base*base)
140 snprintf(buf, len, "%.1f k%s", (double)n / base, suffix);
141 else if (pos < base*base*base)
142 snprintf(buf, len, "%.1f m%s", (double)n / (base*base), suffix);
143 else
144 snprintf(buf, len, "%.1f g%s", (double)n / (base*base*base), suffix);
145
146 _mi_fprintf(out, (fmt==NULL ? "%11s" : fmt), buf);
147}
148
149
150static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out) {
151 mi_printf_amount(n,unit,out,NULL);
152}
153
154static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out) {
155 if (unit==1) _mi_fprintf(out,"%11s"," ");
156 else mi_print_amount(n,0,out);
157}
158
159static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out ) {
160 _mi_fprintf(out,"%10s:", msg);
161 if (unit>0) {
162 mi_print_amount(stat->peak, unit, out);
163 mi_print_amount(stat->allocated, unit, out);
164 mi_print_amount(stat->freed, unit, out);
165 mi_print_amount(unit, 1, out);
166 mi_print_count(stat->allocated, unit, out);
167 if (stat->allocated > stat->freed)
168 _mi_fprintf(out, " not all freed!\n");
169 else
170 _mi_fprintf(out, " ok\n");
171 }
172 else if (unit<0) {
173 mi_print_amount(stat->peak, -1, out);
174 mi_print_amount(stat->allocated, -1, out);
175 mi_print_amount(stat->freed, -1, out);
176 if (unit==-1) {
177 _mi_fprintf(out, "%22s", "");
178 }
179 else {
180 mi_print_amount(-unit, 1, out);
181 mi_print_count((stat->allocated / -unit), 0, out);
182 }
183 if (stat->allocated > stat->freed)
184 _mi_fprintf(out, " not all freed!\n");
185 else
186 _mi_fprintf(out, " ok\n");
187 }
188 else {
189 mi_print_amount(stat->peak, 1, out);
190 mi_print_amount(stat->allocated, 1, out);
191 _mi_fprintf(out, "\n");
192 }
193}
194
195static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out ) {
196 _mi_fprintf(out, "%10s:", msg);
197 mi_print_amount(stat->total, -1, out);
198 _mi_fprintf(out, "\n");
199}
200
201static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out) {
202 double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count);
203 _mi_fprintf(out, "%10s: %7.1f avg\n", msg, avg);
204}
205
206
207static void mi_print_header(mi_output_fun* out ) {
208 _mi_fprintf(out,"%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count ");
209}
210
211#if MI_STAT>1
212static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out) {
213 bool found = false;
214 char buf[64];
215 for (size_t i = 0; i <= max; i++) {
216 if (bins[i].allocated > 0) {
217 found = true;
218 int64_t unit = _mi_bin_size((uint8_t)i);
219 snprintf(buf, 64, "%s %3zu", fmt, i);
220 mi_stat_add(all, &bins[i], unit);
221 mi_stat_print(&bins[i], buf, unit, out);
222 }
223 }
224 //snprintf(buf, 64, "%s all", fmt);
225 //mi_stat_print(all, buf, 1);
226 if (found) {
227 _mi_fprintf(out, "\n");
228 mi_print_header(out);
229 }
230}
231#endif
232
233
234static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit);
235
236static void _mi_stats_print(mi_stats_t* stats, double secs, mi_output_fun* out) mi_attr_noexcept {
237 mi_print_header(out);
238 #if MI_STAT>1
239 mi_stat_count_t normal = { 0,0,0,0 };
240 mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out);
241 mi_stat_print(&normal, "normal", 1, out);
242 mi_stat_print(&stats->huge, "huge", (stats->huge_count.count == 0 ? 1 : -(stats->huge.allocated / stats->huge_count.count)), out);
243 mi_stat_print(&stats->giant, "giant", (stats->giant_count.count == 0 ? 1 : -(stats->giant.allocated / stats->giant_count.count)), out);
244 mi_stat_count_t total = { 0,0,0,0 };
245 mi_stat_add(&total, &normal, 1);
246 mi_stat_add(&total, &stats->huge, 1);
247 mi_stat_add(&total, &stats->giant, 1);
248 mi_stat_print(&total, "total", 1, out);
249 _mi_fprintf(out, "malloc requested: ");
250 mi_print_amount(stats->malloc.allocated, 1, out);
251 _mi_fprintf(out, "\n\n");
252 #endif
253 mi_stat_print(&stats->reserved, "reserved", 1, out);
254 mi_stat_print(&stats->committed, "committed", 1, out);
255 mi_stat_print(&stats->reset, "reset", 1, out);
256 mi_stat_print(&stats->page_committed, "touched", 1, out);
257 mi_stat_print(&stats->segments, "segments", -1, out);
258 mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out);
259 mi_stat_print(&stats->segments_cache, "-cached", -1, out);
260 mi_stat_print(&stats->pages, "pages", -1, out);
261 mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out);
262 mi_stat_counter_print(&stats->pages_extended, "-extended", out);
263 mi_stat_counter_print(&stats->page_no_retire, "-noretire", out);
264 mi_stat_counter_print(&stats->mmap_calls, "mmaps", out);
265 mi_stat_counter_print(&stats->commit_calls, "commits", out);
266 mi_stat_print(&stats->threads, "threads", -1, out);
267 mi_stat_counter_print_avg(&stats->searches, "searches", out);
268
269 if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs);
270
271 double user_time;
272 double sys_time;
273 size_t peak_rss;
274 size_t page_faults;
275 size_t page_reclaim;
276 size_t peak_commit;
277 mi_process_info(&user_time, &sys_time, &peak_rss, &page_faults, &page_reclaim, &peak_commit);
278 _mi_fprintf(out,"%10s: user: %.3f s, system: %.3f s, faults: %lu, reclaims: %lu, rss: ", "process", user_time, sys_time, (unsigned long)page_faults, (unsigned long)page_reclaim );
279 mi_printf_amount((int64_t)peak_rss, 1, out, "%s");
280 if (peak_commit > 0) {
281 _mi_fprintf(out,", commit charge: ");
282 mi_printf_amount((int64_t)peak_commit, 1, out, "%s");
283 }
284 _mi_fprintf(out,"\n");
285}
286
287double _mi_clock_end(double start);
288double _mi_clock_start(void);
289static double mi_time_start = 0.0;
290
291static mi_stats_t* mi_stats_get_default(void) {
292 mi_heap_t* heap = mi_heap_get_default();
293 return &heap->tld->stats;
294}
295
296static void mi_stats_merge_from(mi_stats_t* stats) {
297 if (stats != &_mi_stats_main) {
298 mi_stats_add(&_mi_stats_main, stats);
299 memset(stats, 0, sizeof(mi_stats_t));
300 }
301}
302
303void mi_stats_reset(void) mi_attr_noexcept {
304 mi_stats_t* stats = mi_stats_get_default();
305 if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); }
306 memset(&_mi_stats_main, 0, sizeof(mi_stats_t));
307 mi_time_start = _mi_clock_start();
308}
309
310void mi_stats_merge(void) mi_attr_noexcept {
311 mi_stats_merge_from( mi_stats_get_default() );
312}
313
314void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done`
315 mi_stats_merge_from(stats);
316}
317
318
319static void mi_stats_print_ex(mi_stats_t* stats, double secs, mi_output_fun* out) {
320 mi_stats_merge_from(stats);
321 _mi_stats_print(&_mi_stats_main, secs, out);
322}
323
324void mi_stats_print(mi_output_fun* out) mi_attr_noexcept {
325 mi_stats_print_ex(mi_stats_get_default(),_mi_clock_end(mi_time_start),out);
326}
327
328void mi_thread_stats_print(mi_output_fun* out) mi_attr_noexcept {
329 _mi_stats_print(mi_stats_get_default(), _mi_clock_end(mi_time_start), out);
330}
331
332
333
334// --------------------------------------------------------
335// Basic timer for convenience
336// --------------------------------------------------------
337
338#ifdef _WIN32
339#include <windows.h>
340static double mi_to_seconds(LARGE_INTEGER t) {
341 static double freq = 0.0;
342 if (freq <= 0.0) {
343 LARGE_INTEGER f;
344 QueryPerformanceFrequency(&f);
345 freq = (double)(f.QuadPart);
346 }
347 return ((double)(t.QuadPart) / freq);
348}
349
350static double mi_clock_now(void) {
351 LARGE_INTEGER t;
352 QueryPerformanceCounter(&t);
353 return mi_to_seconds(t);
354}
355#else
356#include <time.h>
357#ifdef CLOCK_REALTIME
358static double mi_clock_now(void) {
359 struct timespec t;
360 clock_gettime(CLOCK_REALTIME, &t);
361 return (double)t.tv_sec + (1.0e-9 * (double)t.tv_nsec);
362}
363#else
364// low resolution timer
365static double mi_clock_now(void) {
366 return ((double)clock() / (double)CLOCKS_PER_SEC);
367}
368#endif
369#endif
370
371
372static double mi_clock_diff = 0.0;
373
374double _mi_clock_start(void) {
375 if (mi_clock_diff == 0.0) {
376 double t0 = mi_clock_now();
377 mi_clock_diff = mi_clock_now() - t0;
378 }
379 return mi_clock_now();
380}
381
382double _mi_clock_end(double start) {
383 double end = mi_clock_now();
384 return (end - start - mi_clock_diff);
385}
386
387
388// --------------------------------------------------------
389// Basic process statistics
390// --------------------------------------------------------
391
392#if defined(_WIN32)
393#include <windows.h>
394#include <psapi.h>
395#pragma comment(lib,"psapi.lib")
396
397static double filetime_secs(const FILETIME* ftime) {
398 ULARGE_INTEGER i;
399 i.LowPart = ftime->dwLowDateTime;
400 i.HighPart = ftime->dwHighDateTime;
401 double secs = (double)(i.QuadPart) * 1.0e-7; // FILETIME is in 100 nano seconds
402 return secs;
403}
404static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) {
405 FILETIME ct;
406 FILETIME ut;
407 FILETIME st;
408 FILETIME et;
409 GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut);
410 *utime = filetime_secs(&ut);
411 *stime = filetime_secs(&st);
412
413 PROCESS_MEMORY_COUNTERS info;
414 GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
415 *peak_rss = (size_t)info.PeakWorkingSetSize;
416 *page_faults = (size_t)info.PageFaultCount;
417 *peak_commit = (size_t)info.PeakPagefileUsage;
418 *page_reclaim = 0;
419}
420
421#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__))
422#include <stdio.h>
423#include <unistd.h>
424#include <sys/resource.h>
425
426#if defined(__APPLE__) && defined(__MACH__)
427#include <mach/mach.h>
428#endif
429
430static double timeval_secs(const struct timeval* tv) {
431 return (double)tv->tv_sec + ((double)tv->tv_usec * 1.0e-6);
432}
433
434static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) {
435 struct rusage rusage;
436 getrusage(RUSAGE_SELF, &rusage);
437#if defined(__APPLE__) && defined(__MACH__)
438 *peak_rss = rusage.ru_maxrss;
439#else
440 *peak_rss = rusage.ru_maxrss * 1024;
441#endif
442 *page_faults = rusage.ru_majflt;
443 *page_reclaim = rusage.ru_minflt;
444 *peak_commit = 0;
445 *utime = timeval_secs(&rusage.ru_utime);
446 *stime = timeval_secs(&rusage.ru_stime);
447}
448
449#else
450#ifndef __wasi__
451// WebAssembly instances are not processes
452#pragma message("define a way to get process info")
453#endif
454
455static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) {
456 *peak_rss = 0;
457 *page_faults = 0;
458 *page_reclaim = 0;
459 *peak_commit = 0;
460 *utime = 0.0;
461 *stime = 0.0;
462}
463#endif
464