1/* Profiling of shared libraries.
2 Copyright (C) 1997-2020 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
5 Based on the BSD mcount implementation.
6
7 The GNU C Library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
11
12 The GNU C Library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; if not, see
19 <https://www.gnu.org/licenses/>. */
20
21#include <assert.h>
22#include <errno.h>
23#include <fcntl.h>
24#include <inttypes.h>
25#include <limits.h>
26#include <stdio.h>
27#include <stdlib.h>
28#include <string.h>
29#include <unistd.h>
30#include <stdint.h>
31#include <ldsodefs.h>
32#include <sys/gmon.h>
33#include <sys/gmon_out.h>
34#include <sys/mman.h>
35#include <sys/param.h>
36#include <sys/stat.h>
37#include <atomic.h>
38#include <not-cancel.h>
39
40/* The LD_PROFILE feature has to be implemented different to the
41 normal profiling using the gmon/ functions. The problem is that an
42 arbitrary amount of processes simulataneously can be run using
43 profiling and all write the results in the same file. To provide
44 this mechanism one could implement a complicated mechanism to merge
45 the content of two profiling runs or one could extend the file
46 format to allow more than one data set. For the second solution we
47 would have the problem that the file can grow in size beyond any
48 limit and both solutions have the problem that the concurrency of
49 writing the results is a big problem.
50
51 Another much simpler method is to use mmap to map the same file in
52 all using programs and modify the data in the mmap'ed area and so
53 also automatically on the disk. Using the MAP_SHARED option of
54 mmap(2) this can be done without big problems in more than one
55 file.
56
57 This approach is very different from the normal profiling. We have
58 to use the profiling data in exactly the way they are expected to
59 be written to disk. But the normal format used by gprof is not usable
60 to do this. It is optimized for size. It writes the tags as single
61 bytes but this means that the following 32/64 bit values are
62 unaligned.
63
64 Therefore we use a new format. This will look like this
65
66 0 1 2 3 <- byte is 32 bit word
67 0000 g m o n
68 0004 *version* <- GMON_SHOBJ_VERSION
69 0008 00 00 00 00
70 000c 00 00 00 00
71 0010 00 00 00 00
72
73 0014 *tag* <- GMON_TAG_TIME_HIST
74 0018 ?? ?? ?? ??
75 ?? ?? ?? ?? <- 32/64 bit LowPC
76 0018+A ?? ?? ?? ??
77 ?? ?? ?? ?? <- 32/64 bit HighPC
78 0018+2*A *histsize*
79 001c+2*A *profrate*
80 0020+2*A s e c o
81 0024+2*A n d s \0
82 0028+2*A \0 \0 \0 \0
83 002c+2*A \0 \0 \0
84 002f+2*A s
85
86 0030+2*A ?? ?? ?? ?? <- Count data
87 ... ...
88 0030+2*A+K ?? ?? ?? ??
89
90 0030+2*A+K *tag* <- GMON_TAG_CG_ARC
91 0034+2*A+K *lastused*
92 0038+2*A+K ?? ?? ?? ??
93 ?? ?? ?? ?? <- FromPC#1
94 0038+3*A+K ?? ?? ?? ??
95 ?? ?? ?? ?? <- ToPC#1
96 0038+4*A+K ?? ?? ?? ?? <- Count#1
97 ... ... ...
98 0038+(2*(CN-1)+2)*A+(CN-1)*4+K ?? ?? ?? ??
99 ?? ?? ?? ?? <- FromPC#CGN
100 0038+(2*(CN-1)+3)*A+(CN-1)*4+K ?? ?? ?? ??
101 ?? ?? ?? ?? <- ToPC#CGN
102 0038+(2*CN+2)*A+(CN-1)*4+K ?? ?? ?? ?? <- Count#CGN
103
104 We put (for now?) no basic block information in the file since this would
105 introduce rase conditions among all the processes who want to write them.
106
107 `K' is the number of count entries which is computed as
108
109 textsize / HISTFRACTION
110
111 `CG' in the above table is the number of call graph arcs. Normally,
112 the table is sparse and the profiling code writes out only the those
113 entries which are really used in the program run. But since we must
114 not extend this table (the profiling file) we'll keep them all here.
115 So CN can be executed in advance as
116
117 MINARCS <= textsize*(ARCDENSITY/100) <= MAXARCS
118
119 Now the remaining question is: how to build the data structures we can
120 work with from this data. We need the from set and must associate the
121 froms with all the associated tos. We will do this by constructing this
122 data structures at the program start. To do this we'll simply visit all
123 entries in the call graph table and add it to the appropriate list. */
124
125extern int __profile_frequency (void);
126libc_hidden_proto (__profile_frequency)
127
128/* We define a special type to address the elements of the arc table.
129 This is basically the `gmon_cg_arc_record' format but it includes
130 the room for the tag and it uses real types. */
131struct here_cg_arc_record
132 {
133 uintptr_t from_pc;
134 uintptr_t self_pc;
135 /* The count field is atomically incremented in _dl_mcount, which
136 requires it to be properly aligned for its type, and for this
137 alignment to be visible to the compiler. The amount of data
138 before an array of this structure is calculated as
139 expected_size in _dl_start_profile. Everything in that
140 calculation is a multiple of 4 bytes (in the case of
141 kcountsize, because it is derived from a subtraction of
142 page-aligned values, and the corresponding calculation in
143 __monstartup also ensures it is at least a multiple of the size
144 of u_long), so all copies of this field do in fact have the
145 appropriate alignment. */
146 uint32_t count __attribute__ ((aligned (__alignof__ (uint32_t))));
147 } __attribute__ ((packed));
148
149static struct here_cg_arc_record *data;
150
151/* Nonzero if profiling is under way. */
152static int running;
153
154/* This is the number of entry which have been incorporated in the toset. */
155static uint32_t narcs;
156/* This is a pointer to the object representing the number of entries
157 currently in the mmaped file. At no point of time this has to be the
158 same as NARCS. If it is equal all entries from the file are in our
159 lists. */
160static volatile uint32_t *narcsp;
161
162
163struct here_fromstruct
164 {
165 struct here_cg_arc_record volatile *here;
166 uint16_t link;
167 };
168
169static volatile uint16_t *tos;
170
171static struct here_fromstruct *froms;
172static uint32_t fromlimit;
173static volatile uint32_t fromidx;
174
175static uintptr_t lowpc;
176static size_t textsize;
177static unsigned int log_hashfraction;
178
179
180
181/* Set up profiling data to profile object desribed by MAP. The output
182 file is found (or created) in OUTPUT_DIR. */
183void
184_dl_start_profile (void)
185{
186 char *filename;
187 int fd;
188 struct stat64 st;
189 const ElfW(Phdr) *ph;
190 ElfW(Addr) mapstart = ~((ElfW(Addr)) 0);
191 ElfW(Addr) mapend = 0;
192 char *hist, *cp;
193 size_t idx;
194 size_t tossize;
195 size_t fromssize;
196 uintptr_t highpc;
197 uint16_t *kcount;
198 size_t kcountsize;
199 struct gmon_hdr *addr = NULL;
200 off_t expected_size;
201 /* See profil(2) where this is described. */
202 int s_scale;
203#define SCALE_1_TO_1 0x10000L
204 const char *errstr = NULL;
205
206 /* Compute the size of the sections which contain program code. */
207 for (ph = GL(dl_profile_map)->l_phdr;
208 ph < &GL(dl_profile_map)->l_phdr[GL(dl_profile_map)->l_phnum]; ++ph)
209 if (ph->p_type == PT_LOAD && (ph->p_flags & PF_X))
210 {
211 ElfW(Addr) start = (ph->p_vaddr & ~(GLRO(dl_pagesize) - 1));
212 ElfW(Addr) end = ((ph->p_vaddr + ph->p_memsz + GLRO(dl_pagesize) - 1)
213 & ~(GLRO(dl_pagesize) - 1));
214
215 if (start < mapstart)
216 mapstart = start;
217 if (end > mapend)
218 mapend = end;
219 }
220
221 /* Now we can compute the size of the profiling data. This is done
222 with the same formulars as in `monstartup' (see gmon.c). */
223 running = 0;
224 lowpc = ROUNDDOWN (mapstart + GL(dl_profile_map)->l_addr,
225 HISTFRACTION * sizeof (HISTCOUNTER));
226 highpc = ROUNDUP (mapend + GL(dl_profile_map)->l_addr,
227 HISTFRACTION * sizeof (HISTCOUNTER));
228 textsize = highpc - lowpc;
229 kcountsize = textsize / HISTFRACTION;
230 if ((HASHFRACTION & (HASHFRACTION - 1)) == 0)
231 {
232 /* If HASHFRACTION is a power of two, mcount can use shifting
233 instead of integer division. Precompute shift amount.
234
235 This is a constant but the compiler cannot compile the
236 expression away since the __ffs implementation is not known
237 to the compiler. Help the compiler by precomputing the
238 usual cases. */
239 assert (HASHFRACTION == 2);
240
241 if (sizeof (*froms) == 8)
242 log_hashfraction = 4;
243 else if (sizeof (*froms) == 16)
244 log_hashfraction = 5;
245 else
246 log_hashfraction = __ffs (HASHFRACTION * sizeof (*froms)) - 1;
247 }
248 else
249 log_hashfraction = -1;
250 tossize = textsize / HASHFRACTION;
251 fromlimit = textsize * ARCDENSITY / 100;
252 if (fromlimit < MINARCS)
253 fromlimit = MINARCS;
254 if (fromlimit > MAXARCS)
255 fromlimit = MAXARCS;
256 fromssize = fromlimit * sizeof (struct here_fromstruct);
257
258 expected_size = (sizeof (struct gmon_hdr)
259 + 4 + sizeof (struct gmon_hist_hdr) + kcountsize
260 + 4 + 4 + fromssize * sizeof (struct here_cg_arc_record));
261
262 /* Create the gmon_hdr we expect or write. */
263 struct real_gmon_hdr
264 {
265 char cookie[4];
266 int32_t version;
267 char spare[3 * 4];
268 } gmon_hdr;
269 if (sizeof (gmon_hdr) != sizeof (struct gmon_hdr)
270 || (offsetof (struct real_gmon_hdr, cookie)
271 != offsetof (struct gmon_hdr, cookie))
272 || (offsetof (struct real_gmon_hdr, version)
273 != offsetof (struct gmon_hdr, version)))
274 abort ();
275
276 memcpy (&gmon_hdr.cookie[0], GMON_MAGIC, sizeof (gmon_hdr.cookie));
277 gmon_hdr.version = GMON_SHOBJ_VERSION;
278 memset (gmon_hdr.spare, '\0', sizeof (gmon_hdr.spare));
279
280 /* Create the hist_hdr we expect or write. */
281 struct real_gmon_hist_hdr
282 {
283 char *low_pc;
284 char *high_pc;
285 int32_t hist_size;
286 int32_t prof_rate;
287 char dimen[15];
288 char dimen_abbrev;
289 } hist_hdr;
290 if (sizeof (hist_hdr) != sizeof (struct gmon_hist_hdr)
291 || (offsetof (struct real_gmon_hist_hdr, low_pc)
292 != offsetof (struct gmon_hist_hdr, low_pc))
293 || (offsetof (struct real_gmon_hist_hdr, high_pc)
294 != offsetof (struct gmon_hist_hdr, high_pc))
295 || (offsetof (struct real_gmon_hist_hdr, hist_size)
296 != offsetof (struct gmon_hist_hdr, hist_size))
297 || (offsetof (struct real_gmon_hist_hdr, prof_rate)
298 != offsetof (struct gmon_hist_hdr, prof_rate))
299 || (offsetof (struct real_gmon_hist_hdr, dimen)
300 != offsetof (struct gmon_hist_hdr, dimen))
301 || (offsetof (struct real_gmon_hist_hdr, dimen_abbrev)
302 != offsetof (struct gmon_hist_hdr, dimen_abbrev)))
303 abort ();
304
305 hist_hdr.low_pc = (char *) mapstart;
306 hist_hdr.high_pc = (char *) mapend;
307 hist_hdr.hist_size = kcountsize / sizeof (HISTCOUNTER);
308 hist_hdr.prof_rate = __profile_frequency ();
309 if (sizeof (hist_hdr.dimen) >= sizeof ("seconds"))
310 {
311 memcpy (hist_hdr.dimen, "seconds", sizeof ("seconds"));
312 memset (hist_hdr.dimen + sizeof ("seconds"), '\0',
313 sizeof (hist_hdr.dimen) - sizeof ("seconds"));
314 }
315 else
316 strncpy (hist_hdr.dimen, "seconds", sizeof (hist_hdr.dimen));
317 hist_hdr.dimen_abbrev = 's';
318
319 /* First determine the output name. We write in the directory
320 OUTPUT_DIR and the name is composed from the shared objects
321 soname (or the file name) and the ending ".profile". */
322 filename = (char *) alloca (strlen (GLRO(dl_profile_output)) + 1
323 + strlen (GLRO(dl_profile)) + sizeof ".profile");
324 cp = __stpcpy (filename, GLRO(dl_profile_output));
325 *cp++ = '/';
326 __stpcpy (__stpcpy (cp, GLRO(dl_profile)), ".profile");
327
328 fd = __open64_nocancel (filename, O_RDWR|O_CREAT|O_NOFOLLOW, DEFFILEMODE);
329 if (fd == -1)
330 {
331 char buf[400];
332 int errnum;
333
334 /* We cannot write the profiling data so don't do anything. */
335 errstr = "%s: cannot open file: %s\n";
336 print_error:
337 errnum = errno;
338 if (fd != -1)
339 __close_nocancel (fd);
340 _dl_error_printf (errstr, filename,
341 __strerror_r (errnum, buf, sizeof buf));
342 return;
343 }
344
345 if (__fxstat64 (_STAT_VER, fd, &st) < 0 || !S_ISREG (st.st_mode))
346 {
347 /* Not stat'able or not a regular file => don't use it. */
348 errstr = "%s: cannot stat file: %s\n";
349 goto print_error;
350 }
351
352 /* Test the size. If it does not match what we expect from the size
353 values in the map MAP we don't use it and warn the user. */
354 if (st.st_size == 0)
355 {
356 /* We have to create the file. */
357 char buf[GLRO(dl_pagesize)];
358
359 memset (buf, '\0', GLRO(dl_pagesize));
360
361 if (__lseek (fd, expected_size & ~(GLRO(dl_pagesize) - 1), SEEK_SET) == -1)
362 {
363 cannot_create:
364 errstr = "%s: cannot create file: %s\n";
365 goto print_error;
366 }
367
368 if (TEMP_FAILURE_RETRY
369 (__write_nocancel (fd, buf, (expected_size & (GLRO(dl_pagesize) - 1))))
370 < 0)
371 goto cannot_create;
372 }
373 else if (st.st_size != expected_size)
374 {
375 __close_nocancel (fd);
376 wrong_format:
377
378 if (addr != NULL)
379 __munmap ((void *) addr, expected_size);
380
381 _dl_error_printf ("%s: file is no correct profile data file for `%s'\n",
382 filename, GLRO(dl_profile));
383 return;
384 }
385
386 addr = (struct gmon_hdr *) __mmap (NULL, expected_size, PROT_READ|PROT_WRITE,
387 MAP_SHARED|MAP_FILE, fd, 0);
388 if (addr == (struct gmon_hdr *) MAP_FAILED)
389 {
390 errstr = "%s: cannot map file: %s\n";
391 goto print_error;
392 }
393
394 /* We don't need the file descriptor anymore. */
395 __close_nocancel (fd);
396
397 /* Pointer to data after the header. */
398 hist = (char *) (addr + 1);
399 kcount = (uint16_t *) ((char *) hist + sizeof (uint32_t)
400 + sizeof (struct gmon_hist_hdr));
401
402 /* Compute pointer to array of the arc information. */
403 narcsp = (uint32_t *) ((char *) kcount + kcountsize + sizeof (uint32_t));
404 data = (struct here_cg_arc_record *) ((char *) narcsp + sizeof (uint32_t));
405
406 if (st.st_size == 0)
407 {
408 /* Create the signature. */
409 memcpy (addr, &gmon_hdr, sizeof (struct gmon_hdr));
410
411 *(uint32_t *) hist = GMON_TAG_TIME_HIST;
412 memcpy (hist + sizeof (uint32_t), &hist_hdr,
413 sizeof (struct gmon_hist_hdr));
414
415 narcsp[-1] = GMON_TAG_CG_ARC;
416 }
417 else
418 {
419 /* Test the signature in the file. */
420 if (memcmp (addr, &gmon_hdr, sizeof (struct gmon_hdr)) != 0
421 || *(uint32_t *) hist != GMON_TAG_TIME_HIST
422 || memcmp (hist + sizeof (uint32_t), &hist_hdr,
423 sizeof (struct gmon_hist_hdr)) != 0
424 || narcsp[-1] != GMON_TAG_CG_ARC)
425 goto wrong_format;
426 }
427
428 /* Allocate memory for the froms data and the pointer to the tos records. */
429 tos = (uint16_t *) calloc (tossize + fromssize, 1);
430 if (tos == NULL)
431 {
432 __munmap ((void *) addr, expected_size);
433 _dl_fatal_printf ("Out of memory while initializing profiler\n");
434 /* NOTREACHED */
435 }
436
437 froms = (struct here_fromstruct *) ((char *) tos + tossize);
438 fromidx = 0;
439
440 /* Now we have to process all the arc count entries. BTW: it is
441 not critical whether the *NARCSP value changes meanwhile. Before
442 we enter a new entry in to toset we will check that everything is
443 available in TOS. This happens in _dl_mcount.
444
445 Loading the entries in reverse order should help to get the most
446 frequently used entries at the front of the list. */
447 for (idx = narcs = MIN (*narcsp, fromlimit); idx > 0; )
448 {
449 size_t to_index;
450 size_t newfromidx;
451 --idx;
452 to_index = (data[idx].self_pc / (HASHFRACTION * sizeof (*tos)));
453 newfromidx = fromidx++;
454 froms[newfromidx].here = &data[idx];
455 froms[newfromidx].link = tos[to_index];
456 tos[to_index] = newfromidx;
457 }
458
459 /* Setup counting data. */
460 if (kcountsize < highpc - lowpc)
461 {
462#if 0
463 s_scale = ((double) kcountsize / (highpc - lowpc)) * SCALE_1_TO_1;
464#else
465 size_t range = highpc - lowpc;
466 size_t quot = range / kcountsize;
467
468 if (quot >= SCALE_1_TO_1)
469 s_scale = 1;
470 else if (quot >= SCALE_1_TO_1 / 256)
471 s_scale = SCALE_1_TO_1 / quot;
472 else if (range > ULONG_MAX / 256)
473 s_scale = (SCALE_1_TO_1 * 256) / (range / (kcountsize / 256));
474 else
475 s_scale = (SCALE_1_TO_1 * 256) / ((range * 256) / kcountsize);
476#endif
477 }
478 else
479 s_scale = SCALE_1_TO_1;
480
481 /* Start the profiler. */
482 __profil ((void *) kcount, kcountsize, lowpc, s_scale);
483
484 /* Turn on profiling. */
485 running = 1;
486}
487
488
489void
490_dl_mcount (ElfW(Addr) frompc, ElfW(Addr) selfpc)
491{
492 volatile uint16_t *topcindex;
493 size_t i, fromindex;
494 struct here_fromstruct *fromp;
495
496 if (! running)
497 return;
498
499 /* Compute relative addresses. The shared object can be loaded at
500 any address. The value of frompc could be anything. We cannot
501 restrict it in any way, just set to a fixed value (0) in case it
502 is outside the allowed range. These calls show up as calls from
503 <external> in the gprof output. */
504 frompc -= lowpc;
505 if (frompc >= textsize)
506 frompc = 0;
507 selfpc -= lowpc;
508 if (selfpc >= textsize)
509 goto done;
510
511 /* Getting here we now have to find out whether the location was
512 already used. If yes we are lucky and only have to increment a
513 counter (this also has to be atomic). If the entry is new things
514 are getting complicated... */
515
516 /* Avoid integer divide if possible. */
517 if ((HASHFRACTION & (HASHFRACTION - 1)) == 0)
518 i = selfpc >> log_hashfraction;
519 else
520 i = selfpc / (HASHFRACTION * sizeof (*tos));
521
522 topcindex = &tos[i];
523 fromindex = *topcindex;
524
525 if (fromindex == 0)
526 goto check_new_or_add;
527
528 fromp = &froms[fromindex];
529
530 /* We have to look through the chain of arcs whether there is already
531 an entry for our arc. */
532 while (fromp->here->from_pc != frompc)
533 {
534 if (fromp->link != 0)
535 do
536 fromp = &froms[fromp->link];
537 while (fromp->link != 0 && fromp->here->from_pc != frompc);
538
539 if (fromp->here->from_pc != frompc)
540 {
541 topcindex = &fromp->link;
542
543 check_new_or_add:
544 /* Our entry is not among the entries we read so far from the
545 data file. Now see whether we have to update the list. */
546 while (narcs != *narcsp && narcs < fromlimit)
547 {
548 size_t to_index;
549 size_t newfromidx;
550 to_index = (data[narcs].self_pc
551 / (HASHFRACTION * sizeof (*tos)));
552 newfromidx = catomic_exchange_and_add (&fromidx, 1) + 1;
553 froms[newfromidx].here = &data[narcs];
554 froms[newfromidx].link = tos[to_index];
555 tos[to_index] = newfromidx;
556 catomic_increment (&narcs);
557 }
558
559 /* If we still have no entry stop searching and insert. */
560 if (*topcindex == 0)
561 {
562 uint_fast32_t newarc = catomic_exchange_and_add (narcsp, 1);
563
564 /* In rare cases it could happen that all entries in FROMS are
565 occupied. So we cannot count this anymore. */
566 if (newarc >= fromlimit)
567 goto done;
568
569 *topcindex = catomic_exchange_and_add (&fromidx, 1) + 1;
570 fromp = &froms[*topcindex];
571
572 fromp->here = &data[newarc];
573 data[newarc].from_pc = frompc;
574 data[newarc].self_pc = selfpc;
575 data[newarc].count = 0;
576 fromp->link = 0;
577 catomic_increment (&narcs);
578
579 break;
580 }
581
582 fromp = &froms[*topcindex];
583 }
584 else
585 /* Found in. */
586 break;
587 }
588
589 /* Increment the counter. */
590 catomic_increment (&fromp->here->count);
591
592 done:
593 ;
594}
595rtld_hidden_def (_dl_mcount)
596