1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#ifndef LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
9#define LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
10
11/* cxa_guard_impl.h - Implements the C++ runtime support for function local
12 * static guards.
13 * The layout of the guard object is the same across ARM and Itanium.
14 *
15 * The first "guard byte" (which is checked by the compiler) is set only upon
16 * the completion of cxa release.
17 *
18 * The second "init byte" does the rest of the bookkeeping. It tracks if
19 * initialization is complete or pending, and if there are waiting threads.
20 *
21 * If the guard variable is 64-bits and the platforms supplies a 32-bit thread
22 * identifier, it is used to detect recursive initialization. The thread ID of
23 * the thread currently performing initialization is stored in the second word.
24 *
25 * Guard Object Layout:
26 * -------------------------------------------------------------------------
27 * |a: guard byte | a+1: init byte | a+2 : unused ... | a+4: thread-id ... |
28 * ------------------------------------------------------------------------
29 *
30 * Access Protocol:
31 * For each implementation the guard byte is checked and set before accessing
32 * the init byte.
33 *
34 * Overall Design:
35 * The implementation was designed to allow each implementation to be tested
36 * independent of the C++ runtime or platform support.
37 *
38 */
39
40#include "__cxxabi_config.h"
41#include "include/atomic_support.h"
42#include <unistd.h>
43#include <sys/types.h>
44#if defined(__has_include)
45# if __has_include(<sys/syscall.h>)
46# include <sys/syscall.h>
47# endif
48#endif
49
50#include <stdlib.h>
51#include <__threading_support>
52#ifndef _LIBCXXABI_HAS_NO_THREADS
53#if defined(__unix__) && defined(__ELF__) && defined(_LIBCXXABI_HAS_COMMENT_LIB_PRAGMA)
54#pragma comment(lib, "pthread")
55#endif
56#endif
57
58// To make testing possible, this header is included from both cxa_guard.cpp
59// and a number of tests.
60//
61// For this reason we place everything in an anonymous namespace -- even though
62// we're in a header. We want the actual implementation and the tests to have
63// unique definitions of the types in this header (since the tests may depend
64// on function local statics).
65//
66// To enforce this either `BUILDING_CXA_GUARD` or `TESTING_CXA_GUARD` must be
67// defined when including this file. Only `src/cxa_guard.cpp` should define
68// the former.
69#ifdef BUILDING_CXA_GUARD
70# include "abort_message.h"
71# define ABORT_WITH_MESSAGE(...) ::abort_message(__VA_ARGS__)
72#elif defined(TESTING_CXA_GUARD)
73# define ABORT_WITH_MESSAGE(...) ::abort()
74#else
75# error "Either BUILDING_CXA_GUARD or TESTING_CXA_GUARD must be defined"
76#endif
77
78#if __has_feature(thread_sanitizer)
79extern "C" void __tsan_acquire(void*);
80extern "C" void __tsan_release(void*);
81#else
82#define __tsan_acquire(addr) ((void)0)
83#define __tsan_release(addr) ((void)0)
84#endif
85
86namespace __cxxabiv1 {
87// Use an anonymous namespace to ensure that the tests and actual implementation
88// have unique definitions of these symbols.
89namespace {
90
91//===----------------------------------------------------------------------===//
92// Misc Utilities
93//===----------------------------------------------------------------------===//
94
95template <class T, T(*Init)()>
96struct LazyValue {
97 LazyValue() : is_init(false) {}
98
99 T& get() {
100 if (!is_init) {
101 value = Init();
102 is_init = true;
103 }
104 return value;
105 }
106 private:
107 T value;
108 bool is_init = false;
109};
110
111//===----------------------------------------------------------------------===//
112// PlatformGetThreadID
113//===----------------------------------------------------------------------===//
114
115#if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
116uint32_t PlatformThreadID() {
117 static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
118 return static_cast<uint32_t>(
119 pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
120}
121#elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
122uint32_t PlatformThreadID() {
123 static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
124 return static_cast<uint32_t>(syscall(SYS_gettid));
125}
126#else
127constexpr uint32_t (*PlatformThreadID)() = nullptr;
128#endif
129
130
131constexpr bool PlatformSupportsThreadID() {
132#ifdef __clang__
133#pragma clang diagnostic push
134#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
135#endif
136 return +PlatformThreadID != nullptr;
137#ifdef __clang__
138#pragma clang diagnostic pop
139#endif
140}
141
142//===----------------------------------------------------------------------===//
143// GuardBase
144//===----------------------------------------------------------------------===//
145
146enum class AcquireResult {
147 INIT_IS_DONE,
148 INIT_IS_PENDING,
149};
150constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE;
151constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING;
152
153static constexpr uint8_t UNSET = 0;
154static constexpr uint8_t COMPLETE_BIT = (1 << 0);
155static constexpr uint8_t PENDING_BIT = (1 << 1);
156static constexpr uint8_t WAITING_BIT = (1 << 2);
157
158template <class Derived>
159struct GuardObject {
160 GuardObject() = delete;
161 GuardObject(GuardObject const&) = delete;
162 GuardObject& operator=(GuardObject const&) = delete;
163
164 explicit GuardObject(uint32_t* g)
165 : base_address(g), guard_byte_address(reinterpret_cast<uint8_t*>(g)),
166 init_byte_address(reinterpret_cast<uint8_t*>(g) + 1),
167 thread_id_address(nullptr) {}
168
169 explicit GuardObject(uint64_t* g)
170 : base_address(g), guard_byte_address(reinterpret_cast<uint8_t*>(g)),
171 init_byte_address(reinterpret_cast<uint8_t*>(g) + 1),
172 thread_id_address(reinterpret_cast<uint32_t*>(g) + 1) {}
173
174public:
175 /// Implements __cxa_guard_acquire
176 AcquireResult cxa_guard_acquire() {
177 AtomicInt<uint8_t> guard_byte(guard_byte_address);
178 if (guard_byte.load(std::_AO_Acquire) == COMPLETE_BIT)
179 return INIT_IS_DONE;
180 return derived()->acquire_init_byte();
181 }
182
183 /// Implements __cxa_guard_release
184 void cxa_guard_release() {
185 AtomicInt<uint8_t> guard_byte(guard_byte_address);
186 // Store complete first, so that when release wakes other folks, they see
187 // it as having been completed.
188 guard_byte.store(COMPLETE_BIT, std::_AO_Release);
189 derived()->release_init_byte();
190 }
191
192 /// Implements __cxa_guard_abort
193 void cxa_guard_abort() { derived()->abort_init_byte(); }
194
195public:
196 /// base_address - the address of the original guard object.
197 void* const base_address;
198 /// The address of the guord byte at offset 0.
199 uint8_t* const guard_byte_address;
200 /// The address of the byte used by the implementation during initialization.
201 uint8_t* const init_byte_address;
202 /// An optional address storing an identifier for the thread performing initialization.
203 /// It's used to detect recursive initialization.
204 uint32_t* const thread_id_address;
205
206private:
207 Derived* derived() { return static_cast<Derived*>(this); }
208};
209
210//===----------------------------------------------------------------------===//
211// Single Threaded Implementation
212//===----------------------------------------------------------------------===//
213
214struct InitByteNoThreads : GuardObject<InitByteNoThreads> {
215 using GuardObject::GuardObject;
216
217 AcquireResult acquire_init_byte() {
218 if (*init_byte_address == COMPLETE_BIT)
219 return INIT_IS_DONE;
220 if (*init_byte_address & PENDING_BIT)
221 ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
222 *init_byte_address = PENDING_BIT;
223 return INIT_IS_PENDING;
224 }
225
226 void release_init_byte() { *init_byte_address = COMPLETE_BIT; }
227 void abort_init_byte() { *init_byte_address = UNSET; }
228};
229
230
231//===----------------------------------------------------------------------===//
232// Global Mutex Implementation
233//===----------------------------------------------------------------------===//
234
235struct LibcppMutex;
236struct LibcppCondVar;
237
238#ifndef _LIBCXXABI_HAS_NO_THREADS
239struct LibcppMutex {
240 LibcppMutex() = default;
241 LibcppMutex(LibcppMutex const&) = delete;
242 LibcppMutex& operator=(LibcppMutex const&) = delete;
243
244 bool lock() { return std::__libcpp_mutex_lock(&mutex); }
245 bool unlock() { return std::__libcpp_mutex_unlock(&mutex); }
246
247private:
248 friend struct LibcppCondVar;
249 std::__libcpp_mutex_t mutex = _LIBCPP_MUTEX_INITIALIZER;
250};
251
252struct LibcppCondVar {
253 LibcppCondVar() = default;
254 LibcppCondVar(LibcppCondVar const&) = delete;
255 LibcppCondVar& operator=(LibcppCondVar const&) = delete;
256
257 bool wait(LibcppMutex& mut) {
258 return std::__libcpp_condvar_wait(&cond, &mut.mutex);
259 }
260 bool broadcast() { return std::__libcpp_condvar_broadcast(&cond); }
261
262private:
263 std::__libcpp_condvar_t cond = _LIBCPP_CONDVAR_INITIALIZER;
264};
265#else
266struct LibcppMutex {};
267struct LibcppCondVar {};
268#endif // !defined(_LIBCXXABI_HAS_NO_THREADS)
269
270
271template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
272 uint32_t (*GetThreadID)() = PlatformThreadID>
273struct InitByteGlobalMutex
274 : GuardObject<InitByteGlobalMutex<Mutex, CondVar, global_mutex, global_cond,
275 GetThreadID>> {
276
277 using BaseT = typename InitByteGlobalMutex::GuardObject;
278 using BaseT::BaseT;
279
280 explicit InitByteGlobalMutex(uint32_t *g)
281 : BaseT(g), has_thread_id_support(false) {}
282 explicit InitByteGlobalMutex(uint64_t *g)
283 : BaseT(g), has_thread_id_support(PlatformSupportsThreadID()) {}
284
285public:
286 AcquireResult acquire_init_byte() {
287 LockGuard g("__cxa_guard_acquire");
288 // Check for possible recursive initialization.
289 if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) {
290 if (*thread_id_address == current_thread_id.get())
291 ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
292 }
293
294 // Wait until the pending bit is not set.
295 while (*init_byte_address & PENDING_BIT) {
296 *init_byte_address |= WAITING_BIT;
297 global_cond.wait(global_mutex);
298 }
299
300 if (*init_byte_address == COMPLETE_BIT)
301 return INIT_IS_DONE;
302
303 if (has_thread_id_support)
304 *thread_id_address = current_thread_id.get();
305
306 *init_byte_address = PENDING_BIT;
307 return INIT_IS_PENDING;
308 }
309
310 void release_init_byte() {
311 bool has_waiting;
312 {
313 LockGuard g("__cxa_guard_release");
314 has_waiting = *init_byte_address & WAITING_BIT;
315 *init_byte_address = COMPLETE_BIT;
316 }
317 if (has_waiting) {
318 if (global_cond.broadcast()) {
319 ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_release");
320 }
321 }
322 }
323
324 void abort_init_byte() {
325 bool has_waiting;
326 {
327 LockGuard g("__cxa_guard_abort");
328 if (has_thread_id_support)
329 *thread_id_address = 0;
330 has_waiting = *init_byte_address & WAITING_BIT;
331 *init_byte_address = UNSET;
332 }
333 if (has_waiting) {
334 if (global_cond.broadcast()) {
335 ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_abort");
336 }
337 }
338 }
339
340private:
341 using BaseT::init_byte_address;
342 using BaseT::thread_id_address;
343 const bool has_thread_id_support;
344 LazyValue<uint32_t, GetThreadID> current_thread_id;
345
346private:
347 struct LockGuard {
348 LockGuard() = delete;
349 LockGuard(LockGuard const&) = delete;
350 LockGuard& operator=(LockGuard const&) = delete;
351
352 explicit LockGuard(const char* calling_func)
353 : calling_func(calling_func) {
354 if (global_mutex.lock())
355 ABORT_WITH_MESSAGE("%s failed to acquire mutex", calling_func);
356 }
357
358 ~LockGuard() {
359 if (global_mutex.unlock())
360 ABORT_WITH_MESSAGE("%s failed to release mutex", calling_func);
361 }
362
363 private:
364 const char* const calling_func;
365 };
366};
367
368//===----------------------------------------------------------------------===//
369// Futex Implementation
370//===----------------------------------------------------------------------===//
371
372#if defined(SYS_futex)
373void PlatformFutexWait(int* addr, int expect) {
374 constexpr int WAIT = 0;
375 syscall(SYS_futex, addr, WAIT, expect, 0);
376 __tsan_acquire(addr);
377}
378void PlatformFutexWake(int* addr) {
379 constexpr int WAKE = 1;
380 __tsan_release(addr);
381 syscall(SYS_futex, addr, WAKE, INT_MAX);
382}
383#else
384constexpr void (*PlatformFutexWait)(int*, int) = nullptr;
385constexpr void (*PlatformFutexWake)(int*) = nullptr;
386#endif
387
388constexpr bool PlatformSupportsFutex() {
389#ifdef __clang__
390#pragma clang diagnostic push
391#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
392#endif
393 return +PlatformFutexWait != nullptr;
394#ifdef __clang__
395#pragma clang diagnostic pop
396#endif
397}
398
399/// InitByteFutex - Manages initialization using atomics and the futex syscall
400/// for waiting and waking.
401template <void (*Wait)(int*, int) = PlatformFutexWait,
402 void (*Wake)(int*) = PlatformFutexWake,
403 uint32_t (*GetThreadIDArg)() = PlatformThreadID>
404struct InitByteFutex : GuardObject<InitByteFutex<Wait, Wake, GetThreadIDArg>> {
405 using BaseT = typename InitByteFutex::GuardObject;
406
407 /// ARM Constructor
408 explicit InitByteFutex(uint32_t *g) : BaseT(g),
409 init_byte(this->init_byte_address),
410 has_thread_id_support(this->thread_id_address && GetThreadIDArg),
411 thread_id(this->thread_id_address) {}
412
413 /// Itanium Constructor
414 explicit InitByteFutex(uint64_t *g) : BaseT(g),
415 init_byte(this->init_byte_address),
416 has_thread_id_support(this->thread_id_address && GetThreadIDArg),
417 thread_id(this->thread_id_address) {}
418
419public:
420 AcquireResult acquire_init_byte() {
421 while (true) {
422 uint8_t last_val = UNSET;
423 if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel,
424 std::_AO_Acquire)) {
425 if (has_thread_id_support) {
426 thread_id.store(current_thread_id.get(), std::_AO_Relaxed);
427 }
428 return INIT_IS_PENDING;
429 }
430
431 if (last_val == COMPLETE_BIT)
432 return INIT_IS_DONE;
433
434 if (last_val & PENDING_BIT) {
435
436 // Check for recursive initialization
437 if (has_thread_id_support && thread_id.load(std::_AO_Relaxed) == current_thread_id.get()) {
438 ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
439 }
440
441 if ((last_val & WAITING_BIT) == 0) {
442 // This compare exchange can fail for several reasons
443 // (1) another thread finished the whole thing before we got here
444 // (2) another thread set the waiting bit we were trying to thread
445 // (3) another thread had an exception and failed to finish
446 if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT,
447 std::_AO_Acq_Rel, std::_AO_Release)) {
448 // (1) success, via someone else's work!
449 if (last_val == COMPLETE_BIT)
450 return INIT_IS_DONE;
451
452 // (3) someone else, bailed on doing the work, retry from the start!
453 if (last_val == UNSET)
454 continue;
455
456 // (2) the waiting bit got set, so we are happy to keep waiting
457 }
458 }
459 wait_on_initialization();
460 }
461 }
462 }
463
464 void release_init_byte() {
465 uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel);
466 if (old & WAITING_BIT)
467 wake_all();
468 }
469
470 void abort_init_byte() {
471 if (has_thread_id_support)
472 thread_id.store(0, std::_AO_Relaxed);
473
474 uint8_t old = init_byte.exchange(0, std::_AO_Acq_Rel);
475 if (old & WAITING_BIT)
476 wake_all();
477 }
478
479private:
480 /// Use the futex to wait on the current guard variable. Futex expects a
481 /// 32-bit 4-byte aligned address as the first argument, so we have to use use
482 /// the base address of the guard variable (not the init byte).
483 void wait_on_initialization() {
484 Wait(static_cast<int*>(this->base_address),
485 expected_value_for_futex(PENDING_BIT | WAITING_BIT));
486 }
487 void wake_all() { Wake(static_cast<int*>(this->base_address)); }
488
489private:
490 AtomicInt<uint8_t> init_byte;
491
492 const bool has_thread_id_support;
493 // Unsafe to use unless has_thread_id_support
494 AtomicInt<uint32_t> thread_id;
495 LazyValue<uint32_t, GetThreadIDArg> current_thread_id;
496
497 /// Create the expected integer value for futex `wait(int* addr, int expected)`.
498 /// We pass the base address as the first argument, So this function creates
499 /// an zero-initialized integer with `b` copied at the correct offset.
500 static int expected_value_for_futex(uint8_t b) {
501 int dest_val = 0;
502 std::memcpy(reinterpret_cast<char*>(&dest_val) + 1, &b, 1);
503 return dest_val;
504 }
505
506 static_assert(Wait != nullptr && Wake != nullptr, "");
507};
508
509//===----------------------------------------------------------------------===//
510//
511//===----------------------------------------------------------------------===//
512
513template <class T>
514struct GlobalStatic {
515 static T instance;
516};
517template <class T>
518_LIBCPP_SAFE_STATIC T GlobalStatic<T>::instance = {};
519
520enum class Implementation {
521 NoThreads,
522 GlobalLock,
523 Futex
524};
525
526template <Implementation Impl>
527struct SelectImplementation;
528
529template <>
530struct SelectImplementation<Implementation::NoThreads> {
531 using type = InitByteNoThreads;
532};
533
534template <>
535struct SelectImplementation<Implementation::GlobalLock> {
536 using type = InitByteGlobalMutex<
537 LibcppMutex, LibcppCondVar, GlobalStatic<LibcppMutex>::instance,
538 GlobalStatic<LibcppCondVar>::instance, PlatformThreadID>;
539};
540
541template <>
542struct SelectImplementation<Implementation::Futex> {
543 using type =
544 InitByteFutex<PlatformFutexWait, PlatformFutexWake, PlatformThreadID>;
545};
546
547// TODO(EricWF): We should prefer the futex implementation when available. But
548// it should be done in a separate step from adding the implementation.
549constexpr Implementation CurrentImplementation =
550#if defined(_LIBCXXABI_HAS_NO_THREADS)
551 Implementation::NoThreads;
552#elif defined(_LIBCXXABI_USE_FUTEX)
553 Implementation::Futex;
554#else
555 Implementation::GlobalLock;
556#endif
557
558static_assert(CurrentImplementation != Implementation::Futex
559 || PlatformSupportsFutex(), "Futex selected but not supported");
560
561using SelectedImplementation =
562 SelectImplementation<CurrentImplementation>::type;
563
564} // end namespace
565} // end namespace __cxxabiv1
566
567#endif // LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
568