1// Copyright 2017 The Abseil Authors.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14//
15// Each active thread has an ThreadIdentity that may represent the thread in
16// various level interfaces. ThreadIdentity objects are never deallocated.
17// When a thread terminates, its ThreadIdentity object may be reused for a
18// thread created later.
19
20#ifndef ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_
21#define ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_
22
23#ifndef _WIN32
24#include <pthread.h>
25// Defines __GOOGLE_GRTE_VERSION__ (via glibc-specific features.h) when
26// supported.
27#include <unistd.h>
28#endif
29
30#include <atomic>
31#include <cstdint>
32
33#include "absl/base/internal/per_thread_tls.h"
34
35namespace absl {
36
37struct SynchLocksHeld;
38struct SynchWaitParams;
39
40namespace base_internal {
41
42class SpinLock;
43struct ThreadIdentity;
44
45// Used by the implementation of absl::Mutex and absl::CondVar.
46struct PerThreadSynch {
47 // The internal representation of absl::Mutex and absl::CondVar rely
48 // on the alignment of PerThreadSynch. Both store the address of the
49 // PerThreadSynch in the high-order bits of their internal state,
50 // which means the low kLowZeroBits of the address of PerThreadSynch
51 // must be zero.
52 static constexpr int kLowZeroBits = 8;
53 static constexpr int kAlignment = 1 << kLowZeroBits;
54
55 // Returns the associated ThreadIdentity.
56 // This can be implemented as a cast because we guarantee
57 // PerThreadSynch is the first element of ThreadIdentity.
58 ThreadIdentity* thread_identity() {
59 return reinterpret_cast<ThreadIdentity*>(this);
60 }
61
62 PerThreadSynch *next; // Circular waiter queue; initialized to 0.
63 PerThreadSynch *skip; // If non-zero, all entries in Mutex queue
64 // up to and including "skip" have same
65 // condition as this, and will be woken later
66 bool may_skip; // if false while on mutex queue, a mutex unlocker
67 // is using this PerThreadSynch as a terminator. Its
68 // skip field must not be filled in because the loop
69 // might then skip over the terminator.
70
71 // The wait parameters of the current wait. waitp is null if the
72 // thread is not waiting. Transitions from null to non-null must
73 // occur before the enqueue commit point (state = kQueued in
74 // Enqueue() and CondVarEnqueue()). Transitions from non-null to
75 // null must occur after the wait is finished (state = kAvailable in
76 // Mutex::Block() and CondVar::WaitCommon()). This field may be
77 // changed only by the thread that describes this PerThreadSynch. A
78 // special case is Fer(), which calls Enqueue() on another thread,
79 // but with an identical SynchWaitParams pointer, thus leaving the
80 // pointer unchanged.
81 SynchWaitParams *waitp;
82
83 bool suppress_fatal_errors; // If true, try to proceed even in the face of
84 // broken invariants. This is used within fatal
85 // signal handlers to improve the chances of
86 // debug logging information being output
87 // successfully.
88
89 intptr_t readers; // Number of readers in mutex.
90 int priority; // Priority of thread (updated every so often).
91
92 // When priority will next be read (cycles).
93 int64_t next_priority_read_cycles;
94
95 // State values:
96 // kAvailable: This PerThreadSynch is available.
97 // kQueued: This PerThreadSynch is unavailable, it's currently queued on a
98 // Mutex or CondVar waistlist.
99 //
100 // Transitions from kQueued to kAvailable require a release
101 // barrier. This is needed as a waiter may use "state" to
102 // independently observe that it's no longer queued.
103 //
104 // Transitions from kAvailable to kQueued require no barrier, they
105 // are externally ordered by the Mutex.
106 enum State {
107 kAvailable,
108 kQueued
109 };
110 std::atomic<State> state;
111
112 bool maybe_unlocking; // Valid at head of Mutex waiter queue;
113 // true if UnlockSlow could be searching
114 // for a waiter to wake. Used for an optimization
115 // in Enqueue(). true is always a valid value.
116 // Can be reset to false when the unlocker or any
117 // writer releases the lock, or a reader fully releases
118 // the lock. It may not be set to false by a reader
119 // that decrements the count to non-zero.
120 // protected by mutex spinlock
121
122 bool wake; // This thread is to be woken from a Mutex.
123
124 // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the
125 // waiter is waiting on the mutex as part of a CV Wait or Mutex Await.
126 //
127 // The value of "x->cond_waiter" is meaningless if "x" is not on a
128 // Mutex waiter list.
129 bool cond_waiter;
130
131 // Locks held; used during deadlock detection.
132 // Allocated in Synch_GetAllLocks() and freed in ReclaimThreadIdentity().
133 SynchLocksHeld *all_locks;
134};
135
136struct ThreadIdentity {
137 // Must be the first member. The Mutex implementation requires that
138 // the PerThreadSynch object associated with each thread is
139 // PerThreadSynch::kAlignment aligned. We provide this alignment on
140 // ThreadIdentity itself.
141 PerThreadSynch per_thread_synch;
142
143 // Private: Reserved for absl::synchronization_internal::Waiter.
144 struct WaiterState {
145 char data[128];
146 } waiter_state;
147
148 // Used by PerThreadSem::{Get,Set}ThreadBlockedCounter().
149 std::atomic<int>* blocked_count_ptr;
150
151 // The following variables are mostly read/written just by the
152 // thread itself. The only exception is that these are read by
153 // a ticker thread as a hint.
154 std::atomic<int> ticker; // Tick counter, incremented once per second.
155 std::atomic<int> wait_start; // Ticker value when thread started waiting.
156 std::atomic<bool> is_idle; // Has thread become idle yet?
157
158 ThreadIdentity* next;
159};
160
161// Returns the ThreadIdentity object representing the calling thread; guaranteed
162// to be unique for its lifetime. The returned object will remain valid for the
163// program's lifetime; although it may be re-assigned to a subsequent thread.
164// If one does not exist, return nullptr instead.
165//
166// Does not malloc(*), and is async-signal safe.
167// [*] Technically pthread_setspecific() does malloc on first use; however this
168// is handled internally within tcmalloc's initialization already.
169//
170// New ThreadIdentity objects can be constructed and associated with a thread
171// by calling GetOrCreateCurrentThreadIdentity() in per-thread-sem.h.
172ThreadIdentity* CurrentThreadIdentityIfPresent();
173
174using ThreadIdentityReclaimerFunction = void (*)(void*);
175
176// Sets the current thread identity to the given value. 'reclaimer' is a
177// pointer to the global function for cleaning up instances on thread
178// destruction.
179void SetCurrentThreadIdentity(ThreadIdentity* identity,
180 ThreadIdentityReclaimerFunction reclaimer);
181
182// Removes the currently associated ThreadIdentity from the running thread.
183// This must be called from inside the ThreadIdentityReclaimerFunction, and only
184// from that function.
185void ClearCurrentThreadIdentity();
186
187// May be chosen at compile time via: -DABSL_FORCE_THREAD_IDENTITY_MODE=<mode
188// index>
189#ifdef ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC
190#error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be direcly set
191#else
192#define ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC 0
193#endif
194
195#ifdef ABSL_THREAD_IDENTITY_MODE_USE_TLS
196#error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be direcly set
197#else
198#define ABSL_THREAD_IDENTITY_MODE_USE_TLS 1
199#endif
200
201#ifdef ABSL_THREAD_IDENTITY_MODE_USE_CPP11
202#error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be direcly set
203#else
204#define ABSL_THREAD_IDENTITY_MODE_USE_CPP11 2
205#endif
206
207#ifdef ABSL_THREAD_IDENTITY_MODE
208#error ABSL_THREAD_IDENTITY_MODE cannot be direcly set
209#elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE)
210#define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE
211#elif defined(_WIN32)
212#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11
213#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \
214 (__GOOGLE_GRTE_VERSION__ >= 20140228L)
215// Support for async-safe TLS was specifically added in GRTEv4. It's not
216// present in the upstream eglibc.
217// Note: Current default for production systems.
218#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_TLS
219#else
220#define ABSL_THREAD_IDENTITY_MODE \
221 ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC
222#endif
223
224#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \
225 ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11
226
227extern ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr;
228
229inline ThreadIdentity* CurrentThreadIdentityIfPresent() {
230 return thread_identity_ptr;
231}
232
233#elif ABSL_THREAD_IDENTITY_MODE != \
234 ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC
235#error Unknown ABSL_THREAD_IDENTITY_MODE
236#endif
237
238} // namespace base_internal
239} // namespace absl
240
241#endif // ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_
242