1// Copyright 2017 The Abseil Authors.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14//
15// -----------------------------------------------------------------------------
16// File: call_once.h
17// -----------------------------------------------------------------------------
18//
19// This header file provides an Abseil version of `std::call_once` for invoking
20// a given function at most once, across all threads. This Abseil version is
21// faster than the C++11 version and incorporates the C++17 argument-passing
22// fix, so that (for example) non-const references may be passed to the invoked
23// function.
24
25#ifndef ABSL_BASE_CALL_ONCE_H_
26#define ABSL_BASE_CALL_ONCE_H_
27
28#include <algorithm>
29#include <atomic>
30#include <cstdint>
31#include <type_traits>
32#include <utility>
33
34#include "absl/base/internal/invoke.h"
35#include "absl/base/internal/low_level_scheduling.h"
36#include "absl/base/internal/raw_logging.h"
37#include "absl/base/internal/scheduling_mode.h"
38#include "absl/base/internal/spinlock_wait.h"
39#include "absl/base/macros.h"
40#include "absl/base/optimization.h"
41#include "absl/base/port.h"
42
43namespace absl {
44
45class once_flag;
46
47namespace base_internal {
48std::atomic<uint32_t>* ControlWord(absl::once_flag* flag);
49} // namespace base_internal
50
51// call_once()
52//
53// For all invocations using a given `once_flag`, invokes a given `fn` exactly
54// once across all threads. The first call to `call_once()` with a particular
55// `once_flag` argument (that does not throw an exception) will run the
56// specified function with the provided `args`; other calls with the same
57// `once_flag` argument will not run the function, but will wait
58// for the provided function to finish running (if it is still running).
59//
60// This mechanism provides a safe, simple, and fast mechanism for one-time
61// initialization in a multi-threaded process.
62//
63// Example:
64//
65// class MyInitClass {
66// public:
67// ...
68// mutable absl::once_flag once_;
69//
70// MyInitClass* init() const {
71// absl::call_once(once_, &MyInitClass::Init, this);
72// return ptr_;
73// }
74//
75template <typename Callable, typename... Args>
76void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args);
77
78// once_flag
79//
80// Objects of this type are used to distinguish calls to `call_once()` and
81// ensure the provided function is only invoked once across all threads. This
82// type is not copyable or movable. However, it has a `constexpr`
83// constructor, and is safe to use as a namespace-scoped global variable.
84class once_flag {
85 public:
86 constexpr once_flag() : control_(0) {}
87 once_flag(const once_flag&) = delete;
88 once_flag& operator=(const once_flag&) = delete;
89
90 private:
91 friend std::atomic<uint32_t>* base_internal::ControlWord(once_flag* flag);
92 std::atomic<uint32_t> control_;
93};
94
95//------------------------------------------------------------------------------
96// End of public interfaces.
97// Implementation details follow.
98//------------------------------------------------------------------------------
99
100namespace base_internal {
101
102// Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to
103// initialize entities used by the scheduler implementation.
104template <typename Callable, typename... Args>
105void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args);
106
107// Disables scheduling while on stack when scheduling mode is non-cooperative.
108// No effect for cooperative scheduling modes.
109class SchedulingHelper {
110 public:
111 explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) {
112 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
113 guard_result_ = base_internal::SchedulingGuard::DisableRescheduling();
114 }
115 }
116
117 ~SchedulingHelper() {
118 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
119 base_internal::SchedulingGuard::EnableRescheduling(guard_result_);
120 }
121 }
122
123 private:
124 base_internal::SchedulingMode mode_;
125 bool guard_result_;
126};
127
128// Bit patterns for call_once state machine values. Internal implementation
129// detail, not for use by clients.
130//
131// The bit patterns are arbitrarily chosen from unlikely values, to aid in
132// debugging. However, kOnceInit must be 0, so that a zero-initialized
133// once_flag will be valid for immediate use.
134enum {
135 kOnceInit = 0,
136 kOnceRunning = 0x65C2937B,
137 kOnceWaiter = 0x05A308D2,
138 // A very small constant is chosen for kOnceDone so that it fit in a single
139 // compare with immediate instruction for most common ISAs. This is verified
140 // for x86, POWER and ARM.
141 kOnceDone = 221, // Random Number
142};
143
144template <typename Callable, typename... Args>
145void CallOnceImpl(std::atomic<uint32_t>* control,
146 base_internal::SchedulingMode scheduling_mode, Callable&& fn,
147 Args&&... args) {
148#ifndef NDEBUG
149 {
150 uint32_t old_control = control->load(std::memory_order_acquire);
151 if (old_control != kOnceInit &&
152 old_control != kOnceRunning &&
153 old_control != kOnceWaiter &&
154 old_control != kOnceDone) {
155 ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx",
156 static_cast<unsigned long>(old_control)); // NOLINT
157 }
158 }
159#endif // NDEBUG
160 static const base_internal::SpinLockWaitTransition trans[] = {
161 {kOnceInit, kOnceRunning, true},
162 {kOnceRunning, kOnceWaiter, false},
163 {kOnceDone, kOnceDone, true}};
164
165 // Must do this before potentially modifying control word's state.
166 base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode);
167 // Short circuit the simplest case to avoid procedure call overhead.
168 uint32_t old_control = kOnceInit;
169 if (control->compare_exchange_strong(old_control, kOnceRunning,
170 std::memory_order_acquire,
171 std::memory_order_relaxed) ||
172 base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans,
173 scheduling_mode) == kOnceInit) {
174 base_internal::Invoke(std::forward<Callable>(fn),
175 std::forward<Args>(args)...);
176 old_control = control->load(std::memory_order_relaxed);
177 control->store(base_internal::kOnceDone, std::memory_order_release);
178 if (old_control == base_internal::kOnceWaiter) {
179 base_internal::SpinLockWake(control, true);
180 }
181 } // else *control is already kOnceDone
182}
183
184inline std::atomic<uint32_t>* ControlWord(once_flag* flag) {
185 return &flag->control_;
186}
187
188template <typename Callable, typename... Args>
189void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args) {
190 std::atomic<uint32_t>* once = base_internal::ControlWord(flag);
191 uint32_t s = once->load(std::memory_order_acquire);
192 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
193 base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY,
194 std::forward<Callable>(fn),
195 std::forward<Args>(args)...);
196 }
197}
198
199} // namespace base_internal
200
201template <typename Callable, typename... Args>
202void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) {
203 std::atomic<uint32_t>* once = base_internal::ControlWord(&flag);
204 uint32_t s = once->load(std::memory_order_acquire);
205 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
206 base_internal::CallOnceImpl(
207 once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL,
208 std::forward<Callable>(fn), std::forward<Args>(args)...);
209 }
210}
211
212} // namespace absl
213
214#endif // ABSL_BASE_CALL_ONCE_H_
215