1/*
2 * Copyright 2014-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#include <folly/fibers/FiberManagerInternal.h>
17
18#include <signal.h>
19
20#include <cassert>
21#include <stdexcept>
22
23#include <glog/logging.h>
24
25#include <folly/fibers/Fiber.h>
26#include <folly/fibers/LoopController.h>
27
28#include <folly/ConstexprMath.h>
29#include <folly/SingletonThreadLocal.h>
30#include <folly/portability/SysSyscall.h>
31#include <folly/portability/Unistd.h>
32
33#ifdef FOLLY_SANITIZE_ADDRESS
34
35#include <dlfcn.h>
36
37static void __sanitizer_start_switch_fiber_weak(
38 void** fake_stack_save,
39 void const* fiber_stack_base,
40 size_t fiber_stack_extent)
41 __attribute__((__weakref__("__sanitizer_start_switch_fiber")));
42static void __sanitizer_finish_switch_fiber_weak(
43 void* fake_stack_save,
44 void const** old_stack_base,
45 size_t* old_stack_extent)
46 __attribute__((__weakref__("__sanitizer_finish_switch_fiber")));
47static void __asan_unpoison_memory_region_weak(
48 void const /* nolint */ volatile* addr,
49 size_t size) __attribute__((__weakref__("__asan_unpoison_memory_region")));
50
51typedef void (*AsanStartSwitchStackFuncPtr)(void**, void const*, size_t);
52typedef void (*AsanFinishSwitchStackFuncPtr)(void*, void const**, size_t*);
53typedef void (*AsanUnpoisonMemoryRegionFuncPtr)(
54 void const /* nolint */ volatile*,
55 size_t);
56
57namespace folly {
58namespace fibers {
59
60static AsanStartSwitchStackFuncPtr getStartSwitchStackFunc();
61static AsanFinishSwitchStackFuncPtr getFinishSwitchStackFunc();
62static AsanUnpoisonMemoryRegionFuncPtr getUnpoisonMemoryRegionFunc();
63} // namespace fibers
64} // namespace folly
65
66#endif
67
68namespace folly {
69namespace fibers {
70
71FOLLY_TLS FiberManager* FiberManager::currentFiberManager_ = nullptr;
72
73FiberManager::FiberManager(
74 std::unique_ptr<LoopController> loopController,
75 Options options)
76 : FiberManager(
77 LocalType<void>(),
78 std::move(loopController),
79 std::move(options)) {}
80
81FiberManager::~FiberManager() {
82 loopController_.reset();
83
84 while (!fibersPool_.empty()) {
85 fibersPool_.pop_front_and_dispose([](Fiber* fiber) { delete fiber; });
86 }
87 assert(readyFibers_.empty());
88 assert(fibersActive_ == 0);
89}
90
91LoopController& FiberManager::loopController() {
92 return *loopController_;
93}
94
95const LoopController& FiberManager::loopController() const {
96 return *loopController_;
97}
98
99bool FiberManager::hasTasks() const {
100 return fibersActive_ > 0 || !remoteReadyQueue_.empty() ||
101 !remoteTaskQueue_.empty() || remoteCount_ > 0;
102}
103
104Fiber* FiberManager::getFiber() {
105 Fiber* fiber = nullptr;
106
107 if (options_.fibersPoolResizePeriodMs > 0 && !fibersPoolResizerScheduled_) {
108 fibersPoolResizer_.run();
109 fibersPoolResizerScheduled_ = true;
110 }
111
112 if (fibersPool_.empty()) {
113 fiber = new Fiber(*this);
114 ++fibersAllocated_;
115 } else {
116 fiber = &fibersPool_.front();
117 fibersPool_.pop_front();
118 assert(fibersPoolSize_ > 0);
119 --fibersPoolSize_;
120 }
121 assert(fiber);
122 if (++fibersActive_ > maxFibersActiveLastPeriod_) {
123 maxFibersActiveLastPeriod_ = fibersActive_;
124 }
125 ++fiberId_;
126 bool recordStack = (options_.recordStackEvery != 0) &&
127 (fiberId_ % options_.recordStackEvery == 0);
128 fiber->init(recordStack);
129 return fiber;
130}
131
132void FiberManager::setExceptionCallback(FiberManager::ExceptionCallback ec) {
133 assert(ec);
134 exceptionCallback_ = std::move(ec);
135}
136
137size_t FiberManager::fibersAllocated() const {
138 return fibersAllocated_;
139}
140
141size_t FiberManager::fibersPoolSize() const {
142 return fibersPoolSize_;
143}
144
145size_t FiberManager::stackHighWatermark() const {
146 return stackHighWatermark_;
147}
148
149void FiberManager::remoteReadyInsert(Fiber* fiber) {
150 if (observer_) {
151 observer_->runnable(reinterpret_cast<uintptr_t>(fiber));
152 }
153 if (remoteReadyQueue_.insertHead(fiber)) {
154 loopController_->scheduleThreadSafe();
155 }
156}
157
158void FiberManager::setObserver(ExecutionObserver* observer) {
159 observer_ = observer;
160}
161
162ExecutionObserver* FiberManager::getObserver() {
163 return observer_;
164}
165
166void FiberManager::setPreemptRunner(InlineFunctionRunner* preemptRunner) {
167 preemptRunner_ = preemptRunner;
168}
169
170void FiberManager::doFibersPoolResizing() {
171 while (fibersAllocated_ > maxFibersActiveLastPeriod_ &&
172 fibersPoolSize_ > options_.maxFibersPoolSize) {
173 auto fiber = &fibersPool_.front();
174 assert(fiber != nullptr);
175 fibersPool_.pop_front();
176 delete fiber;
177 --fibersPoolSize_;
178 --fibersAllocated_;
179 }
180
181 maxFibersActiveLastPeriod_ = fibersActive_;
182}
183
184void FiberManager::FibersPoolResizer::run() {
185 fiberManager_.doFibersPoolResizing();
186 fiberManager_.loopController_->timer().scheduleTimeout(
187 this,
188 std::chrono::milliseconds(
189 fiberManager_.options_.fibersPoolResizePeriodMs));
190}
191
192#ifdef FOLLY_SANITIZE_ADDRESS
193
194void FiberManager::registerStartSwitchStackWithAsan(
195 void** saveFakeStack,
196 const void* stackBottom,
197 size_t stackSize) {
198 // Check if we can find a fiber enter function and call it if we find one
199 static AsanStartSwitchStackFuncPtr fn = getStartSwitchStackFunc();
200 if (fn == nullptr) {
201 LOG(FATAL) << "The version of ASAN in use doesn't support fibers";
202 } else {
203 fn(saveFakeStack, stackBottom, stackSize);
204 }
205}
206
207void FiberManager::registerFinishSwitchStackWithAsan(
208 void* saveFakeStack,
209 const void** saveStackBottom,
210 size_t* saveStackSize) {
211 // Check if we can find a fiber exit function and call it if we find one
212 static AsanFinishSwitchStackFuncPtr fn = getFinishSwitchStackFunc();
213 if (fn == nullptr) {
214 LOG(FATAL) << "The version of ASAN in use doesn't support fibers";
215 } else {
216 fn(saveFakeStack, saveStackBottom, saveStackSize);
217 }
218}
219
220void FiberManager::freeFakeStack(void* fakeStack) {
221 static AsanStartSwitchStackFuncPtr fnStart = getStartSwitchStackFunc();
222 static AsanFinishSwitchStackFuncPtr fnFinish = getFinishSwitchStackFunc();
223 if (fnStart == nullptr || fnFinish == nullptr) {
224 LOG(FATAL) << "The version of ASAN in use doesn't support fibers";
225 }
226
227 void* saveFakeStack;
228 const void* stackBottom;
229 size_t stackSize;
230 fnStart(&saveFakeStack, nullptr, 0);
231 fnFinish(fakeStack, &stackBottom, &stackSize);
232 fnStart(nullptr, stackBottom, stackSize);
233 fnFinish(saveFakeStack, nullptr, nullptr);
234}
235
236void FiberManager::unpoisonFiberStack(const Fiber* fiber) {
237 auto stack = fiber->getStack();
238
239 // Check if we can find a fiber enter function and call it if we find one
240 static AsanUnpoisonMemoryRegionFuncPtr fn = getUnpoisonMemoryRegionFunc();
241 if (fn == nullptr) {
242 LOG(FATAL) << "This version of ASAN doesn't support memory unpoisoning";
243 } else {
244 fn(stack.first, stack.second);
245 }
246}
247
248static AsanStartSwitchStackFuncPtr getStartSwitchStackFunc() {
249 AsanStartSwitchStackFuncPtr fn{nullptr};
250
251 // Check whether weak reference points to statically linked enter function
252 if (nullptr != (fn = &::__sanitizer_start_switch_fiber_weak)) {
253 return fn;
254 }
255
256 // Check whether we can find a dynamically linked enter function
257 if (nullptr !=
258 (fn = (AsanStartSwitchStackFuncPtr)dlsym(
259 RTLD_DEFAULT, "__sanitizer_start_switch_fiber"))) {
260 return fn;
261 }
262
263 // Couldn't find the function at all
264 return nullptr;
265}
266
267static AsanFinishSwitchStackFuncPtr getFinishSwitchStackFunc() {
268 AsanFinishSwitchStackFuncPtr fn{nullptr};
269
270 // Check whether weak reference points to statically linked exit function
271 if (nullptr != (fn = &::__sanitizer_finish_switch_fiber_weak)) {
272 return fn;
273 }
274
275 // Check whether we can find a dynamically linked exit function
276 if (nullptr !=
277 (fn = (AsanFinishSwitchStackFuncPtr)dlsym(
278 RTLD_DEFAULT, "__sanitizer_finish_switch_fiber"))) {
279 return fn;
280 }
281
282 // Couldn't find the function at all
283 return nullptr;
284}
285
286static AsanUnpoisonMemoryRegionFuncPtr getUnpoisonMemoryRegionFunc() {
287 AsanUnpoisonMemoryRegionFuncPtr fn{nullptr};
288
289 // Check whether weak reference points to statically linked unpoison function
290 if (nullptr != (fn = &::__asan_unpoison_memory_region_weak)) {
291 return fn;
292 }
293
294 // Check whether we can find a dynamically linked unpoison function
295 if (nullptr !=
296 (fn = (AsanUnpoisonMemoryRegionFuncPtr)dlsym(
297 RTLD_DEFAULT, "__asan_unpoison_memory_region"))) {
298 return fn;
299 }
300
301 // Couldn't find the function at all
302 return nullptr;
303}
304
305#endif // FOLLY_SANITIZE_ADDRESS
306
307#ifndef _WIN32
308namespace {
309
310// SIGSTKSZ (8 kB on our architectures) isn't always enough for
311// folly::symbolizer, so allocate 32 kB.
312constexpr size_t kAltStackSize = folly::constexpr_max(SIGSTKSZ, 32 * 1024);
313
314bool hasAlternateStack() {
315 stack_t ss;
316 sigaltstack(nullptr, &ss);
317 return !(ss.ss_flags & SS_DISABLE);
318}
319
320int setAlternateStack(char* sp, size_t size) {
321 CHECK(sp);
322 stack_t ss{};
323 ss.ss_sp = sp;
324 ss.ss_size = size;
325 return sigaltstack(&ss, nullptr);
326}
327
328int unsetAlternateStack() {
329 stack_t ss{};
330 ss.ss_flags = SS_DISABLE;
331 return sigaltstack(&ss, nullptr);
332}
333
334class ScopedAlternateSignalStack {
335 public:
336 ScopedAlternateSignalStack() {
337 if (hasAlternateStack()) {
338 return;
339 }
340
341 stack_ = std::make_unique<AltStackBuffer>();
342
343 setAlternateStack(stack_->data(), stack_->size());
344 }
345
346 ScopedAlternateSignalStack(ScopedAlternateSignalStack&&) = default;
347 ScopedAlternateSignalStack& operator=(ScopedAlternateSignalStack&&) = default;
348
349 ~ScopedAlternateSignalStack() {
350 if (stack_) {
351 unsetAlternateStack();
352 }
353 }
354
355 private:
356 using AltStackBuffer = std::array<char, kAltStackSize>;
357 std::unique_ptr<AltStackBuffer> stack_;
358};
359} // namespace
360
361void FiberManager::registerAlternateSignalStack() {
362 SingletonThreadLocal<ScopedAlternateSignalStack>::get();
363
364 alternateSignalStackRegistered_ = true;
365}
366#endif
367} // namespace fibers
368} // namespace folly
369