1/*
2 * Copyright 2016-present Facebook, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include <folly/synchronization/AsymmetricMemoryBarrier.h>
18
19#include <folly/Exception.h>
20#include <folly/Indestructible.h>
21#include <folly/portability/SysMembarrier.h>
22#include <folly/portability/SysMman.h>
23#include <mutex>
24
25namespace folly {
26
27namespace {
28
29struct DummyPageCreator {
30 DummyPageCreator() {
31 get();
32 }
33
34 static void* get() {
35 static auto ptr = kIsLinux ? create() : nullptr;
36 return ptr;
37 }
38
39 private:
40 static void* create() {
41 auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
42 checkUnixError(reinterpret_cast<ssize_t>(ptr), "mmap");
43
44 // Optimistically try to lock the page so it stays resident. Could make
45 // the heavy barrier faster.
46 auto r = mlock(ptr, 1);
47 if (r != 0) {
48 // Do nothing.
49 }
50
51 return ptr;
52 }
53};
54
55// Make sure dummy page is always initialized before shutdown.
56DummyPageCreator dummyPageCreator;
57
58void mprotectMembarrier() {
59 auto dummyPage = dummyPageCreator.get();
60
61 // This function is required to be safe to call on shutdown,
62 // so we must leak the mutex.
63 static Indestructible<std::mutex> mprotectMutex;
64 std::lock_guard<std::mutex> lg(*mprotectMutex);
65
66 int r = 0;
67
68 // We want to downgrade the page while it is resident. To do that, it must
69 // first be upgraded and forced to be resident.
70 r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE);
71 checkUnixError(r, "mprotect");
72
73 // Force the page to be resident. If it is already resident, almost no-op.
74 *static_cast<char*>(dummyPage) = 0;
75
76 // Downgrade the page. Forces a memory barrier in every core running any
77 // of the process's threads. On a sane platform.
78 r = mprotect(dummyPage, 1, PROT_READ);
79 checkUnixError(r, "mprotect");
80}
81} // namespace
82
83void asymmetricHeavyBarrier(AMBFlags flags) {
84 if (kIsLinux) {
85 static const bool useSysMembarrier = detail::sysMembarrierAvailable();
86 // sys_membarrier currently does not support EXPEDITED
87 if (useSysMembarrier && flags != AMBFlags::EXPEDITED) {
88 auto r = detail::sysMembarrier();
89 checkUnixError(r, "membarrier");
90 } else {
91 mprotectMembarrier();
92 }
93 } else {
94 std::atomic_thread_fence(std::memory_order_seq_cst);
95 }
96}
97} // namespace folly
98