1 | /* |
2 | * Copyright 2016-present Facebook, Inc. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #include <folly/synchronization/AsymmetricMemoryBarrier.h> |
18 | |
19 | #include <folly/Exception.h> |
20 | #include <folly/Indestructible.h> |
21 | #include <folly/portability/SysMembarrier.h> |
22 | #include <folly/portability/SysMman.h> |
23 | #include <mutex> |
24 | |
25 | namespace folly { |
26 | |
27 | namespace { |
28 | |
29 | struct DummyPageCreator { |
30 | DummyPageCreator() { |
31 | get(); |
32 | } |
33 | |
34 | static void* get() { |
35 | static auto ptr = kIsLinux ? create() : nullptr; |
36 | return ptr; |
37 | } |
38 | |
39 | private: |
40 | static void* create() { |
41 | auto ptr = mmap(nullptr, 1, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); |
42 | checkUnixError(reinterpret_cast<ssize_t>(ptr), "mmap" ); |
43 | |
44 | // Optimistically try to lock the page so it stays resident. Could make |
45 | // the heavy barrier faster. |
46 | auto r = mlock(ptr, 1); |
47 | if (r != 0) { |
48 | // Do nothing. |
49 | } |
50 | |
51 | return ptr; |
52 | } |
53 | }; |
54 | |
55 | // Make sure dummy page is always initialized before shutdown. |
56 | DummyPageCreator dummyPageCreator; |
57 | |
58 | void mprotectMembarrier() { |
59 | auto dummyPage = dummyPageCreator.get(); |
60 | |
61 | // This function is required to be safe to call on shutdown, |
62 | // so we must leak the mutex. |
63 | static Indestructible<std::mutex> mprotectMutex; |
64 | std::lock_guard<std::mutex> lg(*mprotectMutex); |
65 | |
66 | int r = 0; |
67 | |
68 | // We want to downgrade the page while it is resident. To do that, it must |
69 | // first be upgraded and forced to be resident. |
70 | r = mprotect(dummyPage, 1, PROT_READ | PROT_WRITE); |
71 | checkUnixError(r, "mprotect" ); |
72 | |
73 | // Force the page to be resident. If it is already resident, almost no-op. |
74 | *static_cast<char*>(dummyPage) = 0; |
75 | |
76 | // Downgrade the page. Forces a memory barrier in every core running any |
77 | // of the process's threads. On a sane platform. |
78 | r = mprotect(dummyPage, 1, PROT_READ); |
79 | checkUnixError(r, "mprotect" ); |
80 | } |
81 | } // namespace |
82 | |
83 | void asymmetricHeavyBarrier(AMBFlags flags) { |
84 | if (kIsLinux) { |
85 | static const bool useSysMembarrier = detail::sysMembarrierAvailable(); |
86 | // sys_membarrier currently does not support EXPEDITED |
87 | if (useSysMembarrier && flags != AMBFlags::EXPEDITED) { |
88 | auto r = detail::sysMembarrier(); |
89 | checkUnixError(r, "membarrier" ); |
90 | } else { |
91 | mprotectMembarrier(); |
92 | } |
93 | } else { |
94 | std::atomic_thread_fence(std::memory_order_seq_cst); |
95 | } |
96 | } |
97 | } // namespace folly |
98 | |